<commit_before>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"net/http"
	"strings"
	"sync"
	"time"

	"github.com/prometheus/client_golang/prometheus"
)

type collectdMetric struct {
	Values         []float64
	Dstypes        []string
	Dsnames        []string
	Time           float64
	Interval       float64
	Host           string
	Plugin         string
	PluginInstance string `json:"plugin_instance"`
	Type           string
	TypeInstance   string `json:"type_instance"`
}

var (
	addr     = flag.String("listen-address", ":6060", "The address to listen on for HTTP requests.")
	lastPush = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Name: "collectd_last_push_timestamp_seconds",
			Help: "Unix timestamp of the last received collectd metrics push in seconds.",
		},
	)
)

func metricName(m collectdMetric, dstype string, dsname string) string {
	result := "collectd"
	if m.Plugin != m.Type && !strings.HasPrefix(m.Type, m.Plugin+"_") {
		result += "_" + m.Plugin
	}
	result += "_" + m.Type
	if dsname != "value" {
		result += "_" + dsname
	}
	if dstype == "counter" {
		result += "_total"
	}
	return result
}

func metricHelp(m collectdMetric, dstype string, dsname string) string {
	return fmt.Sprintf("Collectd exporter: '%s' Type: '%s' Dstype: '%s' Dsname: '%s'",
		m.Plugin, m.Type, dstype, dsname)
}

func metricType(dstype string) prometheus.ValueType {
	if dstype == "counter" {
		return prometheus.CounterValue
	}
	return prometheus.GaugeValue
}

type collectdSample struct {
	Name    string
	Labels  map[string]string
	Help    string
	Value   float64
	Type    prometheus.ValueType
	Expires time.Time
}

type collectdSampleLabelset struct {
	Name           string
	Instance       string
	Type           string
	Plugin         string
	PluginInstance string
}

type collectdCollector struct {
	samples map[collectdSampleLabelset]*collectdSample
	mu      *sync.Mutex
	ch      chan *collectdSample
}

func newCollectdCollector() *collectdCollector {
	c := &collectdCollector{
		ch:      make(chan *collectdSample, 0),
		mu:      &sync.Mutex{},
		samples: map[collectdSampleLabelset]*collectdSample{},
	}
	go c.processSamples()
	return c
}

func (c *collectdCollector) collectdPost(w http.ResponseWriter, r *http.Request) {
	var postedMetrics []collectdMetric
	err := json.NewDecoder(r.Body).Decode(&postedMetrics)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	now := time.Now()
	lastPush.Set(float64(now.UnixNano()) / 1e9)
	for _, metric := range postedMetrics {
		if len(metric.Values) != len(metric.Dstypes) || len(metric.Values) != len(metric.Dsnames) {
			w.WriteHeader(http.StatusBadRequest)
			return
		}
		for i, value := range metric.Values {
			labels := prometheus.Labels{}
			if metric.PluginInstance != "" {
				labels[metric.Plugin] = metric.PluginInstance
			}
			if metric.TypeInstance != "" {
				if metric.PluginInstance == "" {
					labels[metric.Plugin] = metric.TypeInstance
				} else {
					labels["type"] = metric.TypeInstance
				}
			}
			labels["instance"] = metric.Host
			c.ch <- &collectdSample{
				Name:    metricName(metric, metric.Dstypes[i], metric.Dsnames[i]),
				Labels:  labels,
				Help:    metricHelp(metric, metric.Dstypes[i], metric.Dsnames[i]),
				Value:   value,
				Type:    metricType(metric.Dstypes[i]),
				Expires: now.Add(time.Duration(metric.Interval) * time.Second * 2),
			}
		}
	}
}

func (c *collectdCollector) processSamples() {
	ticker := time.NewTicker(time.Minute).C
	for {
		select {
		case sample := <-c.ch:
			labelset := collectdSampleLabelset{
				Name: sample.Name,
			}
			for k, v := range sample.Labels {
				switch k {
				case "instance":
					labelset.Instance = v
				case "type":
					labelset.Type = v
				default:
					labelset.Plugin = k
					labelset.PluginInstance = v
				}
			}
			c.mu.Lock()
			c.samples[labelset] = sample
			c.mu.Unlock()
		case <-ticker:
			// Garbage collect expired samples.
			now := time.Now()
			c.mu.Lock()
			for k, sample := range c.samples {
				if now.After(sample.Expires) {
					delete(c.samples, k)
				}
			}
			c.mu.Unlock()
		}
	}
}

// Collect implements prometheus.Collector.
func (c collectdCollector) Collect(ch chan<- prometheus.Metric) {
	ch <- lastPush
	c.mu.Lock()
	samples := c.samples
	c.mu.Unlock()
	now := time.Now()
	for _, sample := range samples {
		if now.After(sample.Expires) {
			continue
		}
		ch <- prometheus.MustNewConstMetric(
			prometheus.NewDesc(sample.Name, sample.Help, []string{}, sample.Labels),
			sample.Type,
			sample.Value,
		)
	}
}

// Describe implements prometheus.Collector.
func (c collectdCollector) Describe(ch chan<- *prometheus.Desc) {
	ch <- lastPush.Desc()
}

func main() {
	flag.Parse()
	http.Handle("/metrics", prometheus.Handler())
	c := newCollectdCollector()
	http.HandleFunc("/collectd-post", c.collectdPost)
	prometheus.MustRegister(c)
	http.ListenAndServe(*addr, nil)
}
<commit_msg>Make sample read access thread-safe in Collect().<commit_after>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"net/http"
	"strings"
	"sync"
	"time"

	"github.com/prometheus/client_golang/prometheus"
)

type collectdMetric struct {
	Values         []float64
	Dstypes        []string
	Dsnames        []string
	Time           float64
	Interval       float64
	Host           string
	Plugin         string
	PluginInstance string `json:"plugin_instance"`
	Type           string
	TypeInstance   string `json:"type_instance"`
}

var (
	addr     = flag.String("listen-address", ":6060", "The address to listen on for HTTP requests.")
	lastPush = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Name: "collectd_last_push_timestamp_seconds",
			Help: "Unix timestamp of the last received collectd metrics push in seconds.",
		},
	)
)

func metricName(m collectdMetric, dstype string, dsname string) string {
	result := "collectd"
	if m.Plugin != m.Type && !strings.HasPrefix(m.Type, m.Plugin+"_") {
		result += "_" + m.Plugin
	}
	result += "_" + m.Type
	if dsname != "value" {
		result += "_" + dsname
	}
	if dstype == "counter" {
		result += "_total"
	}
	return result
}

func metricHelp(m collectdMetric, dstype string, dsname string) string {
	return fmt.Sprintf("Collectd exporter: '%s' Type: '%s' Dstype: '%s' Dsname: '%s'",
		m.Plugin, m.Type, dstype, dsname)
}

func metricType(dstype string) prometheus.ValueType {
	if dstype == "counter" {
		return prometheus.CounterValue
	}
	return prometheus.GaugeValue
}

type collectdSample struct {
	Name    string
	Labels  map[string]string
	Help    string
	Value   float64
	Type    prometheus.ValueType
	Expires time.Time
}

type collectdSampleLabelset struct {
	Name           string
	Instance       string
	Type           string
	Plugin         string
	PluginInstance string
}

type collectdCollector struct {
	samples map[collectdSampleLabelset]*collectdSample
	mu      *sync.Mutex
	ch      chan *collectdSample
}

func newCollectdCollector() *collectdCollector {
	c := &collectdCollector{
		ch:      make(chan *collectdSample, 0),
		mu:      &sync.Mutex{},
		samples: map[collectdSampleLabelset]*collectdSample{},
	}
	go c.processSamples()
	return c
}

func (c *collectdCollector) collectdPost(w http.ResponseWriter, r *http.Request) {
	var postedMetrics []collectdMetric
	err := json.NewDecoder(r.Body).Decode(&postedMetrics)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	now := time.Now()
	lastPush.Set(float64(now.UnixNano()) / 1e9)
	for _, metric := range postedMetrics {
		if len(metric.Values) != len(metric.Dstypes) || len(metric.Values) != len(metric.Dsnames) {
			w.WriteHeader(http.StatusBadRequest)
			return
		}
		for i, value := range metric.Values {
			labels := prometheus.Labels{}
			if metric.PluginInstance != "" {
				labels[metric.Plugin] = metric.PluginInstance
			}
			if metric.TypeInstance != "" {
				if metric.PluginInstance == "" {
					labels[metric.Plugin] = metric.TypeInstance
				} else {
					labels["type"] = metric.TypeInstance
				}
			}
			labels["instance"] = metric.Host
			c.ch <- &collectdSample{
				Name:    metricName(metric, metric.Dstypes[i], metric.Dsnames[i]),
				Labels:  labels,
				Help:    metricHelp(metric, metric.Dstypes[i], metric.Dsnames[i]),
				Value:   value,
				Type:    metricType(metric.Dstypes[i]),
				Expires: now.Add(time.Duration(metric.Interval) * time.Second * 2),
			}
		}
	}
}

func (c *collectdCollector) processSamples() {
	ticker := time.NewTicker(time.Minute).C
	for {
		select {
		case sample := <-c.ch:
			labelset := collectdSampleLabelset{
				Name: sample.Name,
			}
			for k, v := range sample.Labels {
				switch k {
				case "instance":
					labelset.Instance = v
				case "type":
					labelset.Type = v
				default:
					labelset.Plugin = k
					labelset.PluginInstance = v
				}
			}
			c.mu.Lock()
			c.samples[labelset] = sample
			c.mu.Unlock()
		case <-ticker:
			// Garbage collect expired samples.
			now := time.Now()
			c.mu.Lock()
			for k, sample := range c.samples {
				if now.After(sample.Expires) {
					delete(c.samples, k)
				}
			}
			c.mu.Unlock()
		}
	}
}

// Collect implements prometheus.Collector.
func (c collectdCollector) Collect(ch chan<- prometheus.Metric) {
	ch <- lastPush

	c.mu.Lock()
	samples := make([]*collectdSample, len(c.samples))
	for _, sample := range c.samples {
		samples = append(samples, sample)
	}
	c.mu.Unlock()

	now := time.Now()
	for _, sample := range samples {
		if now.After(sample.Expires) {
			continue
		}
		ch <- prometheus.MustNewConstMetric(
			prometheus.NewDesc(sample.Name, sample.Help, []string{}, sample.Labels),
			sample.Type,
			sample.Value,
		)
	}
}

// Describe implements prometheus.Collector.
func (c collectdCollector) Describe(ch chan<- *prometheus.Desc) {
	ch <- lastPush.Desc()
}

func main() {
	flag.Parse()
	http.Handle("/metrics", prometheus.Handler())
	c := newCollectdCollector()
	http.HandleFunc("/collectd-post", c.collectdPost)
	prometheus.MustRegister(c)
	http.ListenAndServe(*addr, nil)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"os"
	"fmt"
	"path/filepath"
	"github.com/LapisBot/Lapislazuli/bot"
	"github.com/LapisBot/Lapislazuli/config"
)

const Name = "Lapislazuli v0.1dev"

func main() {
	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "Available flags:")

		// Print defaults
		flag.CommandLine.VisitAll(func(f *flag.Flag) {
			fmt.Fprintf(os.Stderr, "  -%s | %s - (%s)\n", f.Name, f.Usage, f.DefValue)
		})
	}

	// The folder to save the Bot files in
	dir, err := os.Getwd(); assert(err)
	flag.StringVar(&dir, "dir", dir, "Set the folder for all the Bot files")

	configFile := "config.json"
	flag.StringVar(&configFile, "config", configFile, "The file to load the configuration from")

	// Parse the flags given when running the application
	flag.Parse()

	// We have the parsed flags available now, load the configuration
	configFile = filepath.Join(dir, configFile)
	fmt.Println("Loading configuration from", configFile)
	conf := loadConfigFile(configFile)

	// Launch the bot
	fmt.Println("Starting", Name)
	bot.Create(conf).Start()
}

// This method should always complete successfully. If it doesn't, then something is really wrong and we
// shouldn't continue execution.
func assert(err error) {
	if err != nil {
		panic(err)
	}
}

// This method will exist the program if an error occurs and print the error message. Unlike assert() this
// is not that bad and won't panic the complete program.
func require(err error) bool {
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(2)
	}
	return true
}

// Loads the configuration file from the specified file or creates a new empty one if it doesn't exist.
func loadConfigFile(path string) (conf *config.Config) {
	// Try opening the configuration file first so we can check if it exists
	file, err := os.Open(path)

	if os.IsNotExist(err) {
		// Create a new configuration file
		file, err = os.Create(path); require(err)
		defer file.Close()

		// Default values
		conf = config.New()
		conf.Servers["irc.example.com:+6697"] = config.NewServer()

		// Write the default configuration to the file
		require(config.Write(file, conf))
	} else {
		require(err)
		defer file.Close()

		// Read the configuration from the file
		conf, err = config.Read(file); require(err)
	}

	return
}
<commit_msg>Remove unneeded bool return value<commit_after>package main

import (
	"flag"
	"os"
	"fmt"
	"path/filepath"
	"github.com/LapisBot/Lapislazuli/bot"
	"github.com/LapisBot/Lapislazuli/config"
)

const Name = "Lapislazuli v0.1dev"

func main() {
	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "Available flags:")

		// Print defaults
		flag.CommandLine.VisitAll(func(f *flag.Flag) {
			fmt.Fprintf(os.Stderr, "  -%s | %s - (%s)\n", f.Name, f.Usage, f.DefValue)
		})
	}

	// The folder to save the Bot files in
	dir, err := os.Getwd(); assert(err)
	flag.StringVar(&dir, "dir", dir, "Set the folder for all the Bot files")

	configFile := "config.json"
	flag.StringVar(&configFile, "config", configFile, "The file to load the configuration from")

	// Parse the flags given when running the application
	flag.Parse()

	// We have the parsed flags available now, load the configuration
	configFile = filepath.Join(dir, configFile)
	fmt.Println("Loading configuration from", configFile)
	conf := loadConfigFile(configFile)

	// Launch the bot
	fmt.Println("Starting", Name)
	bot.Create(conf).Start()
}

// This method should always complete successfully. If it doesn't, then something is really wrong and we
// shouldn't continue execution.
func assert(err error) {
	if err != nil {
		panic(err)
	}
}

// This method will exist the program if an error occurs and print the error message. Unlike assert() this
// is not that bad and won't panic the complete program.
func require(err error) {
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(2)
	}
}

// Loads the configuration file from the specified file or creates a new empty one if it doesn't exist.
func loadConfigFile(path string) (conf *config.Config) {
	// Try opening the configuration file first so we can check if it exists
	file, err := os.Open(path)

	if os.IsNotExist(err) {
		// Create a new configuration file
		file, err = os.Create(path); require(err)
		defer file.Close()

		// Default values
		conf = config.New()
		conf.Servers["irc.example.com:+6697"] = config.NewServer()

		// Write the default configuration to the file
		require(config.Write(file, conf))
	} else {
		require(err)
		defer file.Close()

		// Read the configuration from the file
		conf, err = config.Read(file); require(err)
	}

	return
}
<|endoftext|>


<commit_before>package main

import (
	whisper "github.com/grobian/go-whisper"
	pickle "github.com/kisielk/og-rek"

	"net/http"
	"encoding/json"
	"path/filepath"
	"strings"
	"strconv"
	"math"
	"fmt"
)

var config = struct {
	WhisperData	string
}{
	WhisperData: "/var/lib/carbon/whisper",
	//WhisperData: "..",
}

type WhisperFetchResponse struct {
	Name        string      `json:"name"`
	StartTime   int         `json:"startTime"`
	StopTime    int         `json:"stopTime"`
	StepTime    int         `json:"stepTime"`
	Values      []float64   `json:"values"`
	IsAbsent    []bool      `json:"isAbsent"`
}

type WhisperGlobResponse struct {
	Name        string      `json:"name"`
	Paths       []string    `json:"paths"`
}

func findHandler(wr http.ResponseWriter, req *http.Request) {
//	GET /metrics/find/?local=1&format=pickle&query=general.hadoop.lhr4.ha201jobtracker-01.jobtracker.NonHeapMemoryUsage.committed HTTP/1.1
//	http://localhost:8080/metrics/find/?query=test
	req.ParseForm()
	glob := req.FormValue("query")
	format := req.FormValue("format")

	if format != "json" && format != "pickle" {
		fmt.Printf("dropping invalid uri (format=%s): %s\n",
				format, req.URL.RequestURI())
		return
	}

	/* things to glob:
	 * - carbon.relays  -> carbon.relays
	 * - carbon.re      -> carbon.relays, carbon.rewhatever
	 * - carbon.[rz]    -> carbon.relays, carbon.zipper
	 * - carbon.{re,zi} -> carbon.relays, carbon.zipper
	 * - implicit * at the end of each query
	 * - match is either dir or .wsp file
	 * unfortunately, filepath.Glob doesn't handle the curly brace
	 * expansion for us */
	lbrace := strings.Index(glob, "{")
	rbrace := -1
	if lbrace > -1 {
		rbrace = strings.Index(glob[lbrace:], "}")
		if rbrace > -1 {
			rbrace += lbrace
		}
	}
	files := make([]string, 0)
	if lbrace > -1 && rbrace > -1 {
		expansion := glob[lbrace + 1:rbrace]
		parts := strings.Split(expansion, ",")
		for _, sub := range parts {
			sglob := glob[:lbrace] + sub + glob[rbrace + 1:]
			path := config.WhisperData + "/" + strings.Replace(sglob, ".", "/", -1) + "*"
			nfiles, err := filepath.Glob(path)
			if err == nil {
				files = append(files, nfiles...)
			}
		}
	} else {
		path := config.WhisperData + "/" + strings.Replace(glob, ".", "/", -1) + "*"
		nfiles, err := filepath.Glob(path)
		if err == nil {
			files = append(files, nfiles...)
		}
	}

	leafs := make([]bool, len(files))
	for i, p := range files {
		p = p[len(config.WhisperData + "/"):]
		if strings.HasSuffix(p, ".wsp") {
			p = p[:len(p) - 4]
			leafs[i] = true
		} else {
			leafs[i] = false
		}
		files[i] = strings.Replace(p, "/", ".", -1)
	}

	if format == "json" {
		response := WhisperGlobResponse {
			Name:		glob,
			Paths:		make([]string, 0),
		}
		for _, p := range files {
			response.Paths = append(response.Paths, p)
		}
		b, err := json.Marshal(response)
		if err != nil {
			fmt.Printf("failed to create JSON data for %s: %s\n", glob, err)
			return
		}
		wr.Write(b)
	} else if format == "pickle" {
		// [{'metric_path': 'metric', 'intervals': [(x,y)], 'isLeaf': True},]
		var metrics []map[string]interface{}
		var m map[string]interface{}

		for i, p := range files {
			m = make(map[string]interface{})
			m["metric_path"] = p
			// m["intervals"] = dunno how to do a tuple here
			m["isLeaf"] = leafs[i]
			metrics = append(metrics, m)
		}

		wr.Header().Set("Content-Type", "application/pickle")
		pEnc := pickle.NewEncoder(wr)
		pEnc.Encode(metrics)
	}
	fmt.Printf("find: %d hits for %s\n", len(files), glob)
	return
}

func fetchHandler(wr http.ResponseWriter, req *http.Request) {
//	GET /render/?target=general.me.1.percent_time_active.pfnredis&format=pickle&from=1396008021&until=1396022421 HTTP/1.1
//	http://localhost:8080/render/?target=testmetric&format=json&from=1395961200&until=1395961800
	req.ParseForm()
	metric := req.FormValue("target")
	format := req.FormValue("format")
	from := req.FormValue("from")
	until := req.FormValue("until")

	if format != "json" && format != "pickle" {
		fmt.Printf("dropping invalid uri (format=%s): %s\n",
				format, req.URL.RequestURI())
		http.Error(wr, "Bad request (unsupported format)", http.StatusBadRequest)
		return
	}

	path := config.WhisperData + "/" + strings.Replace(metric, ".", "/", -1) + ".wsp"
	w, err := whisper.Open(path)
	if err != nil {
		// the FE/carbonzipper often requests metrics we don't have
		//fmt.Printf("failed to open %s: %s\n", path, err)
		http.Error(wr, "Metric not found", http.StatusNotFound)
		return
	}

	i, err := strconv.Atoi(from)
	if err != nil {
		fmt.Printf("fromTime (%s) invalid: %s\n", from, err)
		if w != nil {
			w.Close()
		}
		w = nil
	}
	fromTime := int(i)
	i, err = strconv.Atoi(until)
	if err != nil {
		fmt.Printf("untilTime (%s) invalid: %s\n", from, err)
		if w != nil {
			w.Close()
		}
		w = nil
	}
	untilTime := int(i)

	if (w != nil) {
		defer w.Close()
	} else {
		http.Error(wr, "Bad request (invalid from/until time)", http.StatusBadRequest)
		return
	}

	points, err := w.Fetch(fromTime, untilTime)
	if err != nil {
		fmt.Printf("failed to getch points from %s: %s\n", path, err)
		return
	}
	values := points.Values()

	if format == "json" {
		response := WhisperFetchResponse {
			Name:		metric,
			StartTime:	points.FromTime(),
			StopTime:	points.UntilTime(),
			StepTime:	points.Step(),
			Values:		make([]float64, len(values)),
			IsAbsent:	make([]bool, len(values)),
		}

		for i, p := range values {
			if math.IsNaN(p) {
				response.Values[i] = 0
				response.IsAbsent[i] = true
			} else {
				response.Values[i] = p
				response.IsAbsent[i] = false
			}
		}

		b, err := json.Marshal(response)
		if err != nil {
			fmt.Printf("failed to create JSON data for %s: %s\n", path, err)
			return
		}
		wr.Write(b)
	} else if format == "pickle" {
		//[{'start': 1396271100, 'step': 60, 'name': 'metric',
		//'values': [9.0, 19.0, None], 'end': 1396273140}
		var metrics []map[string]interface{}
		var m map[string]interface{}

		m = make(map[string]interface{})
		m["start"] = points.FromTime()
		m["step"] = points.Step()
		m["end"] = points.UntilTime()
		m["name"] = metric

		mv := make([]interface{}, len(values))
		for i, p := range values {
			if math.IsNaN(p) {
				mv[i] = nil
			} else {
				mv[i] = p
			}
		}

		m["values"] = mv
		metrics = append(metrics, m)

		wr.Header().Set("Content-Type", "application/pickle")
		pEnc := pickle.NewEncoder(wr)
		pEnc.Encode(metrics)
	}

	fmt.Printf("served %d points for %s\n", len(values), metric)
	return
}

func main() {
	http.HandleFunc("/metrics/find/", findHandler)
	http.HandleFunc("/render/", fetchHandler)

	http.ListenAndServe(":8080", nil)
}
<commit_msg>added logging framework<commit_after>package main

import (
	whisper "github.com/grobian/go-whisper"
	pickle "github.com/kisielk/og-rek"

	"net/http"
	"encoding/json"
	"path/filepath"
	"strings"
	"strconv"
	"math"
	"fmt"
	"os"
)

var config = struct {
	WhisperData	string
}{
	WhisperData: "/var/lib/carbon/whisper",
	//WhisperData: "..",
}

type WhisperFetchResponse struct {
	Name        string      `json:"name"`
	StartTime   int         `json:"startTime"`
	StopTime    int         `json:"stopTime"`
	StepTime    int         `json:"stepTime"`
	Values      []float64   `json:"values"`
	IsAbsent    []bool      `json:"isAbsent"`
}

type WhisperGlobResponse struct {
	Name        string      `json:"name"`
	Paths       []string    `json:"paths"`
}

var log Logger

func findHandler(wr http.ResponseWriter, req *http.Request) {
//	GET /metrics/find/?local=1&format=pickle&query=general.hadoop.lhr4.ha201jobtracker-01.jobtracker.NonHeapMemoryUsage.committed HTTP/1.1
//	http://localhost:8080/metrics/find/?query=test
	req.ParseForm()
	glob := req.FormValue("query")
	format := req.FormValue("format")

	if format != "json" && format != "pickle" {
		log.Warn("dropping invalid uri (format=%s): %s",
				format, req.URL.RequestURI())
		http.Error(wr, "Bad request (unsupported format)",
				http.StatusBadRequest)
		return
	}

	/* things to glob:
	 * - carbon.relays  -> carbon.relays
	 * - carbon.re      -> carbon.relays, carbon.rewhatever
	 * - carbon.[rz]    -> carbon.relays, carbon.zipper
	 * - carbon.{re,zi} -> carbon.relays, carbon.zipper
	 * - implicit * at the end of each query
	 * - match is either dir or .wsp file
	 * unfortunately, filepath.Glob doesn't handle the curly brace
	 * expansion for us */
	lbrace := strings.Index(glob, "{")
	rbrace := -1
	if lbrace > -1 {
		rbrace = strings.Index(glob[lbrace:], "}")
		if rbrace > -1 {
			rbrace += lbrace
		}
	}
	files := make([]string, 0)
	if lbrace > -1 && rbrace > -1 {
		expansion := glob[lbrace + 1:rbrace]
		parts := strings.Split(expansion, ",")
		for _, sub := range parts {
			sglob := glob[:lbrace] + sub + glob[rbrace + 1:]
			path := config.WhisperData + "/" + strings.Replace(sglob, ".", "/", -1) + "*"
			nfiles, err := filepath.Glob(path)
			if err == nil {
				files = append(files, nfiles...)
			}
		}
	} else {
		path := config.WhisperData + "/" + strings.Replace(glob, ".", "/", -1) + "*"
		nfiles, err := filepath.Glob(path)
		if err == nil {
			files = append(files, nfiles...)
		}
	}

	leafs := make([]bool, len(files))
	for i, p := range files {
		p = p[len(config.WhisperData + "/"):]
		if strings.HasSuffix(p, ".wsp") {
			p = p[:len(p) - 4]
			leafs[i] = true
		} else {
			leafs[i] = false
		}
		files[i] = strings.Replace(p, "/", ".", -1)
	}

	if format == "json" {
		response := WhisperGlobResponse {
			Name:		glob,
			Paths:		make([]string, 0),
		}
		for _, p := range files {
			response.Paths = append(response.Paths, p)
		}
		b, err := json.Marshal(response)
		if err != nil {
			log.Error("failed to create JSON data for glob %s: %s", glob, err)
			return
		}
		wr.Write(b)
	} else if format == "pickle" {
		// [{'metric_path': 'metric', 'intervals': [(x,y)], 'isLeaf': True},]
		var metrics []map[string]interface{}
		var m map[string]interface{}

		for i, p := range files {
			m = make(map[string]interface{})
			m["metric_path"] = p
			// m["intervals"] = dunno how to do a tuple here
			m["isLeaf"] = leafs[i]
			metrics = append(metrics, m)
		}

		wr.Header().Set("Content-Type", "application/pickle")
		pEnc := pickle.NewEncoder(wr)
		pEnc.Encode(metrics)
	}
	log.Info("find: %d hits for %s", len(files), glob)
	return
}

func fetchHandler(wr http.ResponseWriter, req *http.Request) {
//	GET /render/?target=general.me.1.percent_time_active.pfnredis&format=pickle&from=1396008021&until=1396022421 HTTP/1.1
//	http://localhost:8080/render/?target=testmetric&format=json&from=1395961200&until=1395961800
	req.ParseForm()
	metric := req.FormValue("target")
	format := req.FormValue("format")
	from := req.FormValue("from")
	until := req.FormValue("until")

	if format != "json" && format != "pickle" {
		log.Warn("dropping invalid uri (format=%s): %s",
				format, req.URL.RequestURI())
		http.Error(wr, "Bad request (unsupported format)",
				http.StatusBadRequest)
		return
	}

	path := config.WhisperData + "/" + strings.Replace(metric, ".", "/", -1) + ".wsp"
	w, err := whisper.Open(path)
	if err != nil {
		// the FE/carbonzipper often requests metrics we don't have
		log.Debug("failed to %s", err)
		http.Error(wr, "Metric not found", http.StatusNotFound)
		return
	}

	i, err := strconv.Atoi(from)
	if err != nil {
		log.Debug("fromTime (%s) invalid: %s (in %s)",
				from, err, req.URL.RequestURI)
		if w != nil {
			w.Close()
		}
		w = nil
	}
	fromTime := int(i)
	i, err = strconv.Atoi(until)
	if err != nil {
		log.Debug("untilTime (%s) invalid: %s (in %s)",
				from, err, req.URL.RequestURI)
		if w != nil {
			w.Close()
		}
		w = nil
	}
	untilTime := int(i)

	if (w != nil) {
		defer w.Close()
	} else {
		http.Error(wr, "Bad request (invalid from/until time)",
				http.StatusBadRequest)
		return
	}

	points, err := w.Fetch(fromTime, untilTime)
	if err != nil {
		log.Error("failed to fetch points from %s: %s", path, err)
		http.Error(wr, "Fetching data points failed",
				http.StatusInternalServerError)
		return
	}
	values := points.Values()

	if format == "json" {
		response := WhisperFetchResponse {
			Name:		metric,
			StartTime:	points.FromTime(),
			StopTime:	points.UntilTime(),
			StepTime:	points.Step(),
			Values:		make([]float64, len(values)),
			IsAbsent:	make([]bool, len(values)),
		}

		for i, p := range values {
			if math.IsNaN(p) {
				response.Values[i] = 0
				response.IsAbsent[i] = true
			} else {
				response.Values[i] = p
				response.IsAbsent[i] = false
			}
		}

		b, err := json.Marshal(response)
		if err != nil {
			log.Error("failed to create JSON data for %s: %s", path, err)
			return
		}
		wr.Write(b)
	} else if format == "pickle" {
		//[{'start': 1396271100, 'step': 60, 'name': 'metric',
		//'values': [9.0, 19.0, None], 'end': 1396273140}
		var metrics []map[string]interface{}
		var m map[string]interface{}

		m = make(map[string]interface{})
		m["start"] = points.FromTime()
		m["step"] = points.Step()
		m["end"] = points.UntilTime()
		m["name"] = metric

		mv := make([]interface{}, len(values))
		for i, p := range values {
			if math.IsNaN(p) {
				mv[i] = nil
			} else {
				mv[i] = p
			}
		}

		m["values"] = mv
		metrics = append(metrics, m)

		wr.Header().Set("Content-Type", "application/pickle")
		pEnc := pickle.NewEncoder(wr)
		pEnc.Encode(metrics)
	}

	log.Info("served %d points for %s", len(values), metric)
	return
}

func main() {
	log = NewOutputLogger(INFO);

	http.HandleFunc("/metrics/find/", findHandler)
	http.HandleFunc("/render/", fetchHandler)

	log.Info("listening on port 8080")
	err := http.ListenAndServe(":8080", nil)
	if err != nil {
		log.Fatal("%s", err)
	}
	log.Info("stopped")
}

// Simple wrapper to enable/disable lots of spam
type Logger interface {
	Info(format string, a ...interface{})
	Warn(format string, a ...interface{})
	Error(format string, a ...interface{})
	Fatal(format string, a ...interface{})
	Debug(format string, a ...interface{})
}

type LogLevel int
const (
	FATAL LogLevel = 0
	ERROR LogLevel = 1
	WARN  LogLevel = 2
	INFO  LogLevel = 3
	DEBUG LogLevel = 4
)
type outputLogger struct {
	level LogLevel
	out *os.File
	err *os.File
}

func NewOutputLogger(level LogLevel) *outputLogger {
	r := new(outputLogger)
	r.level = level;
	r.out = os.Stdout;
	r.err = os.Stderr;

	return r
}

func (l *outputLogger) Debug(format string, a ...interface{}) {
	if l.level >= DEBUG {
		l.out.WriteString(fmt.Sprintf("DEBUG: " + format + "\n", a...))
	}
}

func (l *outputLogger) Info(format string, a ...interface{}) {
	if l.level >= INFO {
		l.out.WriteString(fmt.Sprintf("INFO: " + format + "\n", a...))
	}
}

func (l *outputLogger) Warn(format string, a ...interface{}) {
	if l.level >= WARN {
		l.out.WriteString(fmt.Sprintf("WARN: " + format + "\n", a...))
	}
}

func (l *outputLogger) Error(format string, a ...interface{}) {
	if l.level >= ERROR {
		l.err.WriteString(fmt.Sprintf("ERROR: " + format + "\n", a...))
	}
}

func (l *outputLogger) Fatal(format string, a ...interface{}) {
	if l.level >= FATAL {
		l.err.WriteString(fmt.Sprintf("ERROR: " + format + "\n", a...))
	}
	os.Exit(1)
}

<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"os"

	"github.com/cbednarski/hostess/commands"
	"github.com/cbednarski/hostess/hostess"
)

const help = `An idempotent tool for managing %s

Commands

    fmt                  Reformat the hosts file

    add <hostname> <ip>  Add or overwrite a hosts entry
    rm <hostname>        Remote a hosts entry
    on <hostname>        Enable a hosts entry
    off <hostname>       Disable a hosts entry

    ls                   List hosts entries
    has                  Exit 0 if entry present in hosts file, 1 if not

    dump                 Export hosts entries as JSON
    apply                Import hosts entries from JSON

    All commands that change the hosts file will implicitly reformat it.

Flags

    -n will preview changes but not rewrite your hosts file
    -4 limit changes to IPv4 entries
    -6 limit changes to IPv6 entries

Configuration

    HOSTESS_FMT may be set to unix or windows to force that platform's syntax
    HOSTESS_PATH may be set to point to a file other than the platform default

About

    Copyright 2015-2020 Chris Bednarski <chris@cbednarski.com>; MIT Licensed
    Portions Copyright the Go authors, licensed under BSD-style license
    Bugs and updates via https://github.com/cbednarski/hostess
`

var (
	Version           = "dev"
	ErrInvalidCommand = errors.New("invalid command")
)

func ExitWithError(err error) {
	if err != nil {
		os.Stderr.WriteString(err.Error())
		os.Stderr.WriteString("\n")
		os.Exit(1)
	}
}

func CommandUsage(command string) error {
	return fmt.Errorf("Usage: %s %s <hostname>", os.Args[0], command)
}

func wrappedMain() error {
	cli := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
	ipv4 := cli.Bool("4",false, "IPv4")
	ipv6 := cli.Bool("6",false, "IPv6")
	preview := cli.Bool("n",false, "preview")
	cli.Usage = func() {
		fmt.Printf(help, hostess.GetHostsPath())
	}

	if err := cli.Parse(os.Args[1:]); err != nil {
		return err
	}

	options := &commands.Options{
		IPVersion: 0,
		Preview:   *preview,
	}
	if *ipv4 {
		options.IPVersion = options.IPVersion|commands.IPv4
	}
	if *ipv6 {
		options.IPVersion = options.IPVersion|commands.IPv6
	}

	command := cli.Arg(0)
	switch command {

	case "-v", "--version", "version":
		fmt.Println(Version)
		return nil

	case "-h", "--help", "help":
		cli.Usage()
		return nil

	case "fmt":
		return commands.Format(options)

	case "add":
		if len(cli.Args()) != 2 {
			return fmt.Errorf("Usage: %s add <hostname> <ip>", cli.Name())
		}
		return commands.Add(options, cli.Arg(0), cli.Arg(1))

	case "rm":
		if cli.Arg(0) == "" {
			return CommandUsage(command)
		}
		return commands.Remove(options, cli.Arg(0))

	case "on":
		if cli.Arg(0) == "" {
			return CommandUsage(command)
		}
		return commands.Enable(options, cli.Arg(0))

	case "off":
		if cli.Arg(0) == "" {
			return CommandUsage(command)
		}
		return commands.Disable(options, cli.Arg(0))

	case "ls":
		return commands.List(options)

	case "has":
		if cli.Arg(0) == "" {
			return CommandUsage(command)
		}
		return commands.Has(options, cli.Arg(0))

	case "dump":
		return commands.Dump(options)

	case "apply":
		if cli.Arg(0) == "" {
			return fmt.Errorf("Usage: %s apply <filename>", os.Args[0])
		}
		return commands.Apply(options, cli.Arg(0))

	default:
		return ErrInvalidCommand
	}
}

func main() {
	ExitWithError(wrappedMain())
}
<commit_msg>Remove trailing space<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"os"

	"github.com/cbednarski/hostess/commands"
	"github.com/cbednarski/hostess/hostess"
)

const help = `An idempotent tool for managing %s

Commands

    fmt                  Reformat the hosts file

    add <hostname> <ip>  Add or overwrite a hosts entry
    rm <hostname>        Remote a hosts entry
    on <hostname>        Enable a hosts entry
    off <hostname>       Disable a hosts entry

    ls                   List hosts entries
    has                  Exit 0 if entry present in hosts file, 1 if not

    dump                 Export hosts entries as JSON
    apply                Import hosts entries from JSON

    All commands that change the hosts file will implicitly reformat it.

Flags

    -n will preview changes but not rewrite your hosts file
    -4 limit changes to IPv4 entries
    -6 limit changes to IPv6 entries

Configuration

    HOSTESS_FMT may be set to unix or windows to force that platform's syntax
    HOSTESS_PATH may be set to point to a file other than the platform default

About

    Copyright 2015-2020 Chris Bednarski <chris@cbednarski.com>; MIT Licensed
    Portions Copyright the Go authors, licensed under BSD-style license
    Bugs and updates via https://github.com/cbednarski/hostess
`

var (
	Version           = "dev"
	ErrInvalidCommand = errors.New("invalid command")
)

func ExitWithError(err error) {
	if err != nil {
		os.Stderr.WriteString(err.Error())
		os.Stderr.WriteString("\n")
		os.Exit(1)
	}
}

func CommandUsage(command string) error {
	return fmt.Errorf("Usage: %s %s <hostname>", os.Args[0], command)
}

func wrappedMain() error {
	cli := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
	ipv4 := cli.Bool("4",false, "IPv4")
	ipv6 := cli.Bool("6",false, "IPv6")
	preview := cli.Bool("n",false, "preview")
	cli.Usage = func() {
		fmt.Printf(help, hostess.GetHostsPath())
	}

	if err := cli.Parse(os.Args[1:]); err != nil {
		return err
	}

	options := &commands.Options{
		IPVersion: 0,
		Preview:   *preview,
	}
	if *ipv4 {
		options.IPVersion = options.IPVersion|commands.IPv4
	}
	if *ipv6 {
		options.IPVersion = options.IPVersion|commands.IPv6
	}

	command := cli.Arg(0)
	switch command {

	case "-v", "--version", "version":
		fmt.Println(Version)
		return nil

	case "-h", "--help", "help":
		cli.Usage()
		return nil

	case "fmt":
		return commands.Format(options)

	case "add":
		if len(cli.Args()) != 2 {
			return fmt.Errorf("Usage: %s add <hostname> <ip>", cli.Name())
		}
		return commands.Add(options, cli.Arg(0), cli.Arg(1))

	case "rm":
		if cli.Arg(0) == "" {
			return CommandUsage(command)
		}
		return commands.Remove(options, cli.Arg(0))

	case "on":
		if cli.Arg(0) == "" {
			return CommandUsage(command)
		}
		return commands.Enable(options, cli.Arg(0))

	case "off":
		if cli.Arg(0) == "" {
			return CommandUsage(command)
		}
		return commands.Disable(options, cli.Arg(0))

	case "ls":
		return commands.List(options)

	case "has":
		if cli.Arg(0) == "" {
			return CommandUsage(command)
		}
		return commands.Has(options, cli.Arg(0))

	case "dump":
		return commands.Dump(options)

	case "apply":
		if cli.Arg(0) == "" {
			return fmt.Errorf("Usage: %s apply <filename>", os.Args[0])
		}
		return commands.Apply(options, cli.Arg(0))

	default:
		return ErrInvalidCommand
	}
}

func main() {
	ExitWithError(wrappedMain())
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/gin-gonic/gin"
	"github.com/venicegeo/pz-gocommon"
	"gopkg.in/olivere/elastic.v3"
	"log"
	"net/http"
	"os"
	"strings"
)

//---------------------------------------------------------------------------

func makeESIndex(client *elastic.Client, index string) error {
	exists, err := client.IndexExists(index).Do()
	if err != nil {
		return err
	}

	if exists {
		_, err = client.DeleteIndex(index).Do()
		if err != nil {
			return err
		}
	}

	_, err = client.CreateIndex(index).Do()
	if err != nil {
		return err
	}
	return nil
}

func newESClient() (*elastic.Client, error) {
  client, err := elastic.NewClient(
    elastic.SetURL("https://search-venice-es-pjebjkdaueu2gukocyccj4r5m4.us-east-1.es.amazonaws.com"),
    elastic.SetSniff(false),
    elastic.SetMaxRetries(5),
    elastic.SetErrorLog(log.New(os.Stderr, "ELASTIC ", log.LstdFlags)),
    elastic.SetInfoLog(log.New(os.Stdout, "", log.LstdFlags)))
	if err != nil {
		return nil, err
	}
	return client, nil
}

///////////////////////////////////////////////////////////

func runAlertServer(discoveryURL string, port string) error {

	esClient, err := newESClient()
	if err != nil {
		return err
	}

	conditionDB, err := newConditionDB(esClient, "conditions")
	if err != nil {
		return err
	}
	eventDB, err := newEventDB(esClient, "events")
	if err != nil {
		return err
	}
	alertDB, err := newAlertDB(esClient, "alerts")
	if err != nil {
		return err
	}

	myAddress := fmt.Sprintf(":%s", port)
	myURL := fmt.Sprintf("http://%s/alerts", myAddress)

	piazza.RegistryInit(discoveryURL)
	err = piazza.RegisterService("pz-alerter", "core-service", myURL)
	if err != nil {
		return err
	}

	gin.SetMode(gin.ReleaseMode)

	router := gin.New()
	//router.Use(gin.Logger())
	//router.Use(gin.Recovery())

	//---------------------------------

	router.GET("/", func(c *gin.Context) {
		c.JSON(http.StatusOK, "Hi")
	})

	//---------------------------------

	router.POST("/events", func(c *gin.Context) {
		event := &Event{}
		err := c.BindJSON(event)
		if err != nil {
			log.Println(err)
			c.Error(err)
			return
		}
		err = eventDB.write(event)
		if err != nil {
			c.Error(err)
			return
		}
		c.JSON(http.StatusCreated, gin.H{"id": event.ID})

		alertDB.checkConditions(*event, conditionDB)
	})

	router.GET("/events", func(c *gin.Context) {
		m, err := eventDB.getAll()
		if err != nil {
			c.Error(err)
			return
		}
		c.JSON(http.StatusOK, m)
	})

	//---------------------------------

	router.GET("/alerts/:id", func(c *gin.Context) {
		id := c.Param("id")
		v, err := alertDB.getByConditionID(id)
		if err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"condition_id": id})
			return
		}
		if v == nil {
			c.JSON(http.StatusNotFound, gin.H{"condition_id": id})
			return
		}
		c.JSON(http.StatusOK, v)
	})

	router.GET("/alerts", func(c *gin.Context) {
		all, err := alertDB.getAll()
		if err != nil {
			c.Error(err)
			return
		}
		c.JSON(http.StatusOK, all)
	})

	//---------------------------------
	router.POST("/conditions", func(c *gin.Context) {
		var condition Condition
		err := c.BindJSON(&condition)
		if err != nil {
			c.Error(err)
			log.Println(err)
			return
		}
		err = conditionDB.write(&condition)
		if err != nil {
			c.Error(err)
			return
		}
		c.JSON(http.StatusCreated, gin.H{"id": condition.ID})
	})

	/*router.PUT("/conditions", func(c *gin.Context) {
		var condition Condition
		err := c.BindJSON(&condition)
		if err != nil {
			c.Error(err)
			return
		}
		ok := conditionDB.update(&condition)
		if !ok {
			c.JSON(http.StatusNotFound, gin.H{"id": condition.ID})
			return
		}
		c.JSON(http.StatusOK, gin.H{"id": condition.ID})
	})*/

	router.GET("/conditions", func(c *gin.Context) {
		all, err := conditionDB.getAll()
		if err != nil {
			c.Error(err)
			return
		}
		c.JSON(http.StatusOK, all)
	})

	router.GET("/conditions/:id", func(c *gin.Context) {
		id := c.Param("id")
		v, err := conditionDB.readByID(id)
		if err != nil {
			c.Error(err)
			return
		}
		if v == nil {
			c.JSON(http.StatusNotFound, gin.H{"id": id})
			return
		}
		c.JSON(http.StatusOK, v)
	})

	router.DELETE("/conditions/:id", func(c *gin.Context) {
		id := c.Param("id")
		ok, err := conditionDB.deleteByID(id)
		if err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"id": id})
			return
		}
		if !ok {
			c.JSON(http.StatusNotFound, gin.H{"id": id})
			return
		}
		c.JSON(http.StatusOK, nil)
	})

	//---------------------------------

	err = router.Run(":" + port)
	return err
}

func app() int {
  var defaultPort = os.Getenv("PORT")
  if defaultPort == "" {
    defaultPort = "12342"
  }
	var discoveryURL = flag.String("discovery", "http://localhost:3000", "URL of pz-discovery")
	var port = flag.String("port", defaultPort, "port number of this pz-alerter")

	flag.Parse()

	log.Printf("starting: discovery=%s, port=%s", *discoveryURL, *port)

	err := runAlertServer(*discoveryURL, *port)
	if err != nil {
		fmt.Print(err)
		return 1
	}

	// not reached
	return 1
}

func main2(cmd string) int {
	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
	os.Args = strings.Fields("main_tester " + cmd)
	return app()
}

func main() {
	os.Exit(app())
}
<commit_msg>friendlier health msg<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/gin-gonic/gin"
	"github.com/venicegeo/pz-gocommon"
	"gopkg.in/olivere/elastic.v3"
	"log"
	"net/http"
	"os"
	"strings"
)

//---------------------------------------------------------------------------

func makeESIndex(client *elastic.Client, index string) error {
	exists, err := client.IndexExists(index).Do()
	if err != nil {
		return err
	}

	if exists {
		_, err = client.DeleteIndex(index).Do()
		if err != nil {
			return err
		}
	}

	_, err = client.CreateIndex(index).Do()
	if err != nil {
		return err
	}
	return nil
}

func newESClient() (*elastic.Client, error) {
  client, err := elastic.NewClient(
    elastic.SetURL("https://search-venice-es-pjebjkdaueu2gukocyccj4r5m4.us-east-1.es.amazonaws.com"),
    elastic.SetSniff(false),
    elastic.SetMaxRetries(5),
    elastic.SetErrorLog(log.New(os.Stderr, "ELASTIC ", log.LstdFlags)),
    elastic.SetInfoLog(log.New(os.Stdout, "", log.LstdFlags)))
	if err != nil {
		return nil, err
	}
	return client, nil
}

///////////////////////////////////////////////////////////

func runAlertServer(discoveryURL string, port string) error {

	esClient, err := newESClient()
	if err != nil {
		return err
	}

	conditionDB, err := newConditionDB(esClient, "conditions")
	if err != nil {
		return err
	}
	eventDB, err := newEventDB(esClient, "events")
	if err != nil {
		return err
	}
	alertDB, err := newAlertDB(esClient, "alerts")
	if err != nil {
		return err
	}

	myAddress := fmt.Sprintf(":%s", port)
	myURL := fmt.Sprintf("http://%s/alerts", myAddress)

	piazza.RegistryInit(discoveryURL)
	err = piazza.RegisterService("pz-alerter", "core-service", myURL)
	if err != nil {
		return err
	}

	gin.SetMode(gin.ReleaseMode)

	router := gin.New()
	//router.Use(gin.Logger())
	//router.Use(gin.Recovery())

	//---------------------------------

	router.GET("/", func(c *gin.Context) {
		c.JSON(http.StatusOK, "Hi. I'm pz-alerter.")
	})

	//---------------------------------

	router.POST("/events", func(c *gin.Context) {
		event := &Event{}
		err := c.BindJSON(event)
		if err != nil {
			log.Println(err)
			c.Error(err)
			return
		}
		err = eventDB.write(event)
		if err != nil {
			c.Error(err)
			return
		}
		c.JSON(http.StatusCreated, gin.H{"id": event.ID})

		alertDB.checkConditions(*event, conditionDB)
	})

	router.GET("/events", func(c *gin.Context) {
		m, err := eventDB.getAll()
		if err != nil {
			c.Error(err)
			return
		}
		c.JSON(http.StatusOK, m)
	})

	//---------------------------------

	router.GET("/alerts/:id", func(c *gin.Context) {
		id := c.Param("id")
		v, err := alertDB.getByConditionID(id)
		if err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"condition_id": id})
			return
		}
		if v == nil {
			c.JSON(http.StatusNotFound, gin.H{"condition_id": id})
			return
		}
		c.JSON(http.StatusOK, v)
	})

	router.GET("/alerts", func(c *gin.Context) {
		all, err := alertDB.getAll()
		if err != nil {
			c.Error(err)
			return
		}
		c.JSON(http.StatusOK, all)
	})

	//---------------------------------
	router.POST("/conditions", func(c *gin.Context) {
		var condition Condition
		err := c.BindJSON(&condition)
		if err != nil {
			c.Error(err)
			log.Println(err)
			return
		}
		err = conditionDB.write(&condition)
		if err != nil {
			c.Error(err)
			return
		}
		c.JSON(http.StatusCreated, gin.H{"id": condition.ID})
	})

	/*router.PUT("/conditions", func(c *gin.Context) {
		var condition Condition
		err := c.BindJSON(&condition)
		if err != nil {
			c.Error(err)
			return
		}
		ok := conditionDB.update(&condition)
		if !ok {
			c.JSON(http.StatusNotFound, gin.H{"id": condition.ID})
			return
		}
		c.JSON(http.StatusOK, gin.H{"id": condition.ID})
	})*/

	router.GET("/conditions", func(c *gin.Context) {
		all, err := conditionDB.getAll()
		if err != nil {
			c.Error(err)
			return
		}
		c.JSON(http.StatusOK, all)
	})

	router.GET("/conditions/:id", func(c *gin.Context) {
		id := c.Param("id")
		v, err := conditionDB.readByID(id)
		if err != nil {
			c.Error(err)
			return
		}
		if v == nil {
			c.JSON(http.StatusNotFound, gin.H{"id": id})
			return
		}
		c.JSON(http.StatusOK, v)
	})

	router.DELETE("/conditions/:id", func(c *gin.Context) {
		id := c.Param("id")
		ok, err := conditionDB.deleteByID(id)
		if err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"id": id})
			return
		}
		if !ok {
			c.JSON(http.StatusNotFound, gin.H{"id": id})
			return
		}
		c.JSON(http.StatusOK, nil)
	})

	//---------------------------------

	err = router.Run(":" + port)
	return err
}

func app() int {
  var defaultPort = os.Getenv("PORT")
  if defaultPort == "" {
    defaultPort = "12342"
  }
	var discoveryURL = flag.String("discovery", "http://localhost:3000", "URL of pz-discovery")
	var port = flag.String("port", defaultPort, "port number of this pz-alerter")

	flag.Parse()

	log.Printf("starting: discovery=%s, port=%s", *discoveryURL, *port)

	err := runAlertServer(*discoveryURL, *port)
	if err != nil {
		fmt.Print(err)
		return 1
	}

	// not reached
	return 1
}

func main2(cmd string) int {
	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
	os.Args = strings.Fields("main_tester " + cmd)
	return app()
}

func main() {
	os.Exit(app())
}
<|endoftext|>


<commit_before>// Copyright 2013 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/extraction"

	clientmodel "github.com/prometheus/client_golang/model"

	"github.com/prometheus/prometheus/config"
	"github.com/prometheus/prometheus/notification"
	"github.com/prometheus/prometheus/retrieval"
	"github.com/prometheus/prometheus/rules"
	"github.com/prometheus/prometheus/storage/metric"
	"github.com/prometheus/prometheus/web"
	"github.com/prometheus/prometheus/web/api"
)

const deletionBatchSize = 100

// Commandline flags.
var (
	configFile         = flag.String("configFile", "prometheus.conf", "Prometheus configuration file name.")
	metricsStoragePath = flag.String("metricsStoragePath", "/tmp/metrics", "Base path for metrics storage.")

	alertmanagerUrl = flag.String("alertmanager.url", "", "The URL of the alert manager to send notifications to.")

	samplesQueueCapacity      = flag.Int("storage.queue.samplesCapacity", 4096, "The size of the unwritten samples queue.")
	diskAppendQueueCapacity   = flag.Int("storage.queue.diskAppendCapacity", 1000000, "The size of the queue for items that are pending writing to disk.")
	memoryAppendQueueCapacity = flag.Int("storage.queue.memoryAppendCapacity", 10000, "The size of the queue for items that are pending writing to memory.")

	headCompactInterval = flag.Duration("compact.headInterval", 3*time.Hour, "The amount of time between head compactions.")
	bodyCompactInterval = flag.Duration("compact.bodyInterval", 5*time.Hour, "The amount of time between body compactions.")
	tailCompactInterval = flag.Duration("compact.tailInterval", 7*time.Hour, "The amount of time between tail compactions.")

	headGroupSize = flag.Int("compact.headGroupSize", 500, "The minimum group size for head samples.")
	bodyGroupSize = flag.Int("compact.bodyGroupSize", 5000, "The minimum group size for body samples.")
	tailGroupSize = flag.Int("compact.tailGroupSize", 10000, "The minimum group size for tail samples.")

	headAge = flag.Duration("compact.headAgeInclusiveness", 5*time.Minute, "The relative inclusiveness of head samples.")
	bodyAge = flag.Duration("compact.bodyAgeInclusiveness", time.Hour, "The relative inclusiveness of body samples.")
	tailAge = flag.Duration("compact.tailAgeInclusiveness", 24*time.Hour, "The relative inclusiveness of tail samples.")

	deleteInterval = flag.Duration("delete.interval", 11*time.Hour, "The amount of time between deletion of old values.")

	deleteAge = flag.Duration("delete.ageMaximum", 15*24*time.Hour, "The relative maximum age for values before they are deleted.")

	arenaFlushInterval = flag.Duration("arena.flushInterval", 15*time.Minute, "The period at which the in-memory arena is flushed to disk.")
	arenaTTL           = flag.Duration("arena.ttl", 10*time.Minute, "The relative age of values to purge to disk from memory.")

	notificationQueueCapacity = flag.Int("alertmanager.notificationQueueCapacity", 100, "The size of the queue for pending alert manager notifications.")

	concurrentRetrievalAllowance = flag.Int("concurrentRetrievalAllowance", 15, "The number of concurrent metrics retrieval requests allowed.")

	printVersion = flag.Bool("version", false, "print version information")
)

type prometheus struct {
	headCompactionTimer *time.Ticker
	bodyCompactionTimer *time.Ticker
	tailCompactionTimer *time.Ticker
	deletionTimer       *time.Ticker

	curationSema             chan bool
	stopBackgroundOperations chan bool

	unwrittenSamples chan *extraction.Result

	ruleManager   rules.RuleManager
	notifications chan notification.NotificationReqs
	storage       *metric.TieredStorage

	curationState metric.CurationStateUpdater
}

func (p *prometheus) interruptHandler() {
	notifier := make(chan os.Signal)
	signal.Notify(notifier, os.Interrupt, syscall.SIGTERM)

	<-notifier

	glog.Warning("Received SIGINT/SIGTERM; Exiting gracefully...")
	p.close()
	os.Exit(0)
}

func (p *prometheus) compact(olderThan time.Duration, groupSize int) error {
	select {
	case p.curationSema <- true:
	default:
		glog.Warningf("Deferred compaction for %s and %s due to existing operation.", operation, groupSize)
		return
	}

	defer func() {
		<-p.curationSema
	}()

	processor := metric.NewCompactionProcessor(&metric.CompactionProcessorOptions{
		MaximumMutationPoolBatch: groupSize * 3,
		MinimumGroupSize:         groupSize,
	})
	defer processor.Close()

	curator := metric.NewCurator(&metric.CuratorOptions{
		Stop: p.stopBackgroundOperations,

		ViewQueue: p.storage.ViewQueue,
	})
	defer curator.Close()

	return curator.Run(olderThan, time.Now(), processor, p.storage.DiskStorage.CurationRemarks, p.storage.DiskStorage.MetricSamples, p.storage.DiskStorage.MetricHighWatermarks, p.curationState)
}

func (p *prometheus) delete(olderThan time.Duration, batchSize int) error {
	select {
	case p.curationSema <- true:
	default:
		glog.Warningf("Deferred compaction for %s and %s due to existing operation.", operation, groupSize)
		return
	}

	processor := metric.NewDeletionProcessor(&metric.DeletionProcessorOptions{
		MaximumMutationPoolBatch: batchSize,
	})
	defer processor.Close()

	curator := metric.NewCurator(&metric.CuratorOptions{
		Stop: p.stopBackgroundOperations,

		ViewQueue: p.storage.ViewQueue,
	})
	defer curator.Close()

	return curator.Run(olderThan, time.Now(), processor, p.storage.DiskStorage.CurationRemarks, p.storage.DiskStorage.MetricSamples, p.storage.DiskStorage.MetricHighWatermarks, p.curationState)
}

func (p *prometheus) close() {
	select {
	case p.curationSema <- true:
	default:
	}

	if p.headCompactionTimer != nil {
		p.headCompactionTimer.Stop()
	}
	if p.bodyCompactionTimer != nil {
		p.bodyCompactionTimer.Stop()
	}
	if p.tailCompactionTimer != nil {
		p.tailCompactionTimer.Stop()
	}
	if p.deletionTimer != nil {
		p.deletionTimer.Stop()
	}

	if len(p.stopBackgroundOperations) == 0 {
		p.stopBackgroundOperations <- true
	}

	p.ruleManager.Stop()
	p.storage.Close()

	close(p.notifications)
	close(p.stopBackgroundOperations)
}

func main() {
	// TODO(all): Future additions to main should be, where applicable, glumped
	// into the prometheus struct above---at least where the scoping of the entire
	// server is concerned.
	flag.Parse()

	versionInfoTmpl.Execute(os.Stdout, BuildInfo)

	if *printVersion {
		os.Exit(0)
	}

	conf, err := config.LoadFromFile(*configFile)
	if err != nil {
		glog.Fatalf("Error loading configuration from %s: %v", *configFile, err)
	}

	ts, err := metric.NewTieredStorage(uint(*diskAppendQueueCapacity), 100, *arenaFlushInterval, *arenaTTL, *metricsStoragePath)
	if err != nil {
		glog.Fatal("Error opening storage: ", err)
	}

	unwrittenSamples := make(chan *extraction.Result, *samplesQueueCapacity)
	ingester := &retrieval.MergeLabelsIngester{
		Labels:          conf.GlobalLabels(),
		CollisionPrefix: clientmodel.ExporterLabelPrefix,

		Ingester: retrieval.ChannelIngester(unwrittenSamples),
	}
	// Coprime numbers, fool!
	headCompactionTimer := time.NewTicker(*headCompactInterval)
	bodyCompactionTimer := time.NewTicker(*bodyCompactInterval)
	tailCompactionTimer := time.NewTicker(*tailCompactInterval)
	deletionTimer := time.NewTicker(*deleteInterval)

	// Queue depth will need to be exposed
	targetManager := retrieval.NewTargetManager(ingester, *concurrentRetrievalAllowance)
	targetManager.AddTargetsFromConfig(conf)

	notifications := make(chan notification.NotificationReqs, *notificationQueueCapacity)

	// Queue depth will need to be exposed
	ruleManager := rules.NewRuleManager(&rules.RuleManagerOptions{
		Results:            unwrittenSamples,
		Notifications:      notifications,
		EvaluationInterval: conf.EvaluationInterval(),
		Storage:            ts,
		PrometheusUrl:      web.MustBuildServerUrl(),
	})
	if err := ruleManager.AddRulesFromConfig(conf); err != nil {
		glog.Fatal("Error loading rule files: ", err)
	}
	go ruleManager.Run()

	notificationHandler := notification.NewNotificationHandler(*alertmanagerUrl, notifications)
	go notificationHandler.Run()

	flags := map[string]string{}

	flag.VisitAll(func(f *flag.Flag) {
		flags[f.Name] = f.Value.String()
	})

	prometheusStatus := &web.PrometheusStatusHandler{
		BuildInfo:   BuildInfo,
		Config:      conf.String(),
		RuleManager: ruleManager,
		TargetPools: targetManager.Pools(),
		Flags:       flags,
		Birth:       time.Now(),
	}

	alertsHandler := &web.AlertsHandler{
		RuleManager: ruleManager,
	}

	databasesHandler := &web.DatabasesHandler{
		Provider:        ts.DiskStorage,
		RefreshInterval: 5 * time.Minute,
	}

	metricsService := &api.MetricsService{
		Config:        &conf,
		TargetManager: targetManager,
		Storage:       ts,
	}

	webService := &web.WebService{
		StatusHandler:    prometheusStatus,
		MetricsHandler:   metricsService,
		DatabasesHandler: databasesHandler,
		AlertsHandler:    alertsHandler,
	}

	prometheus := &prometheus{
		bodyCompactionTimer: bodyCompactionTimer,
		headCompactionTimer: headCompactionTimer,
		tailCompactionTimer: tailCompactionTimer,

		deletionTimer: deletionTimer,

		curationState: prometheusStatus,
		curationSema:  make(chan bool, 1),

		unwrittenSamples: unwrittenSamples,

		stopBackgroundOperations: make(chan bool, 1),

		ruleManager:   ruleManager,
		notifications: notifications,
		storage:       ts,
	}
	defer prometheus.close()

	storageStarted := make(chan bool)
	go ts.Serve(storageStarted)
	<-storageStarted

	go prometheus.interruptHandler()

	go func() {
		for _ = range prometheus.headCompactionTimer.C {
			glog.Info("Starting head compaction...")
			err := prometheus.compact(*headAge, *headGroupSize)

			if err != nil {
				glog.Error("could not compact: ", err)
			}
			glog.Info("Done")
		}
	}()

	go func() {
		for _ = range prometheus.bodyCompactionTimer.C {
			glog.Info("Starting body compaction...")
			err := prometheus.compact(*bodyAge, *bodyGroupSize)

			if err != nil {
				glog.Error("could not compact: ", err)
			}
			glog.Info("Done")
		}
	}()

	go func() {
		for _ = range prometheus.tailCompactionTimer.C {
			glog.Info("Starting tail compaction...")
			err := prometheus.compact(*tailAge, *tailGroupSize)

			if err != nil {
				glog.Error("could not compact: ", err)
			}
			glog.Info("Done")
		}
	}()

	go func() {
		for _ = range prometheus.deletionTimer.C {
			glog.Info("Starting deletion of stale values...")
			err := prometheus.delete(*deleteAge, deletionBatchSize)

			if err != nil {
				glog.Error("could not delete: ", err)
			}
			glog.Info("Done")
		}
	}()

	go func() {
		err := webService.ServeForever()
		if err != nil {
			glog.Fatal(err)
		}
	}()

	// TODO(all): Migrate this into prometheus.serve().
	for block := range unwrittenSamples {
		if block.Err == nil {
			ts.AppendSamples(block.Samples)
		}
	}
}
<commit_msg>Fix arguments for format string.<commit_after>// Copyright 2013 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/extraction"

	clientmodel "github.com/prometheus/client_golang/model"

	"github.com/prometheus/prometheus/config"
	"github.com/prometheus/prometheus/notification"
	"github.com/prometheus/prometheus/retrieval"
	"github.com/prometheus/prometheus/rules"
	"github.com/prometheus/prometheus/storage/metric"
	"github.com/prometheus/prometheus/web"
	"github.com/prometheus/prometheus/web/api"
)

const deletionBatchSize = 100

// Commandline flags.
var (
	configFile         = flag.String("configFile", "prometheus.conf", "Prometheus configuration file name.")
	metricsStoragePath = flag.String("metricsStoragePath", "/tmp/metrics", "Base path for metrics storage.")

	alertmanagerUrl = flag.String("alertmanager.url", "", "The URL of the alert manager to send notifications to.")

	samplesQueueCapacity      = flag.Int("storage.queue.samplesCapacity", 4096, "The size of the unwritten samples queue.")
	diskAppendQueueCapacity   = flag.Int("storage.queue.diskAppendCapacity", 1000000, "The size of the queue for items that are pending writing to disk.")
	memoryAppendQueueCapacity = flag.Int("storage.queue.memoryAppendCapacity", 10000, "The size of the queue for items that are pending writing to memory.")

	headCompactInterval = flag.Duration("compact.headInterval", 3*time.Hour, "The amount of time between head compactions.")
	bodyCompactInterval = flag.Duration("compact.bodyInterval", 5*time.Hour, "The amount of time between body compactions.")
	tailCompactInterval = flag.Duration("compact.tailInterval", 7*time.Hour, "The amount of time between tail compactions.")

	headGroupSize = flag.Int("compact.headGroupSize", 500, "The minimum group size for head samples.")
	bodyGroupSize = flag.Int("compact.bodyGroupSize", 5000, "The minimum group size for body samples.")
	tailGroupSize = flag.Int("compact.tailGroupSize", 10000, "The minimum group size for tail samples.")

	headAge = flag.Duration("compact.headAgeInclusiveness", 5*time.Minute, "The relative inclusiveness of head samples.")
	bodyAge = flag.Duration("compact.bodyAgeInclusiveness", time.Hour, "The relative inclusiveness of body samples.")
	tailAge = flag.Duration("compact.tailAgeInclusiveness", 24*time.Hour, "The relative inclusiveness of tail samples.")

	deleteInterval = flag.Duration("delete.interval", 11*time.Hour, "The amount of time between deletion of old values.")

	deleteAge = flag.Duration("delete.ageMaximum", 15*24*time.Hour, "The relative maximum age for values before they are deleted.")

	arenaFlushInterval = flag.Duration("arena.flushInterval", 15*time.Minute, "The period at which the in-memory arena is flushed to disk.")
	arenaTTL           = flag.Duration("arena.ttl", 10*time.Minute, "The relative age of values to purge to disk from memory.")

	notificationQueueCapacity = flag.Int("alertmanager.notificationQueueCapacity", 100, "The size of the queue for pending alert manager notifications.")

	concurrentRetrievalAllowance = flag.Int("concurrentRetrievalAllowance", 15, "The number of concurrent metrics retrieval requests allowed.")

	printVersion = flag.Bool("version", false, "print version information")
)

type prometheus struct {
	headCompactionTimer *time.Ticker
	bodyCompactionTimer *time.Ticker
	tailCompactionTimer *time.Ticker
	deletionTimer       *time.Ticker

	curationSema             chan bool
	stopBackgroundOperations chan bool

	unwrittenSamples chan *extraction.Result

	ruleManager   rules.RuleManager
	notifications chan notification.NotificationReqs
	storage       *metric.TieredStorage

	curationState metric.CurationStateUpdater
}

func (p *prometheus) interruptHandler() {
	notifier := make(chan os.Signal)
	signal.Notify(notifier, os.Interrupt, syscall.SIGTERM)

	<-notifier

	glog.Warning("Received SIGINT/SIGTERM; Exiting gracefully...")
	p.close()
	os.Exit(0)
}

func (p *prometheus) compact(olderThan time.Duration, groupSize int) error {
	select {
	case p.curationSema <- true:
	default:
		glog.Warningf("Deferred compaction for %s and %s due to existing operation.", olderThan, groupSize)

		return nil
	}

	defer func() {
		<-p.curationSema
	}()

	processor := metric.NewCompactionProcessor(&metric.CompactionProcessorOptions{
		MaximumMutationPoolBatch: groupSize * 3,
		MinimumGroupSize:         groupSize,
	})
	defer processor.Close()

	curator := metric.NewCurator(&metric.CuratorOptions{
		Stop: p.stopBackgroundOperations,

		ViewQueue: p.storage.ViewQueue,
	})
	defer curator.Close()

	return curator.Run(olderThan, time.Now(), processor, p.storage.DiskStorage.CurationRemarks, p.storage.DiskStorage.MetricSamples, p.storage.DiskStorage.MetricHighWatermarks, p.curationState)
}

func (p *prometheus) delete(olderThan time.Duration, batchSize int) error {
	select {
	case p.curationSema <- true:
	default:
		glog.Warningf("Deferred deletion for %s due to existing operation.", olderThan)

		return nil
	}

	processor := metric.NewDeletionProcessor(&metric.DeletionProcessorOptions{
		MaximumMutationPoolBatch: batchSize,
	})
	defer processor.Close()

	curator := metric.NewCurator(&metric.CuratorOptions{
		Stop: p.stopBackgroundOperations,

		ViewQueue: p.storage.ViewQueue,
	})
	defer curator.Close()

	return curator.Run(olderThan, time.Now(), processor, p.storage.DiskStorage.CurationRemarks, p.storage.DiskStorage.MetricSamples, p.storage.DiskStorage.MetricHighWatermarks, p.curationState)
}

func (p *prometheus) close() {
	select {
	case p.curationSema <- true:
	default:
	}

	if p.headCompactionTimer != nil {
		p.headCompactionTimer.Stop()
	}
	if p.bodyCompactionTimer != nil {
		p.bodyCompactionTimer.Stop()
	}
	if p.tailCompactionTimer != nil {
		p.tailCompactionTimer.Stop()
	}
	if p.deletionTimer != nil {
		p.deletionTimer.Stop()
	}

	if len(p.stopBackgroundOperations) == 0 {
		p.stopBackgroundOperations <- true
	}

	p.ruleManager.Stop()
	p.storage.Close()

	close(p.notifications)
	close(p.stopBackgroundOperations)
}

func main() {
	// TODO(all): Future additions to main should be, where applicable, glumped
	// into the prometheus struct above---at least where the scoping of the entire
	// server is concerned.
	flag.Parse()

	versionInfoTmpl.Execute(os.Stdout, BuildInfo)

	if *printVersion {
		os.Exit(0)
	}

	conf, err := config.LoadFromFile(*configFile)
	if err != nil {
		glog.Fatalf("Error loading configuration from %s: %v", *configFile, err)
	}

	ts, err := metric.NewTieredStorage(uint(*diskAppendQueueCapacity), 100, *arenaFlushInterval, *arenaTTL, *metricsStoragePath)
	if err != nil {
		glog.Fatal("Error opening storage: ", err)
	}

	unwrittenSamples := make(chan *extraction.Result, *samplesQueueCapacity)
	ingester := &retrieval.MergeLabelsIngester{
		Labels:          conf.GlobalLabels(),
		CollisionPrefix: clientmodel.ExporterLabelPrefix,

		Ingester: retrieval.ChannelIngester(unwrittenSamples),
	}
	// Coprime numbers, fool!
	headCompactionTimer := time.NewTicker(*headCompactInterval)
	bodyCompactionTimer := time.NewTicker(*bodyCompactInterval)
	tailCompactionTimer := time.NewTicker(*tailCompactInterval)
	deletionTimer := time.NewTicker(*deleteInterval)

	// Queue depth will need to be exposed
	targetManager := retrieval.NewTargetManager(ingester, *concurrentRetrievalAllowance)
	targetManager.AddTargetsFromConfig(conf)

	notifications := make(chan notification.NotificationReqs, *notificationQueueCapacity)

	// Queue depth will need to be exposed
	ruleManager := rules.NewRuleManager(&rules.RuleManagerOptions{
		Results:            unwrittenSamples,
		Notifications:      notifications,
		EvaluationInterval: conf.EvaluationInterval(),
		Storage:            ts,
		PrometheusUrl:      web.MustBuildServerUrl(),
	})
	if err := ruleManager.AddRulesFromConfig(conf); err != nil {
		glog.Fatal("Error loading rule files: ", err)
	}
	go ruleManager.Run()

	notificationHandler := notification.NewNotificationHandler(*alertmanagerUrl, notifications)
	go notificationHandler.Run()

	flags := map[string]string{}

	flag.VisitAll(func(f *flag.Flag) {
		flags[f.Name] = f.Value.String()
	})

	prometheusStatus := &web.PrometheusStatusHandler{
		BuildInfo:   BuildInfo,
		Config:      conf.String(),
		RuleManager: ruleManager,
		TargetPools: targetManager.Pools(),
		Flags:       flags,
		Birth:       time.Now(),
	}

	alertsHandler := &web.AlertsHandler{
		RuleManager: ruleManager,
	}

	databasesHandler := &web.DatabasesHandler{
		Provider:        ts.DiskStorage,
		RefreshInterval: 5 * time.Minute,
	}

	metricsService := &api.MetricsService{
		Config:        &conf,
		TargetManager: targetManager,
		Storage:       ts,
	}

	webService := &web.WebService{
		StatusHandler:    prometheusStatus,
		MetricsHandler:   metricsService,
		DatabasesHandler: databasesHandler,
		AlertsHandler:    alertsHandler,
	}

	prometheus := &prometheus{
		bodyCompactionTimer: bodyCompactionTimer,
		headCompactionTimer: headCompactionTimer,
		tailCompactionTimer: tailCompactionTimer,

		deletionTimer: deletionTimer,

		curationState: prometheusStatus,
		curationSema:  make(chan bool, 1),

		unwrittenSamples: unwrittenSamples,

		stopBackgroundOperations: make(chan bool, 1),

		ruleManager:   ruleManager,
		notifications: notifications,
		storage:       ts,
	}
	defer prometheus.close()

	storageStarted := make(chan bool)
	go ts.Serve(storageStarted)
	<-storageStarted

	go prometheus.interruptHandler()

	go func() {
		for _ = range prometheus.headCompactionTimer.C {
			glog.Info("Starting head compaction...")
			err := prometheus.compact(*headAge, *headGroupSize)

			if err != nil {
				glog.Error("could not compact: ", err)
			}
			glog.Info("Done")
		}
	}()

	go func() {
		for _ = range prometheus.bodyCompactionTimer.C {
			glog.Info("Starting body compaction...")
			err := prometheus.compact(*bodyAge, *bodyGroupSize)

			if err != nil {
				glog.Error("could not compact: ", err)
			}
			glog.Info("Done")
		}
	}()

	go func() {
		for _ = range prometheus.tailCompactionTimer.C {
			glog.Info("Starting tail compaction...")
			err := prometheus.compact(*tailAge, *tailGroupSize)

			if err != nil {
				glog.Error("could not compact: ", err)
			}
			glog.Info("Done")
		}
	}()

	go func() {
		for _ = range prometheus.deletionTimer.C {
			glog.Info("Starting deletion of stale values...")
			err := prometheus.delete(*deleteAge, deletionBatchSize)

			if err != nil {
				glog.Error("could not delete: ", err)
			}
			glog.Info("Done")
		}
	}()

	go func() {
		err := webService.ServeForever()
		if err != nil {
			glog.Fatal(err)
		}
	}()

	// TODO(all): Migrate this into prometheus.serve().
	for block := range unwrittenSamples {
		if block.Err == nil {
			ts.AppendSamples(block.Samples)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"os/exec"
	"strconv"
	"time"
)

var logic = `
   package logic

    import (
        "net/http"
        "fmt"
        "math/rand"
        "time"
    )

    type Context struct{
        value int64
    }

    func Init() {
        rand.Seed(time.Now().UTC().UnixNano())
    }

    func Start() Context {
        fmt.Println("Start new Context")
        return Context{1 + rand.Int63n(2)};
    }

    func Run(ctx Context, w http.ResponseWriter, r *http.Request) error {
        fmt.Fprintf(w, "Hello from Logic: %d", ctx.value)
        return nil
    }

    func Stop(ctx Context) {return }`

var core = `
    package main

    import(
        "effe/logic"
        "net/http"
        "sync"
        "log/syslog"
        "flag"
        "fmt"
    )

    func generateHandler(pool *sync.Pool, logger *syslog.Writer) func(http.ResponseWriter, *http.Request) {
        return func(w http.ResponseWriter, r *http.Request){
    	ctx := pool.Get().(logic.Context)
    	defer func() {
    	    if r := recover(); r != nil {
    		w.WriteHeader(http.StatusInternalServerError)
    		logger.Crit("Logic Panicked")
    	    }
    	}()
    	err := logic.Run(ctx, w, r)
    	if err != nil {
    	    logger.Debug(err.Error())
    	}
    	pool.Put(ctx)
        }
    }

    func main() {
        port := flag.Int("port", 8080, "Port where serve the effe.")
        flag.Parse()
        url := fmt.Sprintf(":%d", *port)
        logic.Init()
        logger, _ := syslog.New(syslog.LOG_ERR | syslog.LOG_USER, "Logs From Effe ")
        var ctxPool = &sync.Pool{New: func () interface{} {
    	return logic.Start()} }
        http.HandleFunc("/", generateHandler(ctxPool, logger))
        http.ListenAndServe(url, nil)
    }
`

func random_suffix() string {
	return strconv.Itoa(100000 + rand.Intn(1000000))
}

func new_file(path string, source string) error {
	_, err := os.Stat(path)
	if os.IsNotExist(err) {
		err = ioutil.WriteFile(path, []byte(source), 0644)
		if err != nil {
			return err
		}
		return nil
	}
	return errors.New("The file already exist")
}

// main
// create scafholding logic.go package
// create new dir
func main() {

	rand.Seed(time.Now().UnixNano())

	nw := flag.String("new", "", "Create new scafholding file")
	// project := flag.String("project", "", "Create new project")
	compile := flag.String("compile", "", "Compile the single source file")
	// release := flag.String("release", "", "Compile the files of the directories into binaries")
	// indipendent := flag.Bool("indipendent", true, "Compile all the binaries down to completely indipendent executables")

	flag.Parse()

	fmt.Println("Welcome :)")

	if *nw != "" {
		err := new_file(*nw, logic)
		if err != nil {
			fmt.Println("Error creating the new file.")
			fmt.Println(err)
			return
		} else {
			fmt.Println("New file sucessfully create, path: " + *nw)
		}
	}

	if *compile != "" {
		dir := "/tmp/effebuild-" + random_suffix()
		fmt.Println(dir)
		if err := os.Mkdir(dir, 0777); err != nil {
			fmt.Println(err)
			return
		}

		if err = os.Mkdir(dir+"/logic", 0777); err != nil {
			fmt.Println(err)
			return
		}

		if err = os.Link(*compile, dir+"/logic/logic.go"); err != nil {
			fmt.Println(err)
			return
		}

		if err = new_file(dir+"/effe.go", core); err != nil {
			fmt.Println("Impossible to create file, exit.")
			fmt.Println(err)
			return
		}

		cmd := exec.Command("go", "build", "-pkgdir", dir, "-o", dir+"/out", "-buildmode=exe", dir+"/effe.go")

		if err = cmd.Start(); err != nil {
			fmt.Println(err)
			return
		}

		if err = cmd.Wait(); err != nil {
			fmt.Println(err)
			return
		}
	}

}
<commit_msg>easy compilation<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"os/exec"
	"strconv"
	"time"
)

var logic = `
   package logic

    import (
        "net/http"
        "fmt"
        "math/rand"
        "time"
	log "github.com/Sirupsen/logrus"
    )

    type Context struct{
        value int64
    }

    func Init() {
        rand.Seed(time.Now().UTC().UnixNano())
    }

    func Start() Context {
        fmt.Println("Start new Context")
        return Context{1 + rand.Int63n(2)};
    }

    func Run(ctx Context, w http.ResponseWriter, r *http.Request) error {
        fmt.Fprintf(w, "Hello from Logic with log: %d", ctx.value)
	log.WithFields(log.Fields{
    		"animal": "walrus",
  		}).Info("A walrus appears")
        return nil
    }

    func Stop(ctx Context) {return }`

var core = `
    package main

    import(
        "effe/logic"
        "net/http"
        "sync"
        "log/syslog"
        "flag"
        "fmt"
    )

    func generateHandler(pool *sync.Pool, logger *syslog.Writer) func(http.ResponseWriter, *http.Request) {
        return func(w http.ResponseWriter, r *http.Request){
    	ctx := pool.Get().(logic.Context)
    	defer func() {
    	    if r := recover(); r != nil {
    		w.WriteHeader(http.StatusInternalServerError)
    		logger.Crit("Logic Panicked")
    	    }
    	}()
    	err := logic.Run(ctx, w, r)
    	if err != nil {
    	    logger.Debug(err.Error())
    	}
    	pool.Put(ctx)
        }
    }

    func main() {
        port := flag.Int("port", 8080, "Port where serve the effe.")
        flag.Parse()
        url := fmt.Sprintf(":%d", *port)
        logic.Init()
        logger, _ := syslog.New(syslog.LOG_ERR | syslog.LOG_USER, "Logs From Effe ")
        var ctxPool = &sync.Pool{New: func () interface{} {
    	return logic.Start()} }
        http.HandleFunc("/", generateHandler(ctxPool, logger))
        http.ListenAndServe(url, nil)
    }
`

func random_suffix() string {
	return strconv.Itoa(100000 + rand.Intn(1000000))
}

func new_file(path string, source string) error {
	_, err := os.Stat(path)
	if os.IsNotExist(err) {
		err = ioutil.WriteFile(path, []byte(source), 0644)
		if err != nil {
			return err
		}
		return nil
	}
	return errors.New("The file already exist")
}

// main
// create scafholding logic.go package
// create new dir
func main() {

	rand.Seed(time.Now().UnixNano())

	nw := flag.String("new", "", "Create new scafholding file")
	// project := flag.String("project", "", "Create new project")
	compile := flag.String("compile", "", "Compile the single source file")
	// release := flag.String("release", "", "Compile the files of the directories into binaries")
	// indipendent := flag.Bool("indipendent", true, "Compile all the binaries down to completely indipendent executables")

	flag.Parse()

	fmt.Println("Welcome :)")

	if *nw != "" {
		err := new_file(*nw, logic)
		if err != nil {
			fmt.Println("Error creating the new file.")
			fmt.Println(err)
			return
		} else {
			fmt.Println("New file sucessfully create, path: " + *nw)
		}
	}

	if *compile != "" {
		dir := "/tmp/effebuild-" + random_suffix()
		fmt.Println(dir)
		if err := os.Mkdir(dir, 0777); err != nil {
			fmt.Println(err)
			return
		}

		dir_src := dir + "/src"
		if err := os.Mkdir(dir_src, 0777); err != nil {
			fmt.Println(err)
			return
		}

		dir_effe := dir_src + "/effe"
		if err := os.Mkdir(dir_effe, 0777); err != nil {
			fmt.Println(err)
			return
		}

		if err := os.Mkdir(dir_effe+"/logic", 0777); err != nil {
			fmt.Println(err)
			return
		}

		if err := os.Link(*compile, dir_effe+"/logic/logic.go"); err != nil {
			fmt.Println(err)
			return
		}

		if err := new_file(dir_effe+"/effe.go", core); err != nil {
			fmt.Println("Impossible to create file, exit.")
			fmt.Println(err)
			return
		}

		gopath := os.Getenv("GOPATH")
		os.Setenv("GOPATH", dir+":"+gopath)
		defer os.Setenv("GOPATH", gopath)

		cmd := exec.Command("go", "build", "-a", "-o", dir+"/out", "-buildmode=exe", dir_effe+"/effe.go")

		if err := cmd.Start(); err != nil {
			fmt.Println(err)
			return
		}

		if err := cmd.Wait(); err != nil {
			fmt.Println(err)
			return
		}
	}

}
<|endoftext|>


<commit_before>package main

import (
    "fmt"
    "log"
    "net/http"
)

const listenAddr = "localhost:8080"

func main() {
    fmt.Println("Starting YAPC server!")

    http.HandleFunc("/", handler)

    err := http.ListenAndServe(listenAddr, nil)
    if err != nil {
        log.Fatal(err)
    }

    fmt.Println("Finishing YAPC server!")
}

func handler(w http.ResponseWriter, r *http.Request) {
    fmt.Fprint(w, "Hello, web")
}
<commit_msg>YAPC reads the request, sends a new request to upstream and does the data pipeline to the downstream.<commit_after>package main

import (
    "fmt"
    "log"
    "net/http"
    "io"
)

const listenAddr = ":8080"

func main() {
    fmt.Println("Starting YAPC server!")

    http.HandleFunc("/", handler)

    err := http.ListenAndServe(listenAddr, nil)
    if err != nil {
        log.Fatal(err)
    }

    fmt.Println("Finishing YAPC server!")
}

func handler(w http.ResponseWriter, r *http.Request) {
    url := r.URL.String()
    response, err := http.Get(url)
    if err != nil {
        log.Fatal(err)
    }

    defer response.Body.Close()

    // Using io.Copy to dump the upstream response to the downstream.
    _, err = io.Copy(w, response.Body)
    if err != nil {
        log.Fatal(err)
    }
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"syscall"
	"time"
)

const VERSION = "v0.0.1"

func main() {
	var i string
	flag.StringVar(&i, "i", "1s", "")
	flag.StringVar(&i, "interval", "1s", "")

	var interrupt bool
	flag.BoolVar(&interrupt, "x", false, "")
	flag.BoolVar(&interrupt, "exit", false, "")

	var clear bool
	flag.BoolVar(&clear, "c", false, "")
	flag.BoolVar(&clear, "clear", false, "")

	var version bool
	flag.BoolVar(&version, "v", false, "")
	flag.BoolVar(&version, "version", false, "")

	var usage bool
	flag.BoolVar(&usage, "h", false, "")
	flag.BoolVar(&usage, "help", false, "")

	flag.Parse()

	if version {
		die(0, VERSION)
	}

	command := flag.Args()

	if usage || len(command) == 0 {
		help()
		die(0, "")
	}

	if !commandExists(command) {
		die(0, "Executable not found in PATH")
	}

	interval := parseInterval(i)

	loop(intervalToTime(interval), func() {
		if clear {
			reset()
		}
		status := run(command)
		if interrupt && status != 0 {
			die(status, "")
		}
	})
}

func intervalSuffixes() []string {
	return []string{"ms", "s", "MS", "S"}
}

func unitToFactor(s string) float64 {
	if s == "ms" || s == "MS" {
		return 1.0
	}
	if s == "s" || s == "S" {
		return 1000.0
	}
	return 1000.0
}

func suffixToInterval(s string, i string) (float64, bool) {
	is_unit := strings.HasSuffix(i, s)
	if is_unit {
		n, err := strconv.ParseFloat(i[:len(i)-len(s)], 64)
		dieIf(err, -1, "Invalid interval")
		return n * unitToFactor(s), true
	}
	return 0, false
}

func parseInterval(i string) float64 {
	suffixes := intervalSuffixes()
	for j := 0; j < len(suffixes); j++ {
		if val, is := suffixToInterval(suffixes[j], i); is {
			return val
		}
	}
	return 0
}

func help() {
	s := `
   Usage: watch [options] <cmd>

   Sample: watch -i=100ms make

   Options:

     -c, --clear                clear screen between command runs
     -i, --interval             interval in seconds or ms, defaulting to 1s
     -x, --exit                 exit on failure

     -h, --help                 this help page
     -v, --version              print out version

`
	fmt.Print(s)
}

func dieIf(err error, status int, message string) {
	if err != nil {
		die(status, message)
	}
}

func die(status int, message string) {
	if len(message) > 0 {
		fmt.Println(message)
	}
	os.Exit(status)
}

func intervalToTime(i float64) time.Duration {
	return time.Duration(i) * time.Millisecond
}

func reset() {
	run([]string{"clear"})
}

func makeCmd(bin string, args []string) *exec.Cmd {
	cmd := exec.Command(bin, args...)
	cmd.Env = os.Environ()
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd
}

func getExitStatus(cmd *exec.Cmd) int {
	status, _ := cmd.ProcessState.Sys().(syscall.WaitStatus)
	return status.ExitStatus()
}

func commandExists(command []string) bool {
	_, result := exec.LookPath(command[0])
	return result == nil
}

func run(command []string) int {
	path, _ := exec.LookPath(command[0])
	cmd := makeCmd(path, command[1:])
	cmd.Run()
	return getExitStatus(cmd)
}

func loop(d time.Duration, fn func()) {
	fn()
	for range time.Tick(d) {
		fn()
	}
}
<commit_msg>Close #12<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"syscall"
	"time"
)

const VERSION = "v0.0.1"

func main() {
	var i string
	flag.StringVar(&i, "i", "1s", "")
	flag.StringVar(&i, "interval", "1s", "")

	var interrupt bool
	flag.BoolVar(&interrupt, "x", false, "")
	flag.BoolVar(&interrupt, "exit", false, "")

	var clear bool
	flag.BoolVar(&clear, "c", false, "")
	flag.BoolVar(&clear, "clear", false, "")

	var version bool
	flag.BoolVar(&version, "v", false, "")
	flag.BoolVar(&version, "version", false, "")

	var usage bool
	flag.BoolVar(&usage, "h", false, "")
	flag.BoolVar(&usage, "help", false, "")

	flag.Parse()

	if version {
		die(0, VERSION)
	}

	command := flag.Args()

	if usage || len(command) == 0 {
		help()
		die(0, "")
	}

	if !commandExists(command) {
		die(0, "Executable not found in PATH")
	}

	interval := parseInterval(i)

	loop(intervalToTime(interval), func() {
		if clear {
			reset()
		}
		status := run(command)
		printStatus(status)
		if interrupt && status != 0 {
			die(status, "")
		}
	})
}

func printStatus(s int) {
	fmt.Printf("\033[90mexit: %d\033[0m\n\n", s)
}

func intervalSuffixes() []string {
	return []string{"ms", "s", "MS", "S"}
}

func unitToFactor(s string) float64 {
	if s == "ms" || s == "MS" {
		return 1.0
	}
	if s == "s" || s == "S" {
		return 1000.0
	}
	return 1000.0
}

func suffixToInterval(s string, i string) (float64, bool) {
	is_unit := strings.HasSuffix(i, s)
	if is_unit {
		n, err := strconv.ParseFloat(i[:len(i)-len(s)], 64)
		dieIf(err, -1, "Invalid interval")
		return n * unitToFactor(s), true
	}
	return 0, false
}

func parseInterval(i string) float64 {
	suffixes := intervalSuffixes()
	for j := 0; j < len(suffixes); j++ {
		if val, is := suffixToInterval(suffixes[j], i); is {
			return val
		}
	}
	return 0
}

func help() {
	s := `
   Usage: watch [options] <cmd>

   Sample: watch -i=100ms make

   Options:

     -c, --clear                clear screen between command runs
     -i, --interval             interval in seconds or ms, defaulting to 1s
     -x, --exit                 exit on failure

     -h, --help                 this help page
     -v, --version              print out version

`
	fmt.Print(s)
}

func dieIf(err error, status int, message string) {
	if err != nil {
		die(status, message)
	}
}

func die(status int, message string) {
	if len(message) > 0 {
		fmt.Println(message)
	}
	os.Exit(status)
}

func intervalToTime(i float64) time.Duration {
	return time.Duration(i) * time.Millisecond
}

func reset() {
	run([]string{"clear"})
}

func makeCmd(bin string, args []string) *exec.Cmd {
	cmd := exec.Command(bin, args...)
	cmd.Env = os.Environ()
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd
}

func getExitStatus(cmd *exec.Cmd) int {
	status, _ := cmd.ProcessState.Sys().(syscall.WaitStatus)
	return status.ExitStatus()
}

func commandExists(command []string) bool {
	_, result := exec.LookPath(command[0])
	return result == nil
}

func run(command []string) int {
	path, _ := exec.LookPath(command[0])
	cmd := makeCmd(path, command[1:])
	cmd.Run()
	return getExitStatus(cmd)
}

func loop(d time.Duration, fn func()) {
	fn()
	for range time.Tick(d) {
		fn()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"time"

	"github.com/tylertreat/vessel/vessel"
)

func main() {
	vessel := vessel.NewSockJSVessel("/vessel")

	vessel.AddChannel("foo", func(msg string, c chan<- string, done chan<- bool) {
		c <- "ping"
		done <- true
	})

	go func() {
		c := time.Tick(5 * time.Second)
		for {
			<-c
			vessel.Broadcast("baz", "testing 123")
		}
	}()

	vessel.Start(":8081", ":8082")
}
<commit_msg>Test periodic responses<commit_after>package main

import (
	"strconv"
	"time"

	"github.com/tylertreat/vessel/vessel"
)

func main() {
	vessel := vessel.NewSockJSVessel("/vessel")

	vessel.AddChannel("foo", func(msg string, c chan<- string, done chan<- bool) {
		for x := 0; x < 10; x++ {
			c <- strconv.Itoa(x)
			time.Sleep(time.Second)
		}
		c <- "ping"
		done <- true
	})

	go func() {
		c := time.Tick(5 * time.Second)
		for {
			<-c
			vessel.Broadcast("baz", "testing 123")
		}
	}()

	vessel.Start(":8081", ":8082")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math/rand"
	"time"
)

func main() {

	const strSize int = 4

	fmt.Println("=======================================================")

	// generate a random string of hex chars
	fmt.Println("GENERATING RANDOM NUMBERS")
	str := make([]byte, strSize) // 0000 0001 0010 -> AS
	rand.Seed(time.Now().UnixNano())
	for i := 0; i < len(str); i++ {
		val := rand.Intn(16)
		if val < 10 {
			str[i] = byte(val + 48) // 48 is 0
		} else {
			str[i] = byte(val - 10 + 97) // 97 is a
		}
	}
	fmt.Println("Done!")
	fmt.Println("=======================================================")

	fmt.Println("CONVERTING BASE 16 to BASE 64")
	start := time.Now()
	b64 := base16to64(str)
	runTime := time.Since(start)
	fmt.Println("time: ", runTime)

	if len(b64) <= 100 {
		fmt.Printf("b16: %s\n", str)
		fmt.Printf("b64: %s\n", b64)
		fmt.Printf("b64: %v\n", b64)
	}

	fmt.Printf("SIZE: %d => %d\n", len(str), len(b64))

	fmt.Println("=======================================================")

	fmt.Println("CONVERTING BASE 64 BACK TO BASE 16")
	start = time.Now()
	b16 := base64to16(b64)
	runTime = time.Since(start)
	fmt.Println("time: ", runTime)

	if len(b64) <= 100 {
		fmt.Printf("b64: %s\n", b64)
		fmt.Printf("b16: %s\n", b16)
		fmt.Printf("b16: %v\n", b16)
	}

	fmt.Printf("SIZE: %d => %d\n", len(b64), len(b16))

	fmt.Println("=======================================================")

	fmt.Println("TEST FOR EQUALITY")
	var eq = true
	if len(b16) != len(str) {
		eq = false
	} else {
		for i := 0; i < len(str); i++ {
			if b16[i] != str[i] {
				eq = false
			}
		}
	}
	if eq == true {
		fmt.Println("SUCCESS! End result matches initial string!")
	} else {
		fmt.Println("FAIL! The result string does NOT match the initial string!")
	}

}
<commit_msg>Update function calls in main to use upper-case function name that I forgot to change in the last commit<commit_after>package main

import (
	"fmt"
	"math/rand"
	"time"
)

func main() {

	const strSize int = 1000000000

	fmt.Println("=======================================================")

	// generate a random string of hex chars
	fmt.Println("GENERATING RANDOM NUMBERS")
	str := make([]byte, strSize) // 0000 0001 0010 -> AS
	rand.Seed(time.Now().UnixNano())
	for i := 0; i < len(str); i++ {
		val := rand.Intn(16)
		if val < 10 {
			str[i] = byte(val + 48) // 48 is 0
		} else {
			str[i] = byte(val - 10 + 97) // 97 is a
		}
	}
	fmt.Println("Done!")
	fmt.Println("=======================================================")

	fmt.Println("CONVERTING BASE 16 to BASE 64")
	start := time.Now()
	b64 := Base16to64(str)
	runTime := time.Since(start)
	fmt.Println("time: ", runTime)

	if len(b64) <= 100 {
		fmt.Printf("b16: %s\n", str)
		fmt.Printf("b64: %s\n", b64)
		fmt.Printf("b64: %v\n", b64)
	}

	fmt.Printf("SIZE: %d => %d\n", len(str), len(b64))

	fmt.Println("=======================================================")

	fmt.Println("CONVERTING BASE 64 BACK TO BASE 16")
	start = time.Now()
	b16 := Base64to16(b64)
	runTime = time.Since(start)
	fmt.Println("time: ", runTime)

	if len(b64) <= 100 {
		fmt.Printf("b64: %s\n", b64)
		fmt.Printf("b16: %s\n", b16)
		fmt.Printf("b16: %v\n", b16)
	}

	fmt.Printf("SIZE: %d => %d\n", len(b64), len(b16))

	fmt.Println("=======================================================")

	fmt.Println("TEST FOR EQUALITY")
	var eq = true
	if len(b16) != len(str) {
		eq = false
	} else {
		for i := 0; i < len(str); i++ {
			if b16[i] != str[i] {
				eq = false
			}
		}
	}
	if eq == true {
		fmt.Println("SUCCESS! End result matches initial string!")
	} else {
		fmt.Println("FAIL! The result string does NOT match the initial string!")
	}

}
<|endoftext|>


<commit_before>// 2015 - Mathieu Lonjaret

// The lw program lists all existing acme windows. The output is sorted
// according to the following criteria, in order:
// - whether the window is clean
// - whether it is corresponding to a file on disk
// - the file modtime
// As usual with acme, if lw is run from the tag bar, the output goes to the
// Errors window.
package main

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"os"
	"sort"
	"strconv"
	"time"

	"9fans.net/go/acme"
)

var fileOnly = flag.Bool("f", false, "Only list windows corresponding to files.")

func usage() {
	fmt.Fprintf(os.Stderr, "usage: lw\n")
	flag.PrintDefaults()
	os.Exit(2)
}

type winInfo struct {
	w       acme.WinInfo
	dirty   bool
	modTime time.Time // if zero -> not a file
}

type winInfos []winInfo

func (w winInfos) Len() int      { return len(w) }
func (w winInfos) Swap(i, j int) { w[i], w[j] = w[j], w[i] }
func (w winInfos) Less(i, j int) bool {
	if w[i].dirty {
		if !w[j].dirty {
			return true
		}
		if w[i].modTime.IsZero() {
			return true
		}
		if w[j].modTime.IsZero() {
			return false
		}
		return w[i].modTime.After(w[j].modTime)
	}
	if w[j].dirty {
		return false
	}
	return w[i].modTime.After(w[j].modTime)
}

func main() {
	flag.Usage = usage
	flag.Parse()

	var allWins []winInfo

	windows, err := acme.Windows()
	if err != nil {
		log.Fatalf("could not get acme windows: %v", err)
	}
	for _, win := range windows {
		w, err := acme.Open(win.ID, nil)
		if err != nil {
			log.Fatalf("could not open window (%v, %d): %v", win.Name, win.ID, err)
		}
		defer w.CloseFiles()
		b, err := w.ReadAll("ctl")
		if err != nil {
			log.Fatalf("could not read ctl file of (%v, %d): %v", win.Name, win.ID, err)
		}
		fields := bytes.Fields(b)
		if len(fields) != 8 {
			log.Fatalf("unexpected number of fields for (%v, %d): wanted %v, got %v", win.Name, win.ID, 8, len(fields))
		}
		isDirty, _ := strconv.ParseBool(string(fields[4]))
		wini := winInfo{
			w:     win,
			dirty: isDirty,
		}
		fi, err := os.Stat(win.Name)
		if err != nil {
			if !os.IsNotExist(err) {
				log.Fatalf("could not stat disk file of (%v, %d): %v", win.Name, win.ID, err)
			}
			if *fileOnly {
				continue
			}
		} else {
			if *fileOnly && fi.IsDir() {
				continue
			}
			wini.modTime = fi.ModTime()
		}
		allWins = append(allWins, wini)
	}
	sort.Sort(winInfos(allWins))
	dirtyness := map[bool]string{
		true:  "dirty",
		false: "clean",
	}
	println()
	for _, v := range allWins {
		fmt.Printf("%v	%v\n", dirtyness[v.dirty], v.w.Name)
	}
}
<commit_msg>Be less strict on nb of fields for ctl output<commit_after>// 2015 - Mathieu Lonjaret

// The lw program lists all existing acme windows. The output is sorted
// according to the following criteria, in order:
// - whether the window is clean
// - whether it is corresponding to a file on disk
// - the file modtime
// As usual with acme, if lw is run from the tag bar, the output goes to the
// Errors window.
package main

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"os"
	"sort"
	"strconv"
	"time"

	"9fans.net/go/acme"
)

var fileOnly = flag.Bool("f", false, "Only list windows corresponding to files.")

func usage() {
	fmt.Fprintf(os.Stderr, "usage: lw\n")
	flag.PrintDefaults()
	os.Exit(2)
}

type winInfo struct {
	w       acme.WinInfo
	dirty   bool
	modTime time.Time // if zero -> not a file
}

type winInfos []winInfo

func (w winInfos) Len() int      { return len(w) }
func (w winInfos) Swap(i, j int) { w[i], w[j] = w[j], w[i] }
func (w winInfos) Less(i, j int) bool {
	if w[i].dirty {
		if !w[j].dirty {
			return true
		}
		if w[i].modTime.IsZero() {
			return true
		}
		if w[j].modTime.IsZero() {
			return false
		}
		return w[i].modTime.After(w[j].modTime)
	}
	if w[j].dirty {
		return false
	}
	return w[i].modTime.After(w[j].modTime)
}

func main() {
	flag.Usage = usage
	flag.Parse()

	var allWins []winInfo

	windows, err := acme.Windows()
	if err != nil {
		log.Fatalf("could not get acme windows: %v", err)
	}
	for _, win := range windows {
		w, err := acme.Open(win.ID, nil)
		if err != nil {
			log.Fatalf("could not open window (%v, %d): %v", win.Name, win.ID, err)
		}
		defer w.CloseFiles()
		b, err := w.ReadAll("ctl")
		if err != nil {
			log.Fatalf("could not read ctl file of (%v, %d): %v", win.Name, win.ID, err)
		}
		fields := bytes.Fields(b)
		// the number of fields is unpredictable as 7th field (the font
		// used) can have spaces in it. I could use a regexp, but who
		// cares, as the field I'm interested in is before in the line
		// anyway.
		if len(fields) < 8 {
			log.Fatalf("unexpected number of fields for (%v, %d): wanted at least %v, got %v", win.Name, win.ID, 8, len(fields))
		}
		isDirty, _ := strconv.ParseBool(string(fields[4]))
		wini := winInfo{
			w:     win,
			dirty: isDirty,
		}
		fi, err := os.Stat(win.Name)
		if err != nil {
			if !os.IsNotExist(err) {
				log.Fatalf("could not stat disk file of (%v, %d): %v", win.Name, win.ID, err)
			}
			if *fileOnly {
				continue
			}
		} else {
			if *fileOnly && fi.IsDir() {
				continue
			}
			wini.modTime = fi.ModTime()
		}
		allWins = append(allWins, wini)
	}
	sort.Sort(winInfos(allWins))
	dirtyness := map[bool]string{
		true:  "dirty",
		false: "clean",
	}
	println()
	for _, v := range allWins {
		fmt.Printf("%v	%v\n", dirtyness[v.dirty], v.w.Name)
	}
}
<|endoftext|>