<commit_before>package main

import (
  "strconv"
  "fmt"
  "encoding/json"
	"./mqttc"
	"git.eclipse.org/gitroot/paho/org.eclipse.paho.mqtt.golang.git"
	log "github.com/Sirupsen/logrus"
	"github.com/marpaia/graphite-golang"
	"gopkg.in/alecthomas/kingpin.v1"
	"net/url"
	"os"
	"strings"
	"time"
)

var (
	Graphite   *graphite.Graphite
	NOOP = false
)

type Host struct {
	IP          string  `json:"ip"`
	Name        string  `json:"hostname"`
	Hop         int     `json:"hop-number"`
	Sent        int     `json:"sent"`
	LostPercent float64 `json:"lost-percent"`
	Last        float64 `json:"last"`
	Avg         float64 `json:"avg"`
	Best        float64 `json:"best"`
	Worst       float64 `json:"worst"`
	StDev       float64 `json:"standard-dev"`
}

type Report struct {
	Time        time.Time       `json:"time"`
	Hosts       []*Host         `json:"hosts"`
	Hops        int             `json:"hops"`
	ElapsedTime time.Duration   `json:"elapsed_time"`
	Location    *ReportLocation `json:"location"`
}

// slightly simpler struct than the one provided by geoipc
type ReportLocation struct {
	IP          string  `json:"ip"`
	CountryCode string  `json:"country_code"`
	CountryName string  `json:"country_name"`
	City        string  `json:"city"`
	Latitude    float64 `json:"latitude"`
	Longitude   float64 `json:"longitude"`
}

func sendMetrics(client *mqtt.MqttClient, msg mqtt.Message) {
	var report Report
	err := json.Unmarshal(msg.Payload(), &report)
	if err != nil {
		log.Error("Error decoding json report")
	}

  ip := report.Location.IP
	if ip == "" {
		log.Warnf("Discarding report with no source IP")
		return
	}
	ip = strings.Replace(ip, ".", "_", -1)

  cc := report.Location.CountryCode
	if cc == "" {
		log.Warnf("Discarding report with no country code")
		return
	}

	city := strings.Replace(report.Location.City, " ", "_", -1)
	hops := report.Hops
	if city == "" {
		city = "nil"
	}
	last_hop := report.Hosts[len(report.Hosts) - 1]

	metric_prefix := strings.ToLower(fmt.Sprintf("push-mtr.%s.%s.%s", ip, cc, city))
	mHops := metric_prefix + ".hops"
	mLastHopAvg := metric_prefix + ".last_hop.avg"
	mLastHopBest := metric_prefix + ".last_hop.best"
	mLastHopWorst := metric_prefix + ".last_hop.worst"

	sendMetric(mHops, strconv.Itoa(hops))
	sendMetric(mLastHopAvg, strconv.FormatFloat(last_hop.Avg, 'f', 3, 64))
	sendMetric(mLastHopBest, strconv.FormatFloat(last_hop.Best, 'f', 3, 64))
	sendMetric(mLastHopWorst, strconv.FormatFloat(last_hop.Worst, 'f', 3, 64))
}

func sendMetric(key, value string) {
	if NOOP {
		log.Infof("NOOP: Sending metric %s %s", key, value)
		return
	}
	err := Graphite.SimpleSend(key, value)
	if err != nil {
		log.Debugf("Error sending metric %s: %s", key, err)
	}
}

func parseBrokerUrls(brokerUrls string) []string {
	tokens := strings.Split(brokerUrls, ",")
	for i, url := range tokens {
		tokens[i] = strings.TrimSpace(url)
	}

	return tokens
}

func main() {
	kingpin.Version(PKG_VERSION)

	brokerUrls := kingpin.Flag("broker-urls", "Comman separated MQTT broker URLs").
		Required().Default("").OverrideDefaultFromEnvar("MQTT_URLS").String()

	cafile := kingpin.Flag("cafile", "CA certificate when using TLS (optional)").
		String()

	topic := kingpin.Flag("topic", "MQTT topic").
		Default("/metrics/mtr").String()

	graphiteHost := kingpin.Flag("graphiteHost", "Graphite host").
		Default("localhost").String()

	graphitePort := kingpin.Flag("graphitePort", "Graphite port").
		Default("2003").Int()

	graphitePing := kingpin.Flag("graphitePing", "Try to reconnect to graphite every X seconds").
		Default("15").Int()

	insecure := kingpin.Flag("insecure", "Don't verify the server's certificate chain and host name.").
		Default("false").Bool()

	debug := kingpin.Flag("debug", "Print debugging messages").
		Default("false").Bool()

	noop := kingpin.Flag("noop", "Print the metrics being sent instead of sending them").
		Default("false").Bool()

	kingpin.Parse()

	if *debug {
		log.SetLevel(log.DebugLevel)
	}

	NOOP = *noop

	if *cafile != "" {
		if _, err := os.Stat(*cafile); err != nil {
			log.Fatalf("Error reading CA certificate %s", err.Error())
			os.Exit(1)
		}
	}

	urlList := parseBrokerUrls(*brokerUrls)

	var gerr error
	Graphite, gerr = graphite.NewGraphite(*graphiteHost, *graphitePort)
	if gerr != nil {
		log.Warn("Error connecting to graphite")
		os.Exit(1)
	} else {
		log.Info("Connected to graphite")
		log.Debugf("Loaded Graphite connection: %#v", Graphite)
	}

	for _, urlStr := range urlList {
		args := mqttc.Args{
			BrokerURLs:    []string{urlStr},
			ClientID:      "mtr-collect",
			Topic:         *topic,
			TLSCACertPath: *cafile,
			TLSSkipVerify: *insecure,
		}

		uri, _ := url.Parse(urlStr)
		c := mqttc.Subscribe(sendMetrics, &args)
		defer c.Disconnect(0)
		host := strings.Split(uri.Host, ":")[0]
		host = strings.Replace(host, ".", "_", -1)
	}

	// Try to reconnect every graphitePing sec if sending fails by sending
	// a fake ping metric
	// FIXME: better handled by the Graphite client
	go func() {
		for {
			time.Sleep(time.Duration(*graphitePing) * time.Second)
			err := Graphite.SimpleSend("ping metric", "")
			if err != nil {
				log.Warn("Ping metric failed, trying to reconnect")
				err = Graphite.Connect()
				if err != nil {
					log.Error("Reconnecting to graphite failed")
				} else {
					log.Info("Reconnected to graphite")
				}
			}
		}
	}()

	// wait endlessly
	var loop chan bool
	loop <- true
}
<commit_msg>Add debugging and send new metric<commit_after>package main

import (
	"./mqttc"
	"encoding/json"
	"fmt"
	"git.eclipse.org/gitroot/paho/org.eclipse.paho.mqtt.golang.git"
	log "github.com/Sirupsen/logrus"
	"github.com/marpaia/graphite-golang"
	"gopkg.in/alecthomas/kingpin.v1"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"
)

var (
	Graphite *graphite.Graphite
	NOOP     = false
	VERBOSE  = true
)

type Host struct {
	IP          string  `json:"ip"`
	Name        string  `json:"hostname"`
	Hop         int     `json:"hop-number"`
	Sent        int     `json:"sent"`
	LostPercent float64 `json:"lost-percent"`
	Last        float64 `json:"last"`
	Avg         float64 `json:"avg"`
	Best        float64 `json:"best"`
	Worst       float64 `json:"worst"`
	StDev       float64 `json:"standard-dev"`
}

type Report struct {
	Time        time.Time       `json:"time"`
	Hosts       []*Host         `json:"hosts"`
	Hops        int             `json:"hops"`
	ElapsedTime time.Duration   `json:"elapsed_time"`
	Location    *ReportLocation `json:"location"`
}

// slightly simpler struct than the one provided by geoipc
type ReportLocation struct {
	IP          string  `json:"ip"`
	CountryCode string  `json:"country_code"`
	CountryName string  `json:"country_name"`
	City        string  `json:"city"`
	Latitude    float64 `json:"latitude"`
	Longitude   float64 `json:"longitude"`
}

func sendMetrics(client *mqtt.MqttClient, msg mqtt.Message) {
	var report Report
	err := json.Unmarshal(msg.Payload(), &report)
	if err != nil {
		log.Error("Error decoding json report")
	}

	ip := report.Location.IP
	if ip == "" {
		log.Warnf("Discarding report with no source IP")
		return
	}
	ip = strings.Replace(ip, ".", "_", -1)

	cc := report.Location.CountryCode
	if cc == "" {
		log.Warnf("Discarding report with no country code")
		return
	}

	city := strings.Replace(report.Location.City, " ", "_", -1)
	hops := report.Hops
	if city == "" {
		city = "nil"
	}

	pktLossHops := 0
	for _, host := range report.Hosts {
		if host.LostPercent != 0 && host.LostPercent != 100 {
			pktLossHops += 1
		}
	}

	last_hop := report.Hosts[len(report.Hosts)-1]

	metric_prefix := strings.ToLower(fmt.Sprintf("push-mtr.%s.%s.%s", ip, cc, city))
	mHops := metric_prefix + ".hops"
	mLastHopAvg := metric_prefix + ".last_hop.avg"
	mLastHopBest := metric_prefix + ".last_hop.best"
	mLastHopWorst := metric_prefix + ".last_hop.worst"

	sendMetric(metric_prefix+".pkt_loss_hops", strconv.Itoa(pktLossHops))
	sendMetric(mHops, strconv.Itoa(hops))
	sendMetric(mLastHopAvg, strconv.FormatFloat(last_hop.Avg, 'f', 3, 64))
	sendMetric(mLastHopBest, strconv.FormatFloat(last_hop.Best, 'f', 3, 64))
	sendMetric(mLastHopWorst, strconv.FormatFloat(last_hop.Worst, 'f', 3, 64))
}

func sendMetric(key, value string) {
	if NOOP {
		log.Infof("NOOP: Sending metric %s %s", key, value)
		return
	}
	err := Graphite.SimpleSend(key, value)
	if err != nil {
		log.Debugf("Error sending metric %s: %s", key, err)
	}
}

func parseBrokerUrls(brokerUrls string) []string {
	tokens := strings.Split(brokerUrls, ",")
	for i, url := range tokens {
		tokens[i] = strings.TrimSpace(url)
	}

	return tokens
}

func main() {
	kingpin.Version(PKG_VERSION)

	brokerUrls := kingpin.Flag("broker-urls", "Comman separated MQTT broker URLs").
		Required().Default("").OverrideDefaultFromEnvar("MQTT_URLS").String()

	cafile := kingpin.Flag("cafile", "CA certificate when using TLS (optional)").
		String()

	topic := kingpin.Flag("topic", "MQTT topic").
		Default("/metrics/mtr").String()

	graphiteHost := kingpin.Flag("graphiteHost", "Graphite host").
		Default("localhost").String()

	clientID := kingpin.Flag("clientid", "Use a custom MQTT client ID").String()

	graphitePort := kingpin.Flag("graphitePort", "Graphite port").
		Default("2003").Int()

	graphitePing := kingpin.Flag("graphitePing", "Try to reconnect to graphite every X seconds").
		Default("15").Int()

	insecure := kingpin.Flag("insecure", "Don't verify the server's certificate chain and host name.").
		Default("false").Bool()

	debug := kingpin.Flag("debug", "Print debugging messages").
		Default("false").Bool()

	VERBOSE = *(kingpin.Flag("verbose", "Print metrics being sent").
		Default("false").Bool())

	noop := kingpin.Flag("noop", "Print the metrics being sent instead of sending them").
		Default("false").Bool()

	kingpin.Parse()

	if *debug {
		log.SetLevel(log.DebugLevel)
	}

	NOOP = *noop
	var err error

	if *cafile != "" {
		if _, err := os.Stat(*cafile); err != nil {
			log.Fatalf("Error reading CA certificate %s", err.Error())
			os.Exit(1)
		}
	}

	urlList := parseBrokerUrls(*brokerUrls)

	var gerr error
	Graphite, gerr = graphite.NewGraphite(*graphiteHost, *graphitePort)
	if gerr != nil {
		log.Warn("Error connecting to graphite")
		os.Exit(1)
	} else {
		log.Info("Connected to graphite")
		log.Debugf("Loaded Graphite connection: %#v", Graphite)
	}

	if *clientID == "" {
		*clientID, err = os.Hostname()
		if err != nil {
			log.Fatal("Can't get the hostname to use it as the ClientID, use --clientid option")
		}
	}
	log.Debugf("MQTT Client ID: %s", *clientID)

	for _, urlStr := range urlList {
		args := mqttc.Args{
			BrokerURLs:    []string{urlStr},
			ClientID:      *clientID,
			Topic:         *topic,
			TLSCACertPath: *cafile,
			TLSSkipVerify: *insecure,
		}

		uri, _ := url.Parse(urlStr)
		log.Debug("Starting mqttc client")
		c := mqttc.Subscribe(sendMetrics, &args)
		defer c.Disconnect(0)
		host := strings.Split(uri.Host, ":")[0]
		host = strings.Replace(host, ".", "_", -1)
	}

	// Try to reconnect every graphitePing sec if sending fails by sending
	// a fake ping metric
	// FIXME: better handled by the Graphite client
	go func() {
		for {
			time.Sleep(time.Duration(*graphitePing) * time.Second)
			err := Graphite.SimpleSend("ping metric", "")
			if err != nil {
				log.Warn("Ping metric failed, trying to reconnect")
				err = Graphite.Connect()
				if err != nil {
					log.Error("Reconnecting to graphite failed")
				} else {
					log.Info("Reconnected to graphite")
				}
			}
		}
	}()

	// wait endlessly
	var loop chan bool
	loop <- true
}
<|endoftext|>


<commit_before>package virtualbox

import (
	"bytes"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	log "github.com/Sirupsen/logrus"
)

var (
	reVMNameUUID      = regexp.MustCompile(`"(.+)" {([0-9a-f-]+)}`)
	reVMInfoLine      = regexp.MustCompile(`(?:"(.+)"|(.+))=(?:"(.*)"|(.*))`)
	reColonLine       = regexp.MustCompile(`(.+):\s+(.*)`)
	reMachineNotFound = regexp.MustCompile(`Could not find a registered machine named '(.+)'`)
)

var (
	ErrMachineExist    = errors.New("machine already exists")
	ErrMachineNotExist = errors.New("machine does not exist")
	ErrVBMNotFound     = errors.New("VBoxManage not found")
	vboxManageCmd      = setVBoxManageCmd()
)

// detect the VBoxManage cmd's path if needed
func setVBoxManageCmd() string {
	cmd := "VBoxManage"
	if path, err := exec.LookPath(cmd); err == nil {
		return path
	}
	if runtime.GOOS == "windows" {
		if p := os.Getenv("VBOX_INSTALL_PATH"); p != "" {
			if path, err := exec.LookPath(filepath.Join(p, cmd)); err == nil {
				return path
			}
		}
		if p := os.Getenv("VBOX_MSI_INSTALL_PATH"); p != "" {
			if path, err := exec.LookPath(filepath.Join(p, cmd)); err == nil {
				return path
			}
		}
		// look at HKEY_LOCAL_MACHINE\SOFTWARE\Oracle\VirtualBox\InstallDir
		p := "C:\\Program Files\\Oracle\\VirtualBox"
		if path, err := exec.LookPath(filepath.Join(p, cmd)); err == nil {
			return path
		}
	}
	return cmd
}

func vbm(args ...string) error {
	cmd := exec.Command(vboxManageCmd, args...)
	if os.Getenv("DEBUG") != "" {
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	}
	log.Debugf("executing: %v %v", vboxManageCmd, strings.Join(args, " "))
	if err := cmd.Run(); err != nil {
		if ee, ok := err.(*exec.Error); ok && ee == exec.ErrNotFound {
			return ErrVBMNotFound
		}
		return fmt.Errorf("%v %v failed: %v", vboxManageCmd, strings.Join(args, " "), err)
	}
	return nil
}

func vbmOut(args ...string) (string, error) {
	cmd := exec.Command(vboxManageCmd, args...)
	if os.Getenv("DEBUG") != "" {
		cmd.Stderr = os.Stderr
	}
	log.Debugf("executing: %v %v", vboxManageCmd, strings.Join(args, " "))

	b, err := cmd.Output()
	if err != nil {
		if ee, ok := err.(*exec.Error); ok && ee == exec.ErrNotFound {
			err = ErrVBMNotFound
		}
	}
	return string(b), err
}

func vbmOutErr(args ...string) (string, string, error) {
	cmd := exec.Command(vboxManageCmd, args...)
	log.Debugf("executing: %v %v", vboxManageCmd, strings.Join(args, " "))
	var stdout bytes.Buffer
	var stderr bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	err := cmd.Run()
	if err != nil {
		if ee, ok := err.(*exec.Error); ok && ee == exec.ErrNotFound {
			err = ErrVBMNotFound
		}
	}
	return stdout.String(), stderr.String(), err
}
<commit_msg>Detect vboxmanage error conditions and pass those on as errors<commit_after>package virtualbox

import (
	"bytes"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	log "github.com/Sirupsen/logrus"
)

var (
	reVMNameUUID      = regexp.MustCompile(`"(.+)" {([0-9a-f-]+)}`)
	reVMInfoLine      = regexp.MustCompile(`(?:"(.+)"|(.+))=(?:"(.*)"|(.*))`)
	reColonLine       = regexp.MustCompile(`(.+):\s+(.*)`)
	reMachineNotFound = regexp.MustCompile(`Could not find a registered machine named '(.+)'`)
)

var (
	ErrMachineExist    = errors.New("machine already exists")
	ErrMachineNotExist = errors.New("machine does not exist")
	ErrVBMNotFound     = errors.New("VBoxManage not found")
	vboxManageCmd      = setVBoxManageCmd()
)

// detect the VBoxManage cmd's path if needed
func setVBoxManageCmd() string {
	cmd := "VBoxManage"
	if path, err := exec.LookPath(cmd); err == nil {
		return path
	}
	if runtime.GOOS == "windows" {
		if p := os.Getenv("VBOX_INSTALL_PATH"); p != "" {
			if path, err := exec.LookPath(filepath.Join(p, cmd)); err == nil {
				return path
			}
		}
		if p := os.Getenv("VBOX_MSI_INSTALL_PATH"); p != "" {
			if path, err := exec.LookPath(filepath.Join(p, cmd)); err == nil {
				return path
			}
		}
		// look at HKEY_LOCAL_MACHINE\SOFTWARE\Oracle\VirtualBox\InstallDir
		p := "C:\\Program Files\\Oracle\\VirtualBox"
		if path, err := exec.LookPath(filepath.Join(p, cmd)); err == nil {
			return path
		}
	}
	return cmd
}

func vbm(args ...string) error {
	_, _, err := vbmOutErr(args...)
	return err
}

func vbmOut(args ...string) (string, error) {
	stdout, _, err := vbmOutErr(args...)
	return stdout, err
}

func vbmOutErr(args ...string) (string, string, error) {
	cmd := exec.Command(vboxManageCmd, args...)
	log.Debugf("executing: %v %v", vboxManageCmd, strings.Join(args, " "))
	var stdout bytes.Buffer
	var stderr bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	err := cmd.Run()
	stderrStr := stderr.String()
	log.Debugf("STDOUT: %v", stdout.String())
	log.Debugf("STDERR: %v", stderrStr)
	if err != nil {
		if ee, ok := err.(*exec.Error); ok && ee == exec.ErrNotFound {
			err = ErrVBMNotFound
		}
	} else {
		// VBoxManage will sometimes not set the return code, but has a fatal error
		// such as VBoxManage.exe: error: VT-x is not available. (VERR_VMX_NO_VMX)
		if strings.Contains(stderrStr, "error:") {
			err = fmt.Errorf("%v %v failed: %v", vboxManageCmd, strings.Join(args, " "), stderrStr)
		}
	}
	return stdout.String(), stderrStr, err
}
<|endoftext|>


<commit_before>package build

import (
	"os"
	"text/template"

	"github.com/drone/drone-cli/drone/internal"
	"github.com/drone/drone-go/drone"
	"github.com/urfave/cli"
)

var buildListCmd = cli.Command{
	Name:      "ls",
	Usage:     "show build history",
	ArgsUsage: "<repo/name>",
	Action:    buildList,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "format",
			Usage: "format output",
			Value: tmplBuildList,
		},
		cli.StringFlag{
			Name:  "branch",
			Usage: "branch filter",
		},
		cli.StringFlag{
			Name:  "event",
			Usage: "event filter",
		},
		cli.StringFlag{
			Name:  "status",
			Usage: "status filter",
		},
		cli.IntFlag{
			Name:  "limit",
			Usage: "limit the list size",
			Value: 25,
		},
		cli.IntFlag{
			Name:  "page",
			Usage: "page number",
			Value: 1,
		},
	},
}

func buildList(c *cli.Context) error {
	repo := c.Args().First()
	owner, name, err := internal.ParseRepo(repo)
	if err != nil {
		return err
	}

	client, err := internal.NewClient(c)
	if err != nil {
		return err
	}

	builds, err := client.BuildList(owner, name, &drone.ListOptions{Page: c.Int("page")})
	if err != nil {
		return err
	}

	tmpl, err := template.New("_").Parse(c.String("format") + "\n")
	if err != nil {
		return err
	}

	branch := c.String("branch")
	event := c.String("event")
	status := c.String("status")
	limit := c.Int("limit")

	var count int
	for _, build := range builds {
		if count >= limit {
			break
		}
		if branch != "" && build.Target != branch {
			continue
		}
		if event != "" && build.Event != event {
			continue
		}
		if status != "" && build.Status != status {
			continue
		}
		tmpl.Execute(os.Stdout, build)
		count++
	}
	return nil
}

// template for build list information
var tmplBuildList = "\x1b[33mBuild #{{ .Number }} \x1b[0m" + `
Status: {{ .Status }}
Event: {{ .Event }}
Commit: {{ .After }}
Branch: {{ .Target }}
Ref: {{ .Ref }}
Author: {{ .Author }} {{ if .AuthorEmail }}<{{.AuthorEmail}}>{{ end }}
Message: {{ .Message }}
`
<commit_msg>Removed pointer when passing ListOptions parameter<commit_after>package build

import (
	"os"
	"text/template"

	"github.com/drone/drone-cli/drone/internal"
	"github.com/drone/drone-go/drone"
	"github.com/urfave/cli"
)

var buildListCmd = cli.Command{
	Name:      "ls",
	Usage:     "show build history",
	ArgsUsage: "<repo/name>",
	Action:    buildList,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "format",
			Usage: "format output",
			Value: tmplBuildList,
		},
		cli.StringFlag{
			Name:  "branch",
			Usage: "branch filter",
		},
		cli.StringFlag{
			Name:  "event",
			Usage: "event filter",
		},
		cli.StringFlag{
			Name:  "status",
			Usage: "status filter",
		},
		cli.IntFlag{
			Name:  "limit",
			Usage: "limit the list size",
			Value: 25,
		},
		cli.IntFlag{
			Name:  "page",
			Usage: "page number",
			Value: 1,
		},
	},
}

func buildList(c *cli.Context) error {
	repo := c.Args().First()
	owner, name, err := internal.ParseRepo(repo)
	if err != nil {
		return err
	}

	client, err := internal.NewClient(c)
	if err != nil {
		return err
	}

	builds, err := client.BuildList(owner, name, drone.ListOptions{Page: c.Int("page")})
	if err != nil {
		return err
	}

	tmpl, err := template.New("_").Parse(c.String("format") + "\n")
	if err != nil {
		return err
	}

	branch := c.String("branch")
	event := c.String("event")
	status := c.String("status")
	limit := c.Int("limit")

	var count int
	for _, build := range builds {
		if count >= limit {
			break
		}
		if branch != "" && build.Target != branch {
			continue
		}
		if event != "" && build.Event != event {
			continue
		}
		if status != "" && build.Status != status {
			continue
		}
		tmpl.Execute(os.Stdout, build)
		count++
	}
	return nil
}

// template for build list information
var tmplBuildList = "\x1b[33mBuild #{{ .Number }} \x1b[0m" + `
Status: {{ .Status }}
Event: {{ .Event }}
Commit: {{ .After }}
Branch: {{ .Target }}
Ref: {{ .Ref }}
Author: {{ .Author }} {{ if .AuthorEmail }}<{{.AuthorEmail}}>{{ end }}
Message: {{ .Message }}
`
<|endoftext|>


<commit_before>package prometheus

import (
	"fmt"
	"runtime"
	"runtime/debug"
	"time"
)

type goCollector struct {
	goroutines Gauge
	gcDesc     *Desc

	// metrics to describe and collect
	metrics memStatsMetrics
}

// NewGoCollector returns a collector which exports metrics about the current
// go process.
func NewGoCollector() Collector {
	return &goCollector{
		goroutines: NewGauge(GaugeOpts{
			Namespace: "go",
			Name:      "goroutines",
			Help:      "Number of goroutines that currently exist.",
		}),
		gcDesc: NewDesc(
			"go_gc_duration_seconds",
			"A summary of the GC invocation durations.",
			nil, nil),
		metrics: memStatsMetrics{
			{
				desc: NewDesc(
					memstatNamespace("alloc_bytes"),
					"Number of bytes allocated and still in use.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.Alloc) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("alloc_bytes_total"),
					"Total number of bytes allocated, even if freed.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.TotalAlloc) },
				valType: CounterValue,
			}, {
				desc: NewDesc(
					memstatNamespace("sys_bytes"),
					"Number of bytes obtained by system. Sum of all system allocations.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.Sys) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("lookups_total"),
					"Total number of pointer lookups.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.Lookups) },
				valType: CounterValue,
			}, {
				desc: NewDesc(
					memstatNamespace("mallocs_total"),
					"Total number of mallocs.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.Mallocs) },
				valType: CounterValue,
			}, {
				desc: NewDesc(
					memstatNamespace("frees_total"),
					"Total number of frees.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.Frees) },
				valType: CounterValue,
			}, {
				desc: NewDesc(
					memstatNamespace("heap_alloc_bytes"),
					"Number of heap bytes allocated and still in use.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.HeapAlloc) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("heap_sys_bytes"),
					"Number of heap bytes obtained from system.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.HeapSys) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("heap_idle_bytes"),
					"Number of heap bytes waiting to be used.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.HeapIdle) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("heap_inuse_bytes"),
					"Number of heap bytes that are in use.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.HeapInuse) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("heap_released_bytes_total"),
					"Total number of heap bytes released to OS.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.HeapReleased) },
				valType: CounterValue,
			}, {
				desc: NewDesc(
					memstatNamespace("heap_objects"),
					"Number of allocated objects.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.HeapObjects) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("stack_inuse_bytes"),
					"Number of bytes in use by the stack allocator.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.StackInuse) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("stack_sys_bytes"),
					"Number of bytes obtained from system for stack allocator.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.StackSys) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("mspan_inuse_bytes"),
					"Number of bytes in use by mspan structures.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.MSpanInuse) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("mspan_sys_bytes"),
					"Number of bytes used for mspan structures obtained from system.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.MSpanSys) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("mcache_inuse_bytes"),
					"Number of bytes in use by mcache structures.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.MCacheInuse) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("mcache_sys_bytes"),
					"Number of bytes used for mcache structures obtained from system.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.MCacheSys) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("buck_hash_sys_bytes"),
					"Number of bytes used by the profiling bucket hash table.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.BuckHashSys) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("gc_sys_bytes"),
					"Number of bytes used for garbage collection system metadata.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.GCSys) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("other_sys_bytes"),
					"Number of bytes used for other system allocations.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.OtherSys) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("next_gc_bytes"),
					"Number of heap bytes when next garbage collection will take place.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.NextGC) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("last_gc_time_seconds"),
					"Number of seconds since 1970 of last garbage collection.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.LastGC) / 1e9 },
				valType: GaugeValue,
			},
		},
	}
}

func memstatNamespace(s string) string {
	return fmt.Sprintf("go_memstats_%s", s)
}

// Describe returns all descriptions of the collector.
func (c *goCollector) Describe(ch chan<- *Desc) {
	ch <- c.goroutines.Desc()
	ch <- c.gcDesc

	for _, i := range c.metrics {
		ch <- i.desc
	}
}

// Collect returns the current state of all metrics of the collector.
func (c *goCollector) Collect(ch chan<- Metric) {
	c.goroutines.Set(float64(runtime.NumGoroutine()))
	ch <- c.goroutines

	var stats debug.GCStats
	stats.PauseQuantiles = make([]time.Duration, 5)
	debug.ReadGCStats(&stats)

	quantiles := make(map[float64]float64)
	for idx, pq := range stats.PauseQuantiles[1:] {
		quantiles[float64(idx+1)/float64(len(stats.PauseQuantiles)-1)] = pq.Seconds()
	}
	quantiles[0.0] = stats.PauseQuantiles[0].Seconds()
	ch <- MustNewConstSummary(c.gcDesc, uint64(stats.NumGC), float64(stats.PauseTotal.Seconds()), quantiles)

	ms := &runtime.MemStats{}
	runtime.ReadMemStats(ms)
	for _, i := range c.metrics {
		ch <- MustNewConstMetric(i.desc, i.valType, i.eval(ms))
	}
}

// memStatsMetrics provide description, value, and value type for memstat metrics.
type memStatsMetrics []struct {
	desc    *Desc
	eval    func(*runtime.MemStats) float64
	valType ValueType
}
<commit_msg>Make heap_released_bytes a Gauge.<commit_after>package prometheus

import (
	"fmt"
	"runtime"
	"runtime/debug"
	"time"
)

type goCollector struct {
	goroutines Gauge
	gcDesc     *Desc

	// metrics to describe and collect
	metrics memStatsMetrics
}

// NewGoCollector returns a collector which exports metrics about the current
// go process.
func NewGoCollector() Collector {
	return &goCollector{
		goroutines: NewGauge(GaugeOpts{
			Namespace: "go",
			Name:      "goroutines",
			Help:      "Number of goroutines that currently exist.",
		}),
		gcDesc: NewDesc(
			"go_gc_duration_seconds",
			"A summary of the GC invocation durations.",
			nil, nil),
		metrics: memStatsMetrics{
			{
				desc: NewDesc(
					memstatNamespace("alloc_bytes"),
					"Number of bytes allocated and still in use.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.Alloc) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("alloc_bytes_total"),
					"Total number of bytes allocated, even if freed.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.TotalAlloc) },
				valType: CounterValue,
			}, {
				desc: NewDesc(
					memstatNamespace("sys_bytes"),
					"Number of bytes obtained from system.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.Sys) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("lookups_total"),
					"Total number of pointer lookups.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.Lookups) },
				valType: CounterValue,
			}, {
				desc: NewDesc(
					memstatNamespace("mallocs_total"),
					"Total number of mallocs.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.Mallocs) },
				valType: CounterValue,
			}, {
				desc: NewDesc(
					memstatNamespace("frees_total"),
					"Total number of frees.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.Frees) },
				valType: CounterValue,
			}, {
				desc: NewDesc(
					memstatNamespace("heap_alloc_bytes"),
					"Number of heap bytes allocated and still in use.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.HeapAlloc) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("heap_sys_bytes"),
					"Number of heap bytes obtained from system.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.HeapSys) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("heap_idle_bytes"),
					"Number of heap bytes waiting to be used.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.HeapIdle) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("heap_inuse_bytes"),
					"Number of heap bytes that are in use.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.HeapInuse) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("heap_released_bytes"),
					"Number of heap bytes released to OS.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.HeapReleased) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("heap_objects"),
					"Number of allocated objects.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.HeapObjects) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("stack_inuse_bytes"),
					"Number of bytes in use by the stack allocator.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.StackInuse) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("stack_sys_bytes"),
					"Number of bytes obtained from system for stack allocator.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.StackSys) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("mspan_inuse_bytes"),
					"Number of bytes in use by mspan structures.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.MSpanInuse) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("mspan_sys_bytes"),
					"Number of bytes used for mspan structures obtained from system.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.MSpanSys) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("mcache_inuse_bytes"),
					"Number of bytes in use by mcache structures.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.MCacheInuse) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("mcache_sys_bytes"),
					"Number of bytes used for mcache structures obtained from system.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.MCacheSys) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("buck_hash_sys_bytes"),
					"Number of bytes used by the profiling bucket hash table.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.BuckHashSys) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("gc_sys_bytes"),
					"Number of bytes used for garbage collection system metadata.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.GCSys) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("other_sys_bytes"),
					"Number of bytes used for other system allocations.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.OtherSys) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("next_gc_bytes"),
					"Number of heap bytes when next garbage collection will take place.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.NextGC) },
				valType: GaugeValue,
			}, {
				desc: NewDesc(
					memstatNamespace("last_gc_time_seconds"),
					"Number of seconds since 1970 of last garbage collection.",
					nil, nil,
				),
				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.LastGC) / 1e9 },
				valType: GaugeValue,
			},
		},
	}
}

func memstatNamespace(s string) string {
	return fmt.Sprintf("go_memstats_%s", s)
}

// Describe returns all descriptions of the collector.
func (c *goCollector) Describe(ch chan<- *Desc) {
	ch <- c.goroutines.Desc()
	ch <- c.gcDesc

	for _, i := range c.metrics {
		ch <- i.desc
	}
}

// Collect returns the current state of all metrics of the collector.
func (c *goCollector) Collect(ch chan<- Metric) {
	c.goroutines.Set(float64(runtime.NumGoroutine()))
	ch <- c.goroutines

	var stats debug.GCStats
	stats.PauseQuantiles = make([]time.Duration, 5)
	debug.ReadGCStats(&stats)

	quantiles := make(map[float64]float64)
	for idx, pq := range stats.PauseQuantiles[1:] {
		quantiles[float64(idx+1)/float64(len(stats.PauseQuantiles)-1)] = pq.Seconds()
	}
	quantiles[0.0] = stats.PauseQuantiles[0].Seconds()
	ch <- MustNewConstSummary(c.gcDesc, uint64(stats.NumGC), float64(stats.PauseTotal.Seconds()), quantiles)

	ms := &runtime.MemStats{}
	runtime.ReadMemStats(ms)
	for _, i := range c.metrics {
		ch <- MustNewConstMetric(i.desc, i.valType, i.eval(ms))
	}
}

// memStatsMetrics provide description, value, and value type for memstat metrics.
type memStatsMetrics []struct {
	desc    *Desc
	eval    func(*runtime.MemStats) float64
	valType ValueType
}
<|endoftext|>


<commit_before>package fuze

import (
	"encoding/json"
	"strconv"
  "errors"
  "fmt"

	fuze "github.com/coreos/container-linux-config-transpiler/config"
	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/schema"
)

func dataSourceFuzeConfig() *schema.Resource {
	return &schema.Resource{
		Read: dataSourceFuzeConfigRead,

		Schema: map[string]*schema.Schema{
			"content": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
			"pretty_print": &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				Default:  false,
			},
			"rendered": &schema.Schema{
				Type:        schema.TypeString,
				Computed:    true,
				Description: "rendered ignition configuration",
			},
		},
	}
}

func dataSourceFuzeConfigRead(d *schema.ResourceData, meta interface{}) error {
	rendered, err := renderFuzeConfig(d)
	if err != nil {
		return err
	}

	d.Set("rendered", rendered)
	d.SetId(strconv.Itoa(hashcode.String(rendered)))
	return nil
}

func renderFuzeConfig(d *schema.ResourceData) (string, error) {
	pretty := d.Get("pretty_print").(bool)
	config := d.Get("content").(string)

	ignition, pR := fuze.Parse([]byte(config))
	if len(pR.Entries) > 0 {
		return "", errors.New(pR.String())
	}

	if pretty {
		ignitionJSON, pErr := json.MarshalIndent(&ignition, "", "  ")
		return string(ignitionJSON), pErr
	}

  fmt.Println(ignition)
  converted, cR := fuze.ConvertAs2_0_0(ignition)
	if len(cR.Entries) > 0 {
		return "", errors.New(cR.String())
	}
  fmt.Println(converted)

	ignitionJSON, mErr := json.Marshal(&converted)
	return string(ignitionJSON), mErr
}

<commit_msg>tweaks.<commit_after>package fuze

import (
	"encoding/json"
	"strconv"
  "errors"
  "fmt"

	fuze "github.com/coreos/container-linux-config-transpiler/config"
	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/schema"
)

func dataSourceFuzeConfig() *schema.Resource {
	return &schema.Resource{
		Read: dataSourceFuzeConfigRead,

		Schema: map[string]*schema.Schema{
			"content": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
			"pretty_print": &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				Default:  false,
			},
			"rendered": &schema.Schema{
				Type:        schema.TypeString,
				Computed:    true,
				Description: "rendered ignition configuration",
			},
		},
	}
}

func dataSourceFuzeConfigRead(d *schema.ResourceData, meta interface{}) error {
	rendered, err := renderFuzeConfig(d)
	if err != nil {
		return err
	}

	d.Set("rendered", rendered)
	d.SetId(strconv.Itoa(hashcode.String(rendered)))
	return nil
}

func renderFuzeConfig(d *schema.ResourceData) (string, error) {
	pretty := d.Get("pretty_print").(bool)
	config := d.Get("content").(string)

	ignition, pR := fuze.Parse([]byte(config))
	if len(pR.Entries) > 0 {
		return "", errors.New(pR.String())
	}

	if pretty {
		ignitionJSON, pErr := json.MarshalIndent(&ignition, "", "  ")
		return string(ignitionJSON), pErr
	}

  converted, cR := fuze.ConvertAs2_0_0(ignition)
	if len(cR.Entries) > 0 {
		return "", errors.New(cR.String())
	}

	ignitionJSON, mErr := json.Marshal(&converted)
	return string(ignitionJSON), mErr
}

<|endoftext|>


<commit_before>// +build linux,!noaio

// The above build tag specifies this file is only to be built on linux (because
// goaio only supports linux). If you are having difficulty building on linux
// or want to build without librados present, then use
//    go build -tags 'noaio'

package nbd

import (
	"github.com/traetox/goaio"
	"golang.org/x/net/context"
	"os"
)

// AioFileBackend implements Backend
type AioFileBackend struct {
	aio  *goaio.AIO
	size uint64
}

// WriteAt implements Backend.WriteAt
func (afb *AioFileBackend) WriteAt(ctx context.Context, b []byte, offset int64, fua bool) (int, error) {
	if err := afb.aio.Wait(); err != nil {
		return 0, err
	}
	requestId, err := afb.aio.WriteAt(b, offset)
	if err != nil {
		return 0, err
	}
	if err := afb.aio.WaitFor(requestId); err != nil {
		return 0, err
	}
	if err := afb.aio.Ack(requestId); err != nil {
		return 0, err
	}
	if fua {
		if err := afb.aio.Flush(); err != nil {
			return 0, err
		}
	}
	return len(b), err
}

// ReadAt implements Backend.ReadAt
func (afb *AioFileBackend) ReadAt(ctx context.Context, b []byte, offset int64) (int, error) {
	if err := afb.aio.Wait(); err != nil {
		return 0, err
	}
	requestId, err := afb.aio.ReadAt(b, offset)
	if err != nil {
		return 0, err
	}
	if err := afb.aio.WaitFor(requestId); err != nil {
		return 0, err
	}
	if err := afb.aio.Ack(requestId); err != nil {
		return 0, err
	}
	return len(b), err
}

// TrimAt implements Backend.TrimAt
func (afb *AioFileBackend) TrimAt(ctx context.Context, length int, offset int64) (int, error) {
	return length, nil
}

// Flush implements Backend.Flush
func (afb *AioFileBackend) Flush(ctx context.Context) error {
	return afb.aio.Flush()
}

// Close implements Backend.Close
func (afb *AioFileBackend) Close(ctx context.Context) error {
	return afb.aio.Close()
}

// Size implements Backend.Size
func (afb *AioFileBackend) Geometry(ctx context.Context) (uint64, uint64, uint64, uint64, error) {
	return afb.size, 1, 4096, 128 * 1024 * 1024, nil
}

// Size implements Backend.HasFua
func (afb *AioFileBackend) HasFua(ctx context.Context) bool {
	return false
}

// Size implements Backend.HasFua
func (afb *AioFileBackend) HasFlush(ctx context.Context) bool {
	return false
}

// Generate a new aio backend
func NewAioFileBackend(ctx context.Context, ec *ExportConfig) (Backend, error) {
	perms := os.O_RDWR
	if ec.ReadOnly {
		perms = os.O_RDONLY
	}
	if s, err := isTrue(ec.DriverParameters["sync"]); err != nil {
		return nil, err
	} else if s {
		perms |= os.O_SYNC
	}
	aio, err := goaio.NewAIO(ec.DriverParameters["path"], perms, 0666)
	if err != nil {
		return nil, err
	}
	stat, err := aio.FD().Stat()
	if err != nil {
		aio.Close()
		return nil, err
	}
	return &AioFileBackend{
		aio:  aio,
		size: uint64(stat.Size()),
	}, nil
}

// Register our backend
func init() {
	RegisterBackend("aiofile", NewAioFileBackend)
}
<commit_msg>removed the need for Ack.  WaitFor returns an error if one ocurred<commit_after>// +build linux,!noaio

// The above build tag specifies this file is only to be built on linux (because
// goaio only supports linux). If you are having difficulty building on linux
// or want to build without librados present, then use
//    go build -tags 'noaio'

package nbd

import (
	"github.com/traetox/goaio"
	"golang.org/x/net/context"
	"os"
)

// AioFileBackend implements Backend
type AioFileBackend struct {
	aio  *goaio.AIO
	size uint64
}

// WriteAt implements Backend.WriteAt
func (afb *AioFileBackend) WriteAt(ctx context.Context, b []byte, offset int64, fua bool) (int, error) {
	if err := afb.aio.Wait(); err != nil {
		return 0, err
	}
	requestId, err := afb.aio.WriteAt(b, offset)
	if err != nil {
		return 0, err
	}
	if err := afb.aio.WaitFor(requestId); err != nil {
		return 0, err
	}
	if fua {
		if err := afb.aio.Flush(); err != nil {
			return 0, err
		}
	}
	return len(b), err
}

// ReadAt implements Backend.ReadAt
func (afb *AioFileBackend) ReadAt(ctx context.Context, b []byte, offset int64) (int, error) {
	if err := afb.aio.Wait(); err != nil {
		return 0, err
	}
	requestId, err := afb.aio.ReadAt(b, offset)
	if err != nil {
		return 0, err
	}
	if err := afb.aio.WaitFor(requestId); err != nil {
		return 0, err
	}
	return len(b), err
}

// TrimAt implements Backend.TrimAt
func (afb *AioFileBackend) TrimAt(ctx context.Context, length int, offset int64) (int, error) {
	return length, nil
}

// Flush implements Backend.Flush
func (afb *AioFileBackend) Flush(ctx context.Context) error {
	return afb.aio.Flush()
}

// Close implements Backend.Close
func (afb *AioFileBackend) Close(ctx context.Context) error {
	return afb.aio.Close()
}

// Size implements Backend.Size
func (afb *AioFileBackend) Geometry(ctx context.Context) (uint64, uint64, uint64, uint64, error) {
	return afb.size, 1, 4096, 128 * 1024 * 1024, nil
}

// Size implements Backend.HasFua
func (afb *AioFileBackend) HasFua(ctx context.Context) bool {
	return false
}

// Size implements Backend.HasFua
func (afb *AioFileBackend) HasFlush(ctx context.Context) bool {
	return false
}

// Generate a new aio backend
func NewAioFileBackend(ctx context.Context, ec *ExportConfig) (Backend, error) {
	perms := os.O_RDWR
	if ec.ReadOnly {
		perms = os.O_RDONLY
	}
	if s, err := isTrue(ec.DriverParameters["sync"]); err != nil {
		return nil, err
	} else if s {
		perms |= os.O_SYNC
	}
	aio, err := goaio.NewAIO(ec.DriverParameters["path"], perms, 0666)
	if err != nil {
		return nil, err
	}
	stat, err := aio.FD().Stat()
	if err != nil {
		aio.Close()
		return nil, err
	}
	return &AioFileBackend{
		aio:  aio,
		size: uint64(stat.Size()),
	}, nil
}

// Register our backend
func init() {
	RegisterBackend("aiofile", NewAioFileBackend)
}
<|endoftext|>


<commit_before>// Package handlers is used by go generate for analizing
// controller package's files and generation of handlers.
package handlers

import (
	"go/ast"
	"path/filepath"

	"github.com/anonx/sunplate/command"
	"github.com/anonx/sunplate/generation/output"
	"github.com/anonx/sunplate/reflect"
)

const (
	// ActionInterfaceImport is a GOPATH to the Result interface
	// that should be implemented by types returned by actions.
	ActionInterfaceImport = "github.com/anonx/sunplate/action"

	// ActionInterfaceName is an interface that should be implemented
	// by types that are returned from actions.
	ActionInterfaceName = "Result"

	// MagicMethodBefore is a name of the magic method that will be executed
	// before every action.
	MagicMethodBefore = "Before"

	// MagicMethodAfter is a name of the magic method that will be executed
	// after every action.
	MagicMethodAfter = "After"

	// MagicMethodFinally is a name of the magic method that will be executed
	// after every action no matter what.
	MagicMethodFinally = "Finally"
)

// Controller is a type that represents application controller,
// a structure that has actions.
type Controller struct {
	Actions reflect.Funcs  // Actions are methods that implement action.Result interface.
	After   reflect.Funcs  // Magic methods that are executed after actions if they return nil.
	Before  reflect.Funcs  // Magic methods that are executed before every action.
	Finally reflect.Funcs  // Finally is executed at the end of every request no matter what.
	Struct  reflect.Struct // Structure of the controller (its name, fields, etc).
}

// Start is an entry point of the generate handlers command.
func Start(basePath string, params command.Data) {
	// Generate and save a new package.
	t := output.NewType(
		params.Default("--package", "handlers"), filepath.Join(basePath, "./handlers.go.template"),
	)
	t.CreateDir(params.Default("--output", "./assets/handlers/"))
	t.Extension = ".go" // Save generated file as a .go source.
	t.Context = map[string]interface{}{
		"rootPath": params.Default("--path", "./controllers/"),
	}
	t.Generate()
}

// extractControllers gets a reflect.Package type and returns
// a slice of controllers that are found there.
func extractControllers(pkg *reflect.Package) (cs []Controller) {
	// Actions are required to return action.Result as the first argument.
	// actionImportName is used to store information on how the action package is named.
	// For illustration, here is an example:
	//	import (
	//		qwerty "github.com/anonx/sunplate/action"
	//	)
	// In the example above action package will be imported as "qwerty".
	// So, we are saving this name to actionImportName[FILE_NAME_WHERE_WE_IMPORT_THIS]
	// to eliminate the need of iterating over all imports over and over again.
	actionImportName := map[string]string{}

	// Files that should be excluded from search of actions
	// as they do not have action package imported.
	// We are using this as a cache.
	ignoreFiles := map[string]bool{}

	// isAction gets a pointer to reflect.Func structure
	// and checks whether it represents action or one of magic methods.
	// If so, it return true.
	isAction := func(f *reflect.Func) bool {
		// Check whether the file where this method located
		// is ignored due to the lack of action subpackage import.
		if ignoreFiles[f.File] {
			return false
		}

		// Check whether the method returns at least one parameter.
		if len(f.Results) == 0 {
			return false
		}

		// Make sure the method we are checking is Exported.
		// Private ones are ignored.
		if !ast.IsExported(f.Name) {
			return false
		}

		// Check whether we already know from previous iterations
		// how action subpackage is imported (its name).
		if _, ok := actionImportName[f.File]; !ok {
			// If not, try to find it out.
			n, ok := pkg.Imports.Name(f.File, ActionInterfaceImport)
			if !ok {
				// Action subpackage import path is not found in this file.
				// So, this is not an action method.
				// Ignore this file (and all methods inside it) in future.
				ignoreFiles[f.File] = true
				return false
			}
			actionImportName[f.File] = n // Save the import name to use in future iterations.
		}

		// Make sure the first result is of type action.Result.
		correctPackage := f.Results[0].Type.Package == actionImportName[f.File]
		correctName := f.Results[0].Type.Name == ActionInterfaceName
		if !correctPackage || !correctName {
			return false
		}
		return true
	}

	// isBefore gets a Func and checks whether it is a Before magic method.
	isBefore := func(f *reflect.Func) bool {
		if f.Name == MagicMethodBefore {
			return true
		}
		return false
	}

	// isAfter gets a Func and checks whether it is an After magic method.
	isAfter := func(f *reflect.Func) bool {
		if f.Name == MagicMethodAfter {
			return true
		}
		return false
	}

	// isFinally gets a Func and checks whether it is a Finally magic method.
	isFinally := func(f *reflect.Func) bool {
		if f.Name == MagicMethodFinally {
			return true
		}
		return false
	}

	// notMagicMethod gets a Func and makes sure it is not a magic method but a usual action.
	notMagicMethod := func(f *reflect.Func) bool {
		if isBefore(f) || isAfter(f) || isFinally(f) {
			return false
		}
		return true
	}

	// Iterating through all available structures and checking
	// whether those structures are controllers (i.e. whether they have actions).
	for i := 0; i < len(pkg.Structs); i++ {
		// Make sure the structure has methods.
		ms, ok := pkg.Methods[pkg.Structs[i].Name]
		if !ok {
			continue
		}

		// Check whether there are actions among those methods.
		as := ms.Filter(isAction, notMagicMethod, isAfter, isBefore, isFinally)
		if len(as) == 0 {
			continue
		}

		// Add a new controller to the list of results.
		cs = append(cs, Controller{
			Actions: as[0],
			After:   as[1],
			Before:  as[2],
			Finally: as[3],
			Struct:  pkg.Structs[i],
		})
	}
	return
}
<commit_msg>Get rid of nested functions<commit_after>// Package handlers is used by go generate for analizing
// controller package's files and generation of handlers.
package handlers

import (
	"go/ast"
	"path/filepath"

	"github.com/anonx/sunplate/command"
	"github.com/anonx/sunplate/generation/output"
	"github.com/anonx/sunplate/reflect"
)

const (
	// ActionInterfaceImport is a GOPATH to the Result interface
	// that should be implemented by types returned by actions.
	ActionInterfaceImport = "github.com/anonx/sunplate/action"

	// ActionInterfaceName is an interface that should be implemented
	// by types that are returned from actions.
	ActionInterfaceName = "Result"

	// MagicMethodBefore is a name of the magic method that will be executed
	// before every action.
	MagicMethodBefore = "Before"

	// MagicMethodAfter is a name of the magic method that will be executed
	// after every action.
	MagicMethodAfter = "After"

	// MagicMethodFinally is a name of the magic method that will be executed
	// after every action no matter what.
	MagicMethodFinally = "Finally"
)

// Controller is a type that represents application controller,
// a structure that has actions.
type Controller struct {
	Actions reflect.Funcs  // Actions are methods that implement action.Result interface.
	After   reflect.Funcs  // Magic methods that are executed after actions if they return nil.
	Before  reflect.Funcs  // Magic methods that are executed before every action.
	Finally reflect.Funcs  // Finally is executed at the end of every request no matter what.
	Struct  reflect.Struct // Structure of the controller (its name, fields, etc).
}

// Start is an entry point of the generate handlers command.
func Start(basePath string, params command.Data) {
	// Generate and save a new package.
	t := output.NewType(
		params.Default("--package", "handlers"), filepath.Join(basePath, "./handlers.go.template"),
	)
	t.CreateDir(params.Default("--output", "./assets/handlers/"))
	t.Extension = ".go" // Save generated file as a .go source.
	t.Context = map[string]interface{}{
		"rootPath": params.Default("--path", "./controllers/"),
	}
	t.Generate()
}

// extractControllers gets a reflect.Package type and returns
// a slice of controllers that are found there.
func extractControllers(pkg *reflect.Package) (cs []Controller) {
	// Actions are required to return action.Result as the first argument.
	// actionImportName is used to store information on how the action package is named.
	// For illustration, here is an example:
	//	import (
	//		qwerty "github.com/anonx/sunplate/action"
	//	)
	// In the example above action package will be imported as "qwerty".
	// So, we are saving this name to actionImportName[FILE_NAME_WHERE_WE_IMPORT_THIS]
	// to eliminate the need of iterating over all imports over and over again.
	actionImportName := map[string]string{}

	// Files that should be excluded from search of actions
	// as they do not have action package imported.
	// We are using this as a cache.
	ignoreFiles := map[string]bool{}

	// isAction gets a pointer to reflect.Func structure
	// and checks whether it represents action or one of magic methods.
	// If so, it return true.
	isAction := func(f *reflect.Func) bool {
		// Check whether the file where this method located
		// is ignored due to the lack of action subpackage import.
		if ignoreFiles[f.File] {
			return false
		}

		// Check whether the method returns at least one parameter.
		if len(f.Results) == 0 {
			return false
		}

		// Make sure the method we are checking is Exported.
		// Private ones are ignored.
		if !ast.IsExported(f.Name) {
			return false
		}

		// Check whether we already know from previous iterations
		// how action subpackage is imported (its name).
		if _, ok := actionImportName[f.File]; !ok {
			// If not, try to find it out.
			n, ok := pkg.Imports.Name(f.File, ActionInterfaceImport)
			if !ok {
				// Action subpackage import path is not found in this file.
				// So, this is not an action method.
				// Ignore this file (and all methods inside it) in future.
				ignoreFiles[f.File] = true
				return false
			}
			actionImportName[f.File] = n // Save the import name to use in future iterations.
		}

		// Make sure the first result is of type action.Result.
		correctPackage := f.Results[0].Type.Package == actionImportName[f.File]
		correctName := f.Results[0].Type.Name == ActionInterfaceName
		if !correctPackage || !correctName {
			return false
		}
		return true
	}

	// Iterating through all available structures and checking
	// whether those structures are controllers (i.e. whether they have actions).
	for i := 0; i < len(pkg.Structs); i++ {
		// Make sure the structure has methods.
		ms, ok := pkg.Methods[pkg.Structs[i].Name]
		if !ok {
			continue
		}

		// Check whether there are actions among those methods.
		as, count := ms.Filter(isAction, notMagicMethod, isAfter, isBefore, isFinally)
		if count == 0 {
			continue
		}

		// Add a new controller to the list of results.
		cs = append(cs, Controller{
			Actions: as[0],
			After:   as[1],
			Before:  as[2],
			Finally: as[3],
			Struct:  pkg.Structs[i],
		})
	}
	return
}

// isBefore gets a Func and checks whether it is a Before magic method.
func isBefore(f *reflect.Func) bool {
	if f.Name == MagicMethodBefore {
		return true
	}
	return false
}

// isAfter gets a Func and checks whether it is an After magic method.
func isAfter(f *reflect.Func) bool {
	if f.Name == MagicMethodAfter {
		return true
	}
	return false
}

// isFinally gets a Func and checks whether it is a Finally magic method.
func isFinally(f *reflect.Func) bool {
	if f.Name == MagicMethodFinally {
		return true
	}
	return false
}

// notMagicMethod gets a Func and makes sure it is not a magic method but a usual action.
func notMagicMethod(f *reflect.Func) bool {
	if isBefore(f) || isAfter(f) || isFinally(f) {
		return false
	}
	return true
}
<|endoftext|>


<commit_before>// Godless is a peer-to-peer database running over IPFS.
//
// Godless uses a Consistent Replicated Data Type called a Namespace to share schemaless data with peers.
//
// This package is a facade to Godless internals.
//
// Godless is in alpha, and should be considered experimental software.
package godless

import (
	"fmt"
	"sync"
	"time"

	gohttp "net/http"

	"github.com/johnny-morrice/godless/api"
	"github.com/johnny-morrice/godless/cache"
	"github.com/johnny-morrice/godless/crdt"
	"github.com/johnny-morrice/godless/crypto"
	"github.com/johnny-morrice/godless/internal/http"
	"github.com/johnny-morrice/godless/internal/ipfs"
	"github.com/johnny-morrice/godless/internal/service"
	"github.com/johnny-morrice/godless/log"
	"github.com/johnny-morrice/godless/query"
	"github.com/pkg/errors"
)

// Godless options.
type Options struct {
	// IpfsServiceUrl is required, unless specifying your own DataPeer or RemoteStore.
	IpfsServiceUrl string
	// DataPeer is optional.
	DataPeer api.DataPeer
	// RemoteStore is optional.
	RemoteStore api.RemoteStore
	// KeyStore is required. A private Key store.
	KeyStore api.KeyStore
	// MemoryImage is required.
	MemoryImage api.MemoryImage
	// WebServiceAddr is optional.  If not set, the webservice will be disabled.
	WebServiceAddr string
	// IndexHash is optional.  Set to load an existing index from IPFS.
	IndexHash string
	// FailEarly will cause the godless process to crash if it cannot contact IPFS on startup.
	FailEarly bool
	// ReplicateInterval is optional.  The duration between peer-to-peer replications.
	ReplicateInterval time.Duration
	Pulse             time.Duration
	// Topics is optional.  Two godless servers which share a topic will replicate indices. An empty topics slice will disable replication.
	Topics []string
	// IpfsClient is optional.  Specify a HTTP client for IPFS.
	IpfsClient *gohttp.Client
	// IpfsPingTimeout is optional.  Specify a lower timeout for "Am I Connected?" checks.
	IpfsPingTimeout time.Duration
	// Cache is optional. Build a 12-factor app by supplying your own remote cache.
	// HeadCache, IndexCache and NamespaceCache can be used to specify different caches for different data types.
	Cache api.Cache
	// HeadCache is optional.  Build a 12-factor app by supplying your own remote cache.
	HeadCache api.HeadCache
	// IndexCache is optional.  Build a 12-factor app by supplying your own remote cache.
	IndexCache api.IndexCache
	// NamespaceCache is optional. Build a 12-factor app by supplying your own remote cache.
	NamespaceCache api.NamespaceCache
	// PriorityQueue is optional. Build a 12-factor app by supplying your own remote cache.
	PriorityQueue api.RequestPriorityQueue
	// APIQueryLimit is optional.  Tune performance by setting the number of simultaneous queries.
	APIQueryLimit int
	// PublicServer is optional.  If false, the index will only be updated from peers who are in your public key list.
	PublicServer bool
}

// Godless is a peer-to-peer database.  It shares structured data between peers, using IPFS as a backing store.
// The core datastructure is a CRDT namespace which resembles a relational scheme in that it has tables, rows, and entries.
type Godless struct {
	Options
	errch    chan error
	errwg    sync.WaitGroup
	stopch   chan struct{}
	stoppers []chan<- struct{}
	remote   api.RemoteNamespace
	api      api.APIService
}

// New creates a godless instance, connecting to any services, and providing any services, specified in the options.
func New(options Options) (*Godless, error) {
	godless := &Godless{Options: options}

	missing := godless.findMissingParameters()

	if missing != nil {
		return nil, missing
	}

	setupFuncs := []func() error{
		godless.connectDataPeer,
		godless.connectRemoteStore,
		godless.connectCache,
		godless.setupNamespace,
		godless.launchAPI,
		godless.serveWeb,
		godless.replicate,
	}

	err := breakOnError(setupFuncs)

	if err != nil {
		return nil, err
	}

	godless.report()

	return godless, nil
}

func (godless *Godless) report() {
	if godless.PublicServer {
		log.Info("Running public Godless API")
	} else {
		log.Info("Running private Godless API")
	}

	privCount := len(godless.KeyStore.GetAllPrivateKeys())
	pubCount := len(godless.KeyStore.GetAllPublicKeys())

	log.Info("Godless API using %v private and %v public keys", privCount, pubCount)
}

func (godless *Godless) findMissingParameters() error {
	var missing error

	if godless.KeyStore == nil {
		msg := godless.missingParameterText("KeyStore")
		missing = addErrorMessage(missing, msg)
	}

	if godless.MemoryImage == nil {
		msg := godless.missingParameterText("MemoryImage")
		missing = addErrorMessage(missing, msg)
	}

	return missing
}

func addErrorMessage(err error, msg string) error {
	if err == nil {
		return errors.New(msg)
	} else {
		return errors.Wrap(err, msg)
	}
}

func (godless *Godless) missingParameterText(param string) string {
	return fmt.Sprintf("Missing required parameter '%v'", param)
}

// Errors provides a stream of errors from godless.  Godless will attempt to handle any errors it can.  Any errors received here indicate that bad things have happened.
func (godless *Godless) Errors() <-chan error {
	return godless.errch
}

// Shutdown stops all godless processes.  It does not wait for those goroutines to stop.
func (godless *Godless) Shutdown() {
	godless.stopch <- struct{}{}
}

func (godless *Godless) connectDataPeer() error {
	if godless.RemoteStore != nil {
		return nil
	}

	if godless.DataPeer == nil {
		if godless.IpfsServiceUrl == "" {
			msg := godless.missingParameterText("IpfsServiceUrl")
			return errors.New(msg)
		}

		peer := &ipfs.WebServiceClient{
			Url:         godless.IpfsServiceUrl,
			PingTimeout: godless.IpfsPingTimeout,
			Http:        godless.IpfsClient,
		}

		godless.DataPeer = peer
	}

	return nil
}

func (godless *Godless) connectRemoteStore() error {
	if godless.RemoteStore == nil {
		ipfs := &ipfs.IpfsRemoteStore{
			Shell: godless.DataPeer,
		}

		if godless.FailEarly {
			err := ipfs.Connect()

			if err != nil {
				return err
			}
		}

		godless.RemoteStore = ipfs
	}

	return nil
}

func (godless *Godless) makeCacheStopper(cache api.Cache) chan<- struct{} {
	stopper := make(chan struct{}, 1)
	go func() {
		<-stopper
		err := cache.CloseCache()
		if err == nil {
			log.Info("Closed cache")
		} else {
			log.Error("Error closing cache: %v", err.Error())
		}
	}()

	return stopper
}

func (godless *Godless) connectCache() error {
	if godless.Cache != nil {
		godless.HeadCache = godless.Cache
		godless.IndexCache = godless.Cache
		godless.NamespaceCache = godless.Cache
		godless.addStopper(godless.makeCacheStopper(godless.Cache))
		return nil
	}

	if godless.HeadCache == nil {
		godless.HeadCache = cache.MakeResidentHeadCache()
	}

	if godless.IndexCache == nil {
		godless.IndexCache = cache.MakeResidentIndexCache(__UNKNOWN_BUFFER_SIZE)
	}

	if godless.NamespaceCache == nil {
		godless.NamespaceCache = cache.MakeResidentNamespaceCache(__UNKNOWN_BUFFER_SIZE)
	}

	return nil
}

func (godless *Godless) setupNamespace() error {
	if godless.IndexHash != "" {
		head := crdt.IPFSPath(godless.IndexHash)

		err := godless.HeadCache.SetHead(head)

		if err != nil {
			return err
		}
	}

	namespaceOptions := service.RemoteNamespaceOptions{
		Pulse:          godless.Pulse,
		Store:          godless.RemoteStore,
		HeadCache:      godless.HeadCache,
		IndexCache:     godless.IndexCache,
		NamespaceCache: godless.NamespaceCache,
		KeyStore:       godless.KeyStore,
		IsPublicIndex:  godless.PublicServer,
		MemoryImage:    godless.MemoryImage,
	}

	godless.remote = service.MakeRemoteNamespace(namespaceOptions)
	return nil
}

func (godless *Godless) launchAPI() error {
	limit := godless.APIQueryLimit

	if limit == 0 {
		limit = 1
	}

	queue := godless.PriorityQueue

	if queue == nil {
		queue = cache.MakeResidentBufferQueue(__UNKNOWN_BUFFER_SIZE)
	}

	api, errch := service.LaunchKeyValueStore(godless.remote, queue, limit)

	godless.addErrors(errch)
	godless.api = api

	return nil
}

// Serve serves the Godless webservice.
func (godless *Godless) serveWeb() error {
	addr := godless.WebServiceAddr

	if addr == "" {
		return nil
	}

	webService := &service.WebService{API: godless.api}
	stopch, err := http.Serve(addr, webService.Handler())

	if err != nil {
		return err
	}

	godless.addStopper(stopch)
	return nil
}

// Replicate shares data via the IPFS pubsub mechanism.
func (godless *Godless) replicate() error {
	topics := godless.Topics
	interval := godless.ReplicateInterval

	if len(topics) == 0 {
		return nil
	}

	pubsubTopics := make([]api.PubSubTopic, len(topics))

	for i, t := range topics {
		pubsubTopics[i] = api.PubSubTopic(t)
	}

	options := service.ReplicateOptions{
		API:         godless.api,
		RemoteStore: godless.RemoteStore,
		Interval:    interval,
		Topics:      pubsubTopics,
		KeyStore:    godless.KeyStore,
	}
	stopch, errch := service.Replicate(options)
	godless.addStopper(stopch)
	godless.addErrors(errch)
	return nil
}

func (godless *Godless) addStopper(stopch chan<- struct{}) {
	if godless.stopch == nil {
		godless.stopch = make(chan struct{})
		go func() {
			godless.handleShutdown()
		}()
	}

	godless.stoppers = append(godless.stoppers, stopch)
}

func (godless *Godless) handleShutdown() {
	<-godless.stopch
	log.Info("Shutting down")
	for _, stopper := range godless.stoppers {
		go close(stopper)
	}

}

func (godless *Godless) addErrors(errch <-chan error) {
	godless.errwg.Add(1)

	if godless.errch == nil {
		godless.errch = make(chan error)
		go func() {
			godless.errwg.Wait()
			close(godless.errch)
		}()
	}

	go func() {
		for err := range errch {
			godless.errch <- err
		}
		godless.errwg.Done()
	}()
}

// Client is a Godless HTTP client.
type Client interface {
	SendQuery(*query.Query) (api.APIResponse, error)
	SendReflection(api.APIReflectionType) (api.APIResponse, error)
}

// MakeClient creates a Godless HTTP Client.
func MakeClient(serviceAddr string) Client {
	return service.MakeClient(serviceAddr)
}

func MakeClientWithHttp(serviceAddr string, webClient *gohttp.Client) Client {
	return service.MakeClientWithHttp(serviceAddr, webClient)
}

func MakeKeyStore() api.KeyStore {
	return &crypto.KeyStore{}
}

func breakOnError(pipeline []func() error) error {
	for _, f := range pipeline {
		err := f()

		if err != nil {
			return err
		}
	}

	return nil
}

// We don't know the right buffer size here, so let the cache package handle it.
const __UNKNOWN_BUFFER_SIZE = -1
<commit_msg>Call godless via root lib<commit_after>// Godless is a peer-to-peer database running over IPFS.
//
// Godless uses a Consistent Replicated Data Type called a Namespace to share schemaless data with peers.
//
// This package is a facade to Godless internals.
//
// Godless is in alpha, and should be considered experimental software.
package godless

import (
	"fmt"
	"sync"
	"time"

	gohttp "net/http"

	"github.com/johnny-morrice/godless/api"
	"github.com/johnny-morrice/godless/cache"
	"github.com/johnny-morrice/godless/crdt"
	"github.com/johnny-morrice/godless/crypto"
	"github.com/johnny-morrice/godless/internal/http"
	"github.com/johnny-morrice/godless/internal/ipfs"
	"github.com/johnny-morrice/godless/internal/service"
	"github.com/johnny-morrice/godless/log"
	"github.com/johnny-morrice/godless/query"
	"github.com/pkg/errors"
)

// Godless options.
type Options struct {
	// IpfsServiceUrl is required, unless specifying your own DataPeer or RemoteStore.
	IpfsServiceUrl string
	// DataPeer is optional.  If specified, none of the IPFS options will be used.
	DataPeer api.DataPeer
	// RemoteStore is optional.  If specified, the DataPeer will not be used, nor any of the IPFS options.
	RemoteStore api.RemoteStore
	// KeyStore is required. A private Key store.
	KeyStore api.KeyStore
	// MemoryImage is required.
	MemoryImage api.MemoryImage
	// WebServiceAddr is optional.  If not set, the webservice will be disabled.
	WebServiceAddr string
	// IndexHash is optional.  Set to load an existing index from IPFS.
	IndexHash string
	// FailEarly will cause the godless process to crash if it cannot contact IPFS on startup.
	FailEarly bool
	// ReplicateInterval is optional.  The duration between peer-to-peer replications.
	ReplicateInterval time.Duration
	// Pulse is optional.  The duration between flushes of the index to IPFS.
	Pulse time.Duration
	// Topics is optional.  Two godless servers which share a topic will replicate indices. An empty topics slice will disable replication.
	Topics []string
	// IpfsClient is optional.  Specify a HTTP client for IPFS.
	IpfsClient *gohttp.Client
	// IpfsPingTimeout is optional.  Specify a lower timeout for "Am I Connected?" checks.
	IpfsPingTimeout time.Duration
	// Cache is optional. Build a 12-factor app by supplying your own remote cache.
	// HeadCache, IndexCache and NamespaceCache can be used to specify different caches for different data types.
	Cache api.Cache
	// HeadCache is optional.  Build a 12-factor app by supplying your own remote cache.
	HeadCache api.HeadCache
	// IndexCache is optional.  Build a 12-factor app by supplying your own remote cache.
	IndexCache api.IndexCache
	// NamespaceCache is optional. Build a 12-factor app by supplying your own remote cache.
	NamespaceCache api.NamespaceCache
	// PriorityQueue is optional. Build a 12-factor app by supplying your own remote cache.
	PriorityQueue api.RequestPriorityQueue
	// APIQueryLimit is optional.  Tune performance by setting the number of simultaneous queries.
	APIQueryLimit int
	// PublicServer is optional.  If false, the index will only be updated from peers who are in your public key list.
	PublicServer bool
}

// Godless is a peer-to-peer database.  It shares structured data between peers, using IPFS as a backing store.
// The core datastructure is a CRDT namespace which resembles a relational scheme in that it has tables, rows, and entries.
type Godless struct {
	Options
	errch    chan error
	errwg    sync.WaitGroup
	stopch   chan struct{}
	stoppers []chan<- struct{}
	remote   api.RemoteNamespace
	api      api.APIService
}

// New creates a godless instance, connecting to any services, and providing any services, specified in the options.
func New(options Options) (*Godless, error) {
	godless := &Godless{Options: options}

	missing := godless.findMissingParameters()

	if missing != nil {
		return nil, missing
	}

	setupFuncs := []func() error{
		godless.connectDataPeer,
		godless.connectRemoteStore,
		godless.connectCache,
		godless.setupNamespace,
		godless.launchAPI,
		godless.serveWeb,
		godless.replicate,
	}

	err := breakOnError(setupFuncs)

	if err != nil {
		return nil, err
	}

	godless.report()

	return godless, nil
}

func (godless *Godless) Call(request api.APIRequest) (<-chan api.APIResponse, error) {
	return godless.api.Call(request)
}

func (godless *Godless) report() {
	if godless.PublicServer {
		log.Info("Running public Godless API")
	} else {
		log.Info("Running private Godless API")
	}

	privCount := len(godless.KeyStore.GetAllPrivateKeys())
	pubCount := len(godless.KeyStore.GetAllPublicKeys())

	log.Info("Godless API using %v private and %v public keys", privCount, pubCount)
}

func (godless *Godless) findMissingParameters() error {
	var missing error

	if godless.KeyStore == nil {
		msg := godless.missingParameterText("KeyStore")
		missing = addErrorMessage(missing, msg)
	}

	if godless.MemoryImage == nil {
		msg := godless.missingParameterText("MemoryImage")
		missing = addErrorMessage(missing, msg)
	}

	return missing
}

func addErrorMessage(err error, msg string) error {
	if err == nil {
		return errors.New(msg)
	} else {
		return errors.Wrap(err, msg)
	}
}

func (godless *Godless) missingParameterText(param string) string {
	return fmt.Sprintf("Missing required parameter '%v'", param)
}

// Errors provides a stream of errors from godless.  Godless will attempt to handle any errors it can.  Any errors received here indicate that bad things have happened.
func (godless *Godless) Errors() <-chan error {
	return godless.errch
}

// Shutdown stops all godless processes.  It does not wait for those goroutines to stop.
func (godless *Godless) Shutdown() {
	godless.stopch <- struct{}{}
}

func (godless *Godless) connectDataPeer() error {
	if godless.RemoteStore != nil {
		return nil
	}

	if godless.DataPeer == nil {
		if godless.IpfsServiceUrl == "" {
			msg := godless.missingParameterText("IpfsServiceUrl")
			return errors.New(msg)
		}

		peer := &ipfs.WebServiceClient{
			Url:         godless.IpfsServiceUrl,
			PingTimeout: godless.IpfsPingTimeout,
			Http:        godless.IpfsClient,
		}

		godless.DataPeer = peer
	}

	return nil
}

func (godless *Godless) connectRemoteStore() error {
	if godless.RemoteStore == nil {
		ipfs := &ipfs.IpfsRemoteStore{
			Shell: godless.DataPeer,
		}

		if godless.FailEarly {
			err := ipfs.Connect()

			if err != nil {
				return err
			}
		}

		godless.RemoteStore = ipfs
	}

	return nil
}

func (godless *Godless) makeCacheStopper(cache api.Cache) chan<- struct{} {
	stopper := make(chan struct{}, 1)
	go func() {
		<-stopper
		err := cache.CloseCache()
		if err == nil {
			log.Info("Closed cache")
		} else {
			log.Error("Error closing cache: %v", err.Error())
		}
	}()

	return stopper
}

func (godless *Godless) connectCache() error {
	if godless.Cache != nil {
		godless.HeadCache = godless.Cache
		godless.IndexCache = godless.Cache
		godless.NamespaceCache = godless.Cache
		godless.addStopper(godless.makeCacheStopper(godless.Cache))
		return nil
	}

	if godless.HeadCache == nil {
		godless.HeadCache = cache.MakeResidentHeadCache()
	}

	if godless.IndexCache == nil {
		godless.IndexCache = cache.MakeResidentIndexCache(__UNKNOWN_BUFFER_SIZE)
	}

	if godless.NamespaceCache == nil {
		godless.NamespaceCache = cache.MakeResidentNamespaceCache(__UNKNOWN_BUFFER_SIZE)
	}

	return nil
}

func (godless *Godless) setupNamespace() error {
	if godless.IndexHash != "" {
		head := crdt.IPFSPath(godless.IndexHash)

		err := godless.HeadCache.SetHead(head)

		if err != nil {
			return err
		}
	}

	namespaceOptions := service.RemoteNamespaceOptions{
		Pulse:          godless.Pulse,
		Store:          godless.RemoteStore,
		HeadCache:      godless.HeadCache,
		IndexCache:     godless.IndexCache,
		NamespaceCache: godless.NamespaceCache,
		KeyStore:       godless.KeyStore,
		IsPublicIndex:  godless.PublicServer,
		MemoryImage:    godless.MemoryImage,
	}

	godless.remote = service.MakeRemoteNamespace(namespaceOptions)
	return nil
}

func (godless *Godless) launchAPI() error {
	limit := godless.APIQueryLimit

	if limit == 0 {
		limit = 1
	}

	queue := godless.PriorityQueue

	if queue == nil {
		queue = cache.MakeResidentBufferQueue(__UNKNOWN_BUFFER_SIZE)
	}

	api, errch := service.LaunchKeyValueStore(godless.remote, queue, limit)

	godless.addErrors(errch)
	godless.api = api

	return nil
}

// Serve serves the Godless webservice.
func (godless *Godless) serveWeb() error {
	addr := godless.WebServiceAddr

	if addr == "" {
		return nil
	}

	webService := &service.WebService{API: godless.api}
	stopch, err := http.Serve(addr, webService.Handler())

	if err != nil {
		return err
	}

	godless.addStopper(stopch)
	return nil
}

// Replicate shares data via the IPFS pubsub mechanism.
func (godless *Godless) replicate() error {
	topics := godless.Topics
	interval := godless.ReplicateInterval

	if len(topics) == 0 {
		return nil
	}

	pubsubTopics := make([]api.PubSubTopic, len(topics))

	for i, t := range topics {
		pubsubTopics[i] = api.PubSubTopic(t)
	}

	options := service.ReplicateOptions{
		API:         godless.api,
		RemoteStore: godless.RemoteStore,
		Interval:    interval,
		Topics:      pubsubTopics,
		KeyStore:    godless.KeyStore,
	}
	stopch, errch := service.Replicate(options)
	godless.addStopper(stopch)
	godless.addErrors(errch)
	return nil
}

func (godless *Godless) addStopper(stopch chan<- struct{}) {
	if godless.stopch == nil {
		godless.stopch = make(chan struct{})
		go func() {
			godless.handleShutdown()
		}()
	}

	godless.stoppers = append(godless.stoppers, stopch)
}

func (godless *Godless) handleShutdown() {
	<-godless.stopch
	log.Info("Shutting down")
	for _, stopper := range godless.stoppers {
		go close(stopper)
	}

}

func (godless *Godless) addErrors(errch <-chan error) {
	godless.errwg.Add(1)

	if godless.errch == nil {
		godless.errch = make(chan error)
		go func() {
			godless.errwg.Wait()
			close(godless.errch)
		}()
	}

	go func() {
		for err := range errch {
			godless.errch <- err
		}
		godless.errwg.Done()
	}()
}

// Client is a Godless HTTP client.
type Client interface {
	SendQuery(*query.Query) (api.APIResponse, error)
	SendReflection(api.APIReflectionType) (api.APIResponse, error)
}

// MakeClient creates a Godless HTTP Client.
func MakeClient(serviceAddr string) Client {
	return service.MakeClient(serviceAddr)
}

func MakeClientWithHttp(serviceAddr string, webClient *gohttp.Client) Client {
	return service.MakeClientWithHttp(serviceAddr, webClient)
}

func MakeKeyStore() api.KeyStore {
	return &crypto.KeyStore{}
}

func breakOnError(pipeline []func() error) error {
	for _, f := range pipeline {
		err := f()

		if err != nil {
			return err
		}
	}

	return nil
}

// We don't know the right buffer size here, so let the cache package handle it.
const __UNKNOWN_BUFFER_SIZE = -1
<|endoftext|>


<commit_before>// Copyright 2020 Anapaya Systems
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package launcher includes the shared application execution boilerplate of all
// SCION servers.
package launcher

import (
	"fmt"
	"io"
	"os"
	"os/signal"
	"path/filepath"
	"syscall"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"

	"github.com/scionproto/scion/go/lib/config"
	libconfig "github.com/scionproto/scion/go/lib/config"
	"github.com/scionproto/scion/go/lib/env"
	"github.com/scionproto/scion/go/lib/fatal"
	"github.com/scionproto/scion/go/lib/log"
	"github.com/scionproto/scion/go/lib/metrics"
	"github.com/scionproto/scion/go/lib/prom"
	"github.com/scionproto/scion/go/lib/serrors"
	"github.com/scionproto/scion/go/pkg/command"
)

// Configuration keys used by the launcher
const (
	cfgLogConsoleLevel           = "log.console.level"
	cfgLogConsoleFormat          = "log.console.format"
	cfgLogConsoleStacktraceLevel = "log.console.stacktrace_level"
	cfgGeneralID                 = "general.id"
	cfgConfigFile                = "config"
)

// Application models a SCION server application.
type Application struct {
	// TOMLConfig holds the Go data structure for the application-specific
	// TOML configuration. The Application launcher will check if the TOMLConfig
	// supports additional methods (e.g., custom logging or instance ID) and
	// extract them from the config if that is the case. See the XxxConfig interfaces
	// in this package for more information.
	TOMLConfig libconfig.Config

	// Samplers contains additional configuration samplers to be included
	// under the sample subcommand. If empty, no additional samplers are
	// listed.
	//
	// DEPRECATED. This field will be removed once Anapaya/scion#5000 is implemented.
	Samplers []func(command.Pather) *cobra.Command

	// ShortName is the short name of the application. If empty, the executable name is used.
	// The ShortName could be, for example, "SCION Daemon" for the SCION Daemon.
	ShortName string

	// Main is the custom logic of the application. If nil, no custom logic is executed
	// (and only the setup/teardown harness runs). If Main returns an error, the
	// Run method will return a non-zero exit code.
	Main func() error

	// ErrorWriter specifies where error output should be printed. If nil, os.Stderr is used.
	ErrorWriter io.Writer

	// config contains the Viper configuration KV store.
	config *viper.Viper
}

// Run sets up the common SCION server harness, and then passes control to the Main
// function (if one exists).
//
// Run uses the following globals:
//   os.Args
//
// Run will exit the application if it encounters a fatal error.
func (a *Application) Run() {
	if err := a.run(); err != nil {
		fmt.Fprintf(a.getErrorWriter(), "fatal error: %v", err)
		os.Exit(1)
	}
}

func (a *Application) run() error {
	executable := filepath.Base(os.Args[0])
	shortName := a.getShortName(executable)

	cmd := newCommandTemplate(executable, shortName, a.TOMLConfig, a.Samplers...)
	cmd.RunE = func(cmd *cobra.Command, args []string) error {
		return a.executeCommand(shortName)
	}
	a.config = viper.New()
	a.config.SetDefault(cfgLogConsoleLevel, log.DefaultConsoleLevel)
	a.config.SetDefault(cfgLogConsoleFormat, "human")
	a.config.SetDefault(cfgLogConsoleStacktraceLevel, log.DefaultStacktraceLevel)
	a.config.SetDefault(cfgGeneralID, executable)
	// The configuration file location is specified through command-line flags.
	// Once the comand-line flags are parsed, we register the location of the
	// config file with the viper config.
	a.config.BindPFlag(cfgConfigFile, cmd.Flags().Lookup(cfgConfigFile))

	// All servers accept SIGTERM to perform clean shutdown (for example, this
	// is used behind the scenes by docker stop to cleanly shut down a container).
	sigterm := make(chan os.Signal)
	signal.Notify(sigterm, syscall.SIGTERM)
	go func() {
		defer log.HandlePanic()
		<-sigterm
		log.Info("Received SIGTERM signal, exiting...")
		// FIXME(scrye): Use context.Context and clean context propagation to
		// server modules instead of a global cancelation signal.
		fatal.Shutdown(env.ShutdownGraceInterval)
	}()

	return cmd.Execute()
}

func (a *Application) getShortName(executable string) string {
	if a.ShortName != "" {
		return a.ShortName
	}
	return executable
}

func (a *Application) executeCommand(shortName string) error {
	os.Setenv("TZ", "UTC")
	fatal.Init()

	// Load launcher configurations from the same config file as the custom
	// application configuration.
	a.config.SetConfigType("toml")
	a.config.SetConfigFile(a.config.GetString(cfgConfigFile))
	if err := a.config.ReadInConfig(); err != nil {
		return serrors.WrapStr("loading generic server config from file", err,
			"file", a.config.GetString(cfgConfigFile))
	}

	if err := libconfig.LoadFile(a.config.GetString(cfgConfigFile), a.TOMLConfig); err != nil {
		return serrors.WrapStr("loading config from file", err,
			"file", a.config.GetString(cfgConfigFile))
	}
	a.TOMLConfig.InitDefaults()

	logEntriesTotal := metrics.NewPromCounterFrom(
		prometheus.CounterOpts{
			Name: "lib_log_emitted_entries_total",
			Help: "Total number of log entries emitted.",
		},
		[]string{"level"},
	)
	opt := log.WithEntriesCounter(log.EntriesCounter{
		Debug: logEntriesTotal.With("level", "debug"),
		Info:  logEntriesTotal.With("level", "info"),
		Error: logEntriesTotal.With("level", "error"),
	})

	if err := log.Setup(a.getLogging(), opt); err != nil {
		return serrors.WrapStr("initialize logging", err)
	}
	defer log.Flush()
	if err := env.LogAppStarted(shortName, a.config.GetString(cfgGeneralID)); err != nil {
		return err
	}
	defer env.LogAppStopped(shortName, a.config.GetString(cfgGeneralID))
	defer log.HandlePanic()

	prom.ExportElementID(a.config.GetString(cfgGeneralID))
	if err := a.TOMLConfig.Validate(); err != nil {
		return serrors.WrapStr("validate config", err)
	}

	if a.Main == nil {
		return nil
	}
	return a.Main()
}

func (a *Application) getLogging() log.Config {
	return log.Config{
		Console: log.ConsoleConfig{
			Level:           a.config.GetString(cfgLogConsoleLevel),
			Format:          a.config.GetString(cfgLogConsoleFormat),
			StacktraceLevel: a.config.GetString(cfgLogConsoleStacktraceLevel),
		},
	}
}

func (a *Application) getErrorWriter() io.Writer {
	if a.ErrorWriter != nil {
		return a.ErrorWriter
	}
	return os.Stderr
}

// LoggingConfig is implemented by configurations that define logging behavior.
// If a application configuration does not implement this interface, then a
// default logging configuration is used.
type LoggingConfig interface {
	// Logging returns the logging configuration. The Get prefix is used to
	// avoid collisions with data members named Logging.
	GetLogging() log.Config
}

// IDConfig is implemented by configurations that define a SCION instance ID.
// If an application configuration does not implement this interface, then the
// SCION instance ID is equal to the application binary name.
type IDConfig interface {
	// ID returns the SCION instance ID of the application. The Get prefix is
	// used to avoid collisions with data members named ID.
	GetID() string
}

// newCommandTemplate returns a cobra command template for a SCION server application.
func newCommandTemplate(executable string, shortName string, config config.Sampler,
	samplers ...func(command.Pather) *cobra.Command) *cobra.Command {

	cmd := &cobra.Command{
		Use:           executable,
		Short:         shortName,
		Example:       fmt.Sprintf("  %s --config %s", executable, "config.toml"),
		SilenceErrors: true,
		SilenceUsage:  true,
		Args:          cobra.NoArgs,
	}
	cmd.AddCommand(
		command.NewCompletion(cmd),
		command.NewSample(
			cmd,
			append(samplers, command.NewSampleConfig(config))...,
		),
		command.NewVersion(cmd),
	)
	cmd.Flags().String(cfgConfigFile, "", "Configuration file (required)")
	cmd.MarkFlagRequired(cfgConfigFile)
	return cmd
}
<commit_msg>app: add newline on fatal error<commit_after>// Copyright 2020 Anapaya Systems
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package launcher includes the shared application execution boilerplate of all
// SCION servers.
package launcher

import (
	"fmt"
	"io"
	"os"
	"os/signal"
	"path/filepath"
	"syscall"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"

	"github.com/scionproto/scion/go/lib/config"
	libconfig "github.com/scionproto/scion/go/lib/config"
	"github.com/scionproto/scion/go/lib/env"
	"github.com/scionproto/scion/go/lib/fatal"
	"github.com/scionproto/scion/go/lib/log"
	"github.com/scionproto/scion/go/lib/metrics"
	"github.com/scionproto/scion/go/lib/prom"
	"github.com/scionproto/scion/go/lib/serrors"
	"github.com/scionproto/scion/go/pkg/command"
)

// Configuration keys used by the launcher
const (
	cfgLogConsoleLevel           = "log.console.level"
	cfgLogConsoleFormat          = "log.console.format"
	cfgLogConsoleStacktraceLevel = "log.console.stacktrace_level"
	cfgGeneralID                 = "general.id"
	cfgConfigFile                = "config"
)

// Application models a SCION server application.
type Application struct {
	// TOMLConfig holds the Go data structure for the application-specific
	// TOML configuration. The Application launcher will check if the TOMLConfig
	// supports additional methods (e.g., custom logging or instance ID) and
	// extract them from the config if that is the case. See the XxxConfig interfaces
	// in this package for more information.
	TOMLConfig libconfig.Config

	// Samplers contains additional configuration samplers to be included
	// under the sample subcommand. If empty, no additional samplers are
	// listed.
	//
	// DEPRECATED. This field will be removed once Anapaya/scion#5000 is implemented.
	Samplers []func(command.Pather) *cobra.Command

	// ShortName is the short name of the application. If empty, the executable name is used.
	// The ShortName could be, for example, "SCION Daemon" for the SCION Daemon.
	ShortName string

	// Main is the custom logic of the application. If nil, no custom logic is executed
	// (and only the setup/teardown harness runs). If Main returns an error, the
	// Run method will return a non-zero exit code.
	Main func() error

	// ErrorWriter specifies where error output should be printed. If nil, os.Stderr is used.
	ErrorWriter io.Writer

	// config contains the Viper configuration KV store.
	config *viper.Viper
}

// Run sets up the common SCION server harness, and then passes control to the Main
// function (if one exists).
//
// Run uses the following globals:
//   os.Args
//
// Run will exit the application if it encounters a fatal error.
func (a *Application) Run() {
	if err := a.run(); err != nil {
		fmt.Fprintf(a.getErrorWriter(), "fatal error: %v\n", err)
		os.Exit(1)
	}
}

func (a *Application) run() error {
	executable := filepath.Base(os.Args[0])
	shortName := a.getShortName(executable)

	cmd := newCommandTemplate(executable, shortName, a.TOMLConfig, a.Samplers...)
	cmd.RunE = func(cmd *cobra.Command, args []string) error {
		return a.executeCommand(shortName)
	}
	a.config = viper.New()
	a.config.SetDefault(cfgLogConsoleLevel, log.DefaultConsoleLevel)
	a.config.SetDefault(cfgLogConsoleFormat, "human")
	a.config.SetDefault(cfgLogConsoleStacktraceLevel, log.DefaultStacktraceLevel)
	a.config.SetDefault(cfgGeneralID, executable)
	// The configuration file location is specified through command-line flags.
	// Once the comand-line flags are parsed, we register the location of the
	// config file with the viper config.
	a.config.BindPFlag(cfgConfigFile, cmd.Flags().Lookup(cfgConfigFile))

	// All servers accept SIGTERM to perform clean shutdown (for example, this
	// is used behind the scenes by docker stop to cleanly shut down a container).
	sigterm := make(chan os.Signal)
	signal.Notify(sigterm, syscall.SIGTERM)
	go func() {
		defer log.HandlePanic()
		<-sigterm
		log.Info("Received SIGTERM signal, exiting...")
		// FIXME(scrye): Use context.Context and clean context propagation to
		// server modules instead of a global cancelation signal.
		fatal.Shutdown(env.ShutdownGraceInterval)
	}()

	return cmd.Execute()
}

func (a *Application) getShortName(executable string) string {
	if a.ShortName != "" {
		return a.ShortName
	}
	return executable
}

func (a *Application) executeCommand(shortName string) error {
	os.Setenv("TZ", "UTC")
	fatal.Init()

	// Load launcher configurations from the same config file as the custom
	// application configuration.
	a.config.SetConfigType("toml")
	a.config.SetConfigFile(a.config.GetString(cfgConfigFile))
	if err := a.config.ReadInConfig(); err != nil {
		return serrors.WrapStr("loading generic server config from file", err,
			"file", a.config.GetString(cfgConfigFile))
	}

	if err := libconfig.LoadFile(a.config.GetString(cfgConfigFile), a.TOMLConfig); err != nil {
		return serrors.WrapStr("loading config from file", err,
			"file", a.config.GetString(cfgConfigFile))
	}
	a.TOMLConfig.InitDefaults()

	logEntriesTotal := metrics.NewPromCounterFrom(
		prometheus.CounterOpts{
			Name: "lib_log_emitted_entries_total",
			Help: "Total number of log entries emitted.",
		},
		[]string{"level"},
	)
	opt := log.WithEntriesCounter(log.EntriesCounter{
		Debug: logEntriesTotal.With("level", "debug"),
		Info:  logEntriesTotal.With("level", "info"),
		Error: logEntriesTotal.With("level", "error"),
	})

	if err := log.Setup(a.getLogging(), opt); err != nil {
		return serrors.WrapStr("initialize logging", err)
	}
	defer log.Flush()
	if err := env.LogAppStarted(shortName, a.config.GetString(cfgGeneralID)); err != nil {
		return err
	}
	defer env.LogAppStopped(shortName, a.config.GetString(cfgGeneralID))
	defer log.HandlePanic()

	prom.ExportElementID(a.config.GetString(cfgGeneralID))
	if err := a.TOMLConfig.Validate(); err != nil {
		return serrors.WrapStr("validate config", err)
	}

	if a.Main == nil {
		return nil
	}
	return a.Main()
}

func (a *Application) getLogging() log.Config {
	return log.Config{
		Console: log.ConsoleConfig{
			Level:           a.config.GetString(cfgLogConsoleLevel),
			Format:          a.config.GetString(cfgLogConsoleFormat),
			StacktraceLevel: a.config.GetString(cfgLogConsoleStacktraceLevel),
		},
	}
}

func (a *Application) getErrorWriter() io.Writer {
	if a.ErrorWriter != nil {
		return a.ErrorWriter
	}
	return os.Stderr
}

// LoggingConfig is implemented by configurations that define logging behavior.
// If a application configuration does not implement this interface, then a
// default logging configuration is used.
type LoggingConfig interface {
	// Logging returns the logging configuration. The Get prefix is used to
	// avoid collisions with data members named Logging.
	GetLogging() log.Config
}

// IDConfig is implemented by configurations that define a SCION instance ID.
// If an application configuration does not implement this interface, then the
// SCION instance ID is equal to the application binary name.
type IDConfig interface {
	// ID returns the SCION instance ID of the application. The Get prefix is
	// used to avoid collisions with data members named ID.
	GetID() string
}

// newCommandTemplate returns a cobra command template for a SCION server application.
func newCommandTemplate(executable string, shortName string, config config.Sampler,
	samplers ...func(command.Pather) *cobra.Command) *cobra.Command {

	cmd := &cobra.Command{
		Use:           executable,
		Short:         shortName,
		Example:       fmt.Sprintf("  %s --config %s", executable, "config.toml"),
		SilenceErrors: true,
		SilenceUsage:  true,
		Args:          cobra.NoArgs,
	}
	cmd.AddCommand(
		command.NewCompletion(cmd),
		command.NewSample(
			cmd,
			append(samplers, command.NewSampleConfig(config))...,
		),
		command.NewVersion(cmd),
	)
	cmd.Flags().String(cfgConfigFile, "", "Configuration file (required)")
	cmd.MarkFlagRequired(cfgConfigFile)
	return cmd
}
<|endoftext|>