<commit_before>package main

import (
	"bytes"
	"crypto/rand"
	"crypto/tls"
	"crypto/x509"
	"expvar"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"syscall"
	"time"

	"github.com/ekanite/ekanite"
	"github.com/ekanite/ekanite/input"
)

var (
	stats = expvar.NewMap("ekanite")
)

// Program parameters
var datadir string
var tcpIface string
var udpIface string
var caPemPath string
var caKeyPath string
var queryIface string
var batchSize int
var batchTimeout int
var indexMaxPending int
var gomaxprocs int
var numShards int
var retentionPeriod string
var cpuProfile string
var memProfile string
var noReport bool

// Flag set
var fs *flag.FlagSet

// Types
const (
	DefaultDataDir         = "/var/opt/ekanite"
	DefaultBatchSize       = 300
	DefaultBatchTimeout    = 1000
	DefaultIndexMaxPending = 1000
	DefaultNumShards       = 4
	DefaultRetentionPeriod = "168h"
	DefaultQueryAddr       = "localhost:9950"
	DefaultDiagsIface      = "localhost:9951"
	DefaultTCPServer       = "localhost:5514"
)

func main() {
	fs = flag.NewFlagSet("", flag.ExitOnError)
	var (
		datadir         = fs.String("datadir", DefaultDataDir, "Set data directory.")
		batchSize       = fs.Int("batchsize", DefaultBatchSize, "Indexing batch size.")
		batchTimeout    = fs.Int("batchtime", DefaultBatchTimeout, "Indexing batch timeout, in milliseconds.")
		indexMaxPending = fs.Int("maxpending", DefaultIndexMaxPending, "Maximum pending index events.")
		tcpIface        = fs.String("tcp", DefaultTCPServer, "Syslog server TCP bind address in the form host:port. If empty, not started.")
		udpIface        = fs.String("udp", "", "Syslog server UDP bind address in the form host:port. If not set, not started.")
		diagIface       = fs.String("diag", DefaultDiagsIface, "expvar and pprof bind address in the form host:port. If not set, not started.")
		caPemPath       = fs.String("tlspem", "", "path to CA PEM file for TLS-enabled TCP server. If not set, TLS not activated")
		caKeyPath       = fs.String("tlskey", "", "path to CA key file for TLS-enabled TCP server. If not set, TLS not activated")
		queryIface      = fs.String("query", DefaultQueryAddr, "TCP Bind address for query server in the form host:port.")
		numShards       = fs.Int("numshards", DefaultNumShards, "Set number of shards per index.")
		retentionPeriod = fs.String("retention", DefaultRetentionPeriod, "Data retention period. Minimum is 24 hours")
		cpuProfile      = fs.String("cpuprof", "", "Where to write CPU profiling data. Not written if not set")
		memProfile      = fs.String("memprof", "", "Where to write memory profiling data. Not written if not set")
		noReport        = fs.Bool("noreport", false, "Do not report anonymous data on launch.")
	)
	fs.Usage = printHelp
	fs.Parse(os.Args[1:])

	absDataDir, err := filepath.Abs(*datadir)
	if err != nil {
		log.Fatalf("failed to get absolute data path for '%s': %s", *datadir, err.Error())
	}

	// Get the retention period.
	retention, err := time.ParseDuration(*retentionPeriod)
	if err != nil {
		log.Fatalf("failed to parse retention period '%s'", *retentionPeriod)
	}

	log.SetFlags(log.LstdFlags)
	log.SetPrefix("[ekanite] ")
	log.Printf("ekanite started using %s for index storage", absDataDir)

	runtime.GOMAXPROCS(runtime.NumCPU())
	log.Println("GOMAXPROCS set to", runtime.GOMAXPROCS(0))

	// Start the expvar handler if requested.
	if *diagIface != "" {
		sock, err := net.Listen("tcp", *diagIface)
		if err != nil {
			log.Fatalf("failed to create diag server: %s", err.Error())
		}
		go func() {
			log.Printf("diags now available at %s", *diagIface)
			http.Serve(sock, nil)
		}()
	}

	// Create and open the Engine.
	engine := ekanite.NewEngine(absDataDir)
	if engine == nil {
		log.Fatalf("failed to create indexing engine at %s", absDataDir)
	}
	engine.NumShards = *numShards
	engine.RetentionPeriod = retention

	if err := engine.Open(); err != nil {
		log.Fatalf("failed to open engine: %s", err.Error())
	}
	log.Printf("engine opened with shard number of %d, retention period of %s",
		engine.NumShards, engine.RetentionPeriod)

	// Start the query server.
	server := ekanite.NewServer(*queryIface, engine)
	if server == nil {
		log.Fatal("failed to create query server")
	}
	if err := server.Start(); err != nil {
		log.Fatalf("failed to start query server: %s", err.Error())
	}
	log.Printf("query server listening to %s", *queryIface)

	// Create and start the batcher.
	batcherTimeout := time.Duration(*batchTimeout) * time.Millisecond
	batcher := ekanite.NewBatcher(engine, *batchSize, batcherTimeout, *indexMaxPending)
	if batcher == nil {
		log.Fatal("failed to create indexing batcher")
	}

	errChan := make(chan error)
	if err := batcher.Start(errChan); err != nil {
		log.Fatalf("failed to start indexing batcher: %s", err.Error())
	}
	log.Printf("batching configured with size %d, timeout %s, max pending %d",
		*batchSize, batcherTimeout, *indexMaxPending)

	// Start draining batcher errors.
	go func() {
		for {
			select {
			case err := <-errChan:
				if err != nil {
					log.Printf("error indexing batch: %s", err.Error())
				}
			}
		}
	}()

	// Start TCP collector if requested.
	if *tcpIface != "" {
		var tlsConfig *tls.Config
		if *caPemPath != "" && *caKeyPath != "" {
			tlsConfig, err = newTLSConfig(*caPemPath, *caKeyPath)
			if err != nil {
				log.Fatalf("failed to configure TLS: %s", err.Error())
			}
			log.Printf("TLS successfully configured")
		}

		collector := input.NewCollector("tcp", *tcpIface, tlsConfig)
		if collector == nil {
			log.Fatalf("failed to created TCP collector bound to %s", *tcpIface)
		}
		if err := collector.Start(batcher.C()); err != nil {
			log.Fatalf("failed to start TCP collector: %s", err.Error())
		}
		log.Printf("TCP collector listening to %s", *tcpIface)
	}

	// Start UDP collector if requested.
	if *udpIface != "" {
		collector := input.NewCollector("udp", *udpIface, nil)
		if collector == nil {
			log.Fatalf("failed to created UDP collector for to %s", *udpIface)
		}
		if err := collector.Start(batcher.C()); err != nil {
			log.Fatalf("failed to start UDP collector: %s", err.Error())
		}
		log.Printf("UDP collector listening to %s", *tcpIface)
	}

	// Start profiling.
	startProfile(*cpuProfile, *memProfile)

	if !*noReport {
		reportLaunch()
	}

	stats.Set("launch", time.Now().UTC())

	signalCh := make(chan os.Signal, 1)
	signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM)
	// Block until one of the signals above is received
	select {
	case <-signalCh:
		log.Println("signal received, shutting down...")
	}

	stopProfile()
}

func newTLSConfig(caPemPath, caKeyPath string) (*tls.Config, error) {
	var config *tls.Config

	caPem, err := ioutil.ReadFile(caPemPath)
	if err != nil {
		return nil, err
	}
	ca, err := x509.ParseCertificate(caPem)
	if err != nil {
		return nil, err
	}

	caKey, err := ioutil.ReadFile(caKeyPath)
	if err != nil {
		return nil, err
	}
	key, err := x509.ParsePKCS1PrivateKey(caKey)
	if err != nil {
		return nil, err
	}
	pool := x509.NewCertPool()
	pool.AddCert(ca)

	cert := tls.Certificate{
		Certificate: [][]byte{caPem},
		PrivateKey:  key,
	}

	config = &tls.Config{
		ClientAuth:   tls.RequireAndVerifyClientCert,
		Certificates: []tls.Certificate{cert},
		ClientCAs:    pool,
	}

	config.Rand = rand.Reader

	return config, nil
}

// prof stores the file locations of active profiles.
var prof struct {
	cpu *os.File
	mem *os.File
}

// StartProfile initializes the cpu and memory profile, if specified.
func startProfile(cpuprofile, memprofile string) {
	if cpuprofile != "" {
		f, err := os.Create(cpuprofile)
		if err != nil {
			log.Fatalf("cpuprofile: %v", err)
		}
		log.Printf("writing CPU profile to: %s\n", cpuprofile)
		prof.cpu = f
		pprof.StartCPUProfile(prof.cpu)
	}

	if memprofile != "" {
		f, err := os.Create(memprofile)
		if err != nil {
			log.Fatalf("memprofile: %v", err)
		}
		log.Printf("writing memory profile to: %s\n", memprofile)
		prof.mem = f
		runtime.MemProfileRate = 4096
	}

}

// StopProfile closes the cpu and memory profiles if they are running.
func stopProfile() {
	if prof.cpu != nil {
		pprof.StopCPUProfile()
		prof.cpu.Close()
		log.Println("CPU profile stopped")
	}
	if prof.mem != nil {
		pprof.Lookup("heap").WriteTo(prof.mem, 0)
		prof.mem.Close()
		log.Println("memory profile stopped")
	}
}

func printHelp() {
	fmt.Println("ekanite [options]")
	fs.PrintDefaults()
}

func reportLaunch() {
	json := fmt.Sprintf(`{"os": "%s", "arch": "%s", "gomaxprocs": %d, "numcpu": %d, "numshards": %d, "app": "ekanited"}`,
		runtime.GOOS, runtime.GOARCH, runtime.GOMAXPROCS(0), runtime.NumCPU(), numShards)
	data := bytes.NewBufferString(json)
	client := http.Client{Timeout: time.Duration(5 * time.Second)}
	go client.Post("https://logs-01.loggly.com/inputs/8a0edd84-92ba-46e4-ada8-c529d0f105af/tag/reporting/",
		"application/json", data)
}
<commit_msg>Allow simple query server to be disabled<commit_after>package main

import (
	"bytes"
	"crypto/rand"
	"crypto/tls"
	"crypto/x509"
	"expvar"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"syscall"
	"time"

	"github.com/ekanite/ekanite"
	"github.com/ekanite/ekanite/input"
)

var (
	stats = expvar.NewMap("ekanite")
)

// Program parameters
var datadir string
var tcpIface string
var udpIface string
var caPemPath string
var caKeyPath string
var queryIface string
var batchSize int
var batchTimeout int
var indexMaxPending int
var gomaxprocs int
var numShards int
var retentionPeriod string
var cpuProfile string
var memProfile string
var noReport bool

// Flag set
var fs *flag.FlagSet

// Types
const (
	DefaultDataDir         = "/var/opt/ekanite"
	DefaultBatchSize       = 300
	DefaultBatchTimeout    = 1000
	DefaultIndexMaxPending = 1000
	DefaultNumShards       = 4
	DefaultRetentionPeriod = "168h"
	DefaultQueryAddr       = "localhost:9950"
	DefaultDiagsIface      = "localhost:9951"
	DefaultTCPServer       = "localhost:5514"
)

func main() {
	fs = flag.NewFlagSet("", flag.ExitOnError)
	var (
		datadir         = fs.String("datadir", DefaultDataDir, "Set data directory.")
		batchSize       = fs.Int("batchsize", DefaultBatchSize, "Indexing batch size.")
		batchTimeout    = fs.Int("batchtime", DefaultBatchTimeout, "Indexing batch timeout, in milliseconds.")
		indexMaxPending = fs.Int("maxpending", DefaultIndexMaxPending, "Maximum pending index events.")
		tcpIface        = fs.String("tcp", DefaultTCPServer, "Syslog server TCP bind address in the form host:port. To disable set to empty string.")
		udpIface        = fs.String("udp", "", "Syslog server UDP bind address in the form host:port. If not set, not started.")
		diagIface       = fs.String("diag", DefaultDiagsIface, "expvar and pprof bind address in the form host:port. If not set, not started.")
		caPemPath       = fs.String("tlspem", "", "path to CA PEM file for TLS-enabled TCP server. If not set, TLS not activated")
		caKeyPath       = fs.String("tlskey", "", "path to CA key file for TLS-enabled TCP server. If not set, TLS not activated")
		queryIface      = fs.String("query", DefaultQueryAddr, "TCP Bind address for query server in the form host:port. To disable set to empty string")
		numShards       = fs.Int("numshards", DefaultNumShards, "Set number of shards per index.")
		retentionPeriod = fs.String("retention", DefaultRetentionPeriod, "Data retention period. Minimum is 24 hours")
		cpuProfile      = fs.String("cpuprof", "", "Where to write CPU profiling data. Not written if not set")
		memProfile      = fs.String("memprof", "", "Where to write memory profiling data. Not written if not set")
		noReport        = fs.Bool("noreport", false, "Do not report anonymous data on launch.")
	)
	fs.Usage = printHelp
	fs.Parse(os.Args[1:])

	absDataDir, err := filepath.Abs(*datadir)
	if err != nil {
		log.Fatalf("failed to get absolute data path for '%s': %s", *datadir, err.Error())
	}

	// Get the retention period.
	retention, err := time.ParseDuration(*retentionPeriod)
	if err != nil {
		log.Fatalf("failed to parse retention period '%s'", *retentionPeriod)
	}

	log.SetFlags(log.LstdFlags)
	log.SetPrefix("[ekanite] ")
	log.Printf("ekanite started using %s for index storage", absDataDir)

	runtime.GOMAXPROCS(runtime.NumCPU())
	log.Println("GOMAXPROCS set to", runtime.GOMAXPROCS(0))

	// Start the expvar handler if requested.
	if *diagIface != "" {
		sock, err := net.Listen("tcp", *diagIface)
		if err != nil {
			log.Fatalf("failed to create diag server: %s", err.Error())
		}
		go func() {
			log.Printf("diags now available at %s", *diagIface)
			http.Serve(sock, nil)
		}()
	}

	// Create and open the Engine.
	engine := ekanite.NewEngine(absDataDir)
	if engine == nil {
		log.Fatalf("failed to create indexing engine at %s", absDataDir)
	}
	engine.NumShards = *numShards
	engine.RetentionPeriod = retention

	if err := engine.Open(); err != nil {
		log.Fatalf("failed to open engine: %s", err.Error())
	}
	log.Printf("engine opened with shard number of %d, retention period of %s",
		engine.NumShards, engine.RetentionPeriod)

	// Start the simple query server if requested.
	if *queryIface != "" {
		server := ekanite.NewServer(*queryIface, engine)
		if server == nil {
			log.Fatal("failed to create query server")
		}
		if err := server.Start(); err != nil {
			log.Fatalf("failed to start query server: %s", err.Error())
		}
		log.Printf("query server listening to %s", *queryIface)
	}

	// Create and start the batcher.
	batcherTimeout := time.Duration(*batchTimeout) * time.Millisecond
	batcher := ekanite.NewBatcher(engine, *batchSize, batcherTimeout, *indexMaxPending)
	if batcher == nil {
		log.Fatal("failed to create indexing batcher")
	}

	errChan := make(chan error)
	if err := batcher.Start(errChan); err != nil {
		log.Fatalf("failed to start indexing batcher: %s", err.Error())
	}
	log.Printf("batching configured with size %d, timeout %s, max pending %d",
		*batchSize, batcherTimeout, *indexMaxPending)

	// Start draining batcher errors.
	go func() {
		for {
			select {
			case err := <-errChan:
				if err != nil {
					log.Printf("error indexing batch: %s", err.Error())
				}
			}
		}
	}()

	// Start TCP collector if requested.
	if *tcpIface != "" {
		var tlsConfig *tls.Config
		if *caPemPath != "" && *caKeyPath != "" {
			tlsConfig, err = newTLSConfig(*caPemPath, *caKeyPath)
			if err != nil {
				log.Fatalf("failed to configure TLS: %s", err.Error())
			}
			log.Printf("TLS successfully configured")
		}

		collector := input.NewCollector("tcp", *tcpIface, tlsConfig)
		if collector == nil {
			log.Fatalf("failed to created TCP collector bound to %s", *tcpIface)
		}
		if err := collector.Start(batcher.C()); err != nil {
			log.Fatalf("failed to start TCP collector: %s", err.Error())
		}
		log.Printf("TCP collector listening to %s", *tcpIface)
	}

	// Start UDP collector if requested.
	if *udpIface != "" {
		collector := input.NewCollector("udp", *udpIface, nil)
		if collector == nil {
			log.Fatalf("failed to created UDP collector for to %s", *udpIface)
		}
		if err := collector.Start(batcher.C()); err != nil {
			log.Fatalf("failed to start UDP collector: %s", err.Error())
		}
		log.Printf("UDP collector listening to %s", *tcpIface)
	}

	// Start profiling.
	startProfile(*cpuProfile, *memProfile)

	if !*noReport {
		reportLaunch()
	}

	stats.Set("launch", time.Now().UTC())

	signalCh := make(chan os.Signal, 1)
	signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM)
	// Block until one of the signals above is received
	select {
	case <-signalCh:
		log.Println("signal received, shutting down...")
	}

	stopProfile()
}

func newTLSConfig(caPemPath, caKeyPath string) (*tls.Config, error) {
	var config *tls.Config

	caPem, err := ioutil.ReadFile(caPemPath)
	if err != nil {
		return nil, err
	}
	ca, err := x509.ParseCertificate(caPem)
	if err != nil {
		return nil, err
	}

	caKey, err := ioutil.ReadFile(caKeyPath)
	if err != nil {
		return nil, err
	}
	key, err := x509.ParsePKCS1PrivateKey(caKey)
	if err != nil {
		return nil, err
	}
	pool := x509.NewCertPool()
	pool.AddCert(ca)

	cert := tls.Certificate{
		Certificate: [][]byte{caPem},
		PrivateKey:  key,
	}

	config = &tls.Config{
		ClientAuth:   tls.RequireAndVerifyClientCert,
		Certificates: []tls.Certificate{cert},
		ClientCAs:    pool,
	}

	config.Rand = rand.Reader

	return config, nil
}

// prof stores the file locations of active profiles.
var prof struct {
	cpu *os.File
	mem *os.File
}

// StartProfile initializes the cpu and memory profile, if specified.
func startProfile(cpuprofile, memprofile string) {
	if cpuprofile != "" {
		f, err := os.Create(cpuprofile)
		if err != nil {
			log.Fatalf("cpuprofile: %v", err)
		}
		log.Printf("writing CPU profile to: %s\n", cpuprofile)
		prof.cpu = f
		pprof.StartCPUProfile(prof.cpu)
	}

	if memprofile != "" {
		f, err := os.Create(memprofile)
		if err != nil {
			log.Fatalf("memprofile: %v", err)
		}
		log.Printf("writing memory profile to: %s\n", memprofile)
		prof.mem = f
		runtime.MemProfileRate = 4096
	}

}

// StopProfile closes the cpu and memory profiles if they are running.
func stopProfile() {
	if prof.cpu != nil {
		pprof.StopCPUProfile()
		prof.cpu.Close()
		log.Println("CPU profile stopped")
	}
	if prof.mem != nil {
		pprof.Lookup("heap").WriteTo(prof.mem, 0)
		prof.mem.Close()
		log.Println("memory profile stopped")
	}
}

func printHelp() {
	fmt.Println("ekanite [options]")
	fs.PrintDefaults()
}

func reportLaunch() {
	json := fmt.Sprintf(`{"os": "%s", "arch": "%s", "gomaxprocs": %d, "numcpu": %d, "numshards": %d, "app": "ekanited"}`,
		runtime.GOOS, runtime.GOARCH, runtime.GOMAXPROCS(0), runtime.NumCPU(), numShards)
	data := bytes.NewBufferString(json)
	client := http.Client{Timeout: time.Duration(5 * time.Second)}
	go client.Post("https://logs-01.loggly.com/inputs/8a0edd84-92ba-46e4-ada8-c529d0f105af/tag/reporting/",
		"application/json", data)
}
<|endoftext|>


<commit_before>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/mvdan/basedir"
)

const (
	cmdName = "fdroidcl"

	repoName = "repo"
	repoURL  = "https://f-droid.org/repo"
)

func subdir(dir, name string) string {
	p := filepath.Join(dir, name)
	if err := os.MkdirAll(p, 0755); err != nil {
		log.Fatalf("Could not create dir '%s': %v", p, err)
	}
	return p
}

func mustCache() string {
	dir, err := basedir.Cache()
	if err != nil {
		log.Fatalf("Could not determine cache dir: %v", err)
	}
	return subdir(dir, cmdName)
}

func mustConfig() string {
	dir, err := basedir.Config()
	if err != nil {
		log.Fatalf("Could not determine config dir: %v", err)
	}
	return subdir(dir, cmdName)
}

var config struct {
	Repos map[string]struct {
		URL string `json:"url"`
	} `json:"repos"`
}

// A Command is an implementation of a go command
// like go build or go fix.
type Command struct {
	// Run runs the command.
	// The args are the arguments after the command name.
	Run func(args []string)

	// UsageLine is the one-line usage message.
	// The first word in the line is taken to be the command name.
	UsageLine string

	// Short is the short description.
	Short string

	// Flag is a set of flags specific to this command.
	Flag flag.FlagSet
}

// Name returns the command's name: the first word in the usage line.
func (c *Command) Name() string {
	name := c.UsageLine
	i := strings.Index(name, " ")
	if i >= 0 {
		name = name[:i]
	}
	return name
}

func (c *Command) Usage() {
	fmt.Fprintf(os.Stderr, "Usage: %s %s [-h]\n", cmdName, c.UsageLine)
	anyFlags := false
	c.Flag.VisitAll(func(f *flag.Flag) { anyFlags = true })
	if anyFlags {
		fmt.Fprintf(os.Stderr, "\nAvailable options:\n")
		c.Flag.PrintDefaults()
	}
	os.Exit(2)
}

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [-h] <command> [<args>]\n\n", cmdName)
		fmt.Fprintf(os.Stderr, "Available commands:\n")
		maxUsageLen := 0
		for _, c := range commands {
			if len(c.UsageLine) > maxUsageLen {
				maxUsageLen = len(c.UsageLine)
			}
		}
		for _, c := range commands {
			fmt.Fprintf(os.Stderr, "   %s%s  %s\n", c.UsageLine,
				strings.Repeat(" ", maxUsageLen-len(c.UsageLine)), c.Short)
		}
		fmt.Fprintf(os.Stderr, "\nUse %s <command> -h for more info\n", cmdName)
	}
}

// Commands lists the available commands.
var commands = []*Command{
	cmdUpdate,
	cmdSearch,
	cmdShow,
	cmdDevices,
	cmdInstall,
	cmdUninstall,
}

func main() {
	flag.Parse()
	args := flag.Args()

	if len(args) < 1 {
		flag.Usage()
		os.Exit(2)
	}

	for _, cmd := range commands {
		if cmd.Name() != args[0] {
			continue
		}
		cmd.Flag.Usage = func() { cmd.Usage() }
		cmd.Flag.Parse(args[1:])
		args = cmd.Flag.Args()
		cmd.Run(args)
		os.Exit(0)
	}

	switch args[0] {
	default:
		log.Printf("Unrecognised command '%s'\n\n", args[0])
		flag.Usage()
		os.Exit(2)
	}
}
<commit_msg>Start using a config file (unused as of now)<commit_after>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/mvdan/basedir"
)

const (
	cmdName = "fdroidcl"

	repoName = "repo"
	repoURL  = "https://f-droid.org/repo"
)

func subdir(dir, name string) string {
	p := filepath.Join(dir, name)
	if err := os.MkdirAll(p, 0755); err != nil {
		log.Fatalf("Could not create dir '%s': %v", p, err)
	}
	return p
}

func mustCache() string {
	dir, err := basedir.Cache()
	if err != nil {
		log.Fatalf("Could not determine cache dir: %v", err)
	}
	return subdir(dir, cmdName)
}

func mustConfig() string {
	dir, err := basedir.Config()
	if err != nil {
		log.Fatalf("Could not determine config dir: %v", err)
	}
	return subdir(dir, cmdName)
}

type Repo struct {
	URL string `json:"url"`
}

type userConfig struct {
	Repos map[string]Repo `json:"repos"`
}

var config = userConfig{
	Repos: map[string]Repo{
		"f-droid": {
			URL: "https://f-droid.org/repo",
		},
	},
}

func readConfig() {
	p := filepath.Join(mustConfig(), "config.json")
	f, err := os.Open(p)
	if err != nil {
		if os.IsNotExist(err) {
			return
		}
		log.Fatalf("Error when opening config file: %v", err)
	}
	defer f.Close()
	if err := json.NewDecoder(f).Decode(&config); err != nil {
		log.Fatalf("Error when parsing config file: %v", err)
	}
}

// A Command is an implementation of a go command
// like go build or go fix.
type Command struct {
	// Run runs the command.
	// The args are the arguments after the command name.
	Run func(args []string)

	// UsageLine is the one-line usage message.
	// The first word in the line is taken to be the command name.
	UsageLine string

	// Short is the short description.
	Short string

	// Flag is a set of flags specific to this command.
	Flag flag.FlagSet
}

// Name returns the command's name: the first word in the usage line.
func (c *Command) Name() string {
	name := c.UsageLine
	i := strings.Index(name, " ")
	if i >= 0 {
		name = name[:i]
	}
	return name
}

func (c *Command) Usage() {
	fmt.Fprintf(os.Stderr, "Usage: %s %s [-h]\n", cmdName, c.UsageLine)
	anyFlags := false
	c.Flag.VisitAll(func(f *flag.Flag) { anyFlags = true })
	if anyFlags {
		fmt.Fprintf(os.Stderr, "\nAvailable options:\n")
		c.Flag.PrintDefaults()
	}
	os.Exit(2)
}

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [-h] <command> [<args>]\n\n", cmdName)
		fmt.Fprintf(os.Stderr, "Available commands:\n")
		maxUsageLen := 0
		for _, c := range commands {
			if len(c.UsageLine) > maxUsageLen {
				maxUsageLen = len(c.UsageLine)
			}
		}
		for _, c := range commands {
			fmt.Fprintf(os.Stderr, "   %s%s  %s\n", c.UsageLine,
				strings.Repeat(" ", maxUsageLen-len(c.UsageLine)), c.Short)
		}
		fmt.Fprintf(os.Stderr, "\nUse %s <command> -h for more info\n", cmdName)
	}
}

// Commands lists the available commands.
var commands = []*Command{
	cmdUpdate,
	cmdSearch,
	cmdShow,
	cmdDevices,
	cmdInstall,
	cmdUninstall,
}

func main() {
	flag.Parse()
	args := flag.Args()

	if len(args) < 1 {
		flag.Usage()
		os.Exit(2)
	}

	for _, cmd := range commands {
		if cmd.Name() != args[0] {
			continue
		}
		readConfig()
		cmd.Flag.Usage = func() { cmd.Usage() }
		cmd.Flag.Parse(args[1:])
		args = cmd.Flag.Args()
		cmd.Run(args)
		os.Exit(0)
	}

	switch args[0] {
	default:
		log.Printf("Unrecognised command '%s'\n\n", args[0])
		flag.Usage()
		os.Exit(2)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"
)

var (
	addr       = flag.String("http", ":8080", "HTTP address to serve on")
	tlsAddr    = flag.String("https", "", "HTTPS address to serve on")
	pemFile    = flag.String("pem-file", "", "HTTPS / SSL certificate .pem file")
	keyFile    = flag.String("key-file", "", "HTTPS / SSL certificate .key file")
	update     = flag.Bool("update", true, "update via Git and shutdown server after pull")
	updateRate = flag.String("update-rate", "60s", "rate at which to check for updates via Git")
	dev        = flag.Bool("dev", false, "reload all templates on each request")

	staticDirPrefix = "/static/" // Path prefix to strip to get static dir root.
	staticDir       = "static/"  // Directory to serve for static files.

	errorTemplate = "error"      // Template to use for errors.
	templateExt   = ".tmpl"      // Filepath extension of template files.
	templateIncl  = ".incl"      // Filepath extension of template includes.
	templateDir   = "templates/" // Relative directory that templates reside in.

	src = &GitUpdater{
		Dir: ".",
	}
	lastUpdate  = time.Now()
	tmpls       *template.Template
	updateRateT time.Duration
)

func reloadTemplates() error {
	// Recursively match all filepaths with the template extension.
	var matches []string
	err := filepath.Walk(templateDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if e := filepath.Ext(path); e == templateExt || e == templateIncl {
			matches = append(matches, path)
		}
		return nil
	})
	if err != nil {
		return err
	}

	// Load the templates with proper names.
	tmpls = template.New("")
	for _, m := range matches {
		name := strings.TrimPrefix(m, templateDir)
		if _, err := tmpls.New(name).Parse(m); err != nil {
			return err
		}
	}
	return nil
}

type TemplateData struct {
	Request *http.Request
	Error   string
}

func handler(w http.ResponseWriter, r *http.Request) error {
	if *dev {
		if err := reloadTemplates(); err != nil {
			return err
		}
	}

	var data = &TemplateData{
		Request: r,
	}

	// Determine template name.
	tmplName := r.URL.Path[1:]
	if stat, err := os.Stat(filepath.Join(templateDir, tmplName)); err != nil {
		return err
	} else {
		if stat.IsDir() {
			tmplName = path.Join(tmplName, "index")
		}
	}

	// Find the requested page.
	root := tmpls.Lookup(tmplName)
	if root == nil {
		// Couldn't find the page, render the error page.
		w.WriteHeader(http.StatusNotFound)
		return fmt.Errorf("%v - %v", http.StatusNotFound, http.StatusText(http.StatusNotFound))
	}

	// Execute the requested template page.
	return root.Execute(w, data)
}

func logHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		log.Printf("%v %v\n", r.Method, r.URL)
		h.ServeHTTP(w, r)
	})
}

func errorHandler(h func(w http.ResponseWriter, r *http.Request) error) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		err := h(w, r)
		if err == nil {
			log.Printf("%v %v\n", r.Method, r.URL)
			return
		}
		log.Printf("[error] %v %v - %v\n", r.Method, r.URL, err)

		// Render the error template.
		data := &TemplateData{
			Request: r,
			Error:   err.Error(),
		}
		if err := tmpls.ExecuteTemplate(w, errorTemplate, data); err != nil {
			panic(err)
		}
	})
}

func checkForUpdates() {
	// If not enough time has elapsed, skip the update.
	if time.Since(lastUpdate) < updateRateT {
		return
	}
	lastUpdate = time.Now()

	// Check for updates.
	updated, err := src.Update()
	if err != nil {
		log.Println("Update error:", err)
		return
	}
	if updated {
		log.Println("Updated source code. Exiting server..")
		os.Exit(0)
	}
	log.Println("checked for updates (none available)")
}

func init() {
	if err := reloadTemplates(); err != nil {
		log.Fatal(err)
	}

	if *update {
		go func() {
			for {
				time.Sleep(updateRateT)
				checkForUpdates()
			}
		}()
	}
}

func main() {
	flag.Parse()

	// Parse update rate flag.
	var err error
	updateRateT, err = time.ParseDuration(*updateRate)
	if err != nil {
		log.Fatal(err)
	}

	// Static file hosting
	http.Handle("/static/", logHandler(http.StripPrefix(staticDirPrefix, http.FileServer(http.Dir(staticDir)))))

	// App handler.
	http.Handle("/", errorHandler(handler))

	// Start HTTPS server:
	if *tlsAddr != "" {
		go func() {
			if *pemFile == "" {
				log.Fatal("expected -pem-file flag")
			}
			if *keyFile == "" {
				log.Fatal("expected -key-file flag")
			}
			log.Println("Serving on", *tlsAddr)
			log.Fatal(http.ListenAndServeTLS(*tlsAddr, *pemFile, *keyFile, nil))
		}()
	}

	// Start HTTP server:
	log.Println("Serving on", *addr)
	log.Fatal(http.ListenAndServe(*addr, nil))
}
<commit_msg>cmd/fiz3d-org: fix error template name<commit_after>package main

import (
	"flag"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"
)

var (
	addr       = flag.String("http", ":8080", "HTTP address to serve on")
	tlsAddr    = flag.String("https", "", "HTTPS address to serve on")
	pemFile    = flag.String("pem-file", "", "HTTPS / SSL certificate .pem file")
	keyFile    = flag.String("key-file", "", "HTTPS / SSL certificate .key file")
	update     = flag.Bool("update", true, "update via Git and shutdown server after pull")
	updateRate = flag.String("update-rate", "60s", "rate at which to check for updates via Git")
	dev        = flag.Bool("dev", false, "reload all templates on each request")

	staticDirPrefix = "/static/" // Path prefix to strip to get static dir root.
	staticDir       = "static/"  // Directory to serve for static files.

	errorTemplate = "error.tmpl" // Template to use for errors.
	templateExt   = ".tmpl"      // Filepath extension of template files.
	templateIncl  = ".incl"      // Filepath extension of template includes.
	templateDir   = "templates/" // Relative directory that templates reside in.

	src = &GitUpdater{
		Dir: ".",
	}
	lastUpdate  = time.Now()
	tmpls       *template.Template
	updateRateT time.Duration
)

func reloadTemplates() error {
	// Recursively match all filepaths with the template extension.
	var matches []string
	err := filepath.Walk(templateDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if e := filepath.Ext(path); e == templateExt || e == templateIncl {
			matches = append(matches, path)
		}
		return nil
	})
	if err != nil {
		return err
	}

	// Load the templates with proper names.
	tmpls = template.New("")
	for _, m := range matches {
		name := strings.TrimPrefix(m, templateDir)
		if _, err := tmpls.New(name).Parse(m); err != nil {
			return err
		}
	}
	return nil
}

type TemplateData struct {
	Request *http.Request
	Error   string
}

func handler(w http.ResponseWriter, r *http.Request) error {
	if *dev {
		if err := reloadTemplates(); err != nil {
			return err
		}
	}

	var data = &TemplateData{
		Request: r,
	}

	// Determine template name.
	tmplName := r.URL.Path[1:]
	if stat, err := os.Stat(filepath.Join(templateDir, tmplName)); err != nil {
		return err
	} else {
		if stat.IsDir() {
			tmplName = path.Join(tmplName, "index")
		}
	}

	// Find the requested page.
	root := tmpls.Lookup(tmplName)
	if root == nil {
		// Couldn't find the page, render the error page.
		w.WriteHeader(http.StatusNotFound)
		return fmt.Errorf("%v - %v", http.StatusNotFound, http.StatusText(http.StatusNotFound))
	}

	// Execute the requested template page.
	return root.Execute(w, data)
}

func logHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		log.Printf("%v %v\n", r.Method, r.URL)
		h.ServeHTTP(w, r)
	})
}

func errorHandler(h func(w http.ResponseWriter, r *http.Request) error) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		err := h(w, r)
		if err == nil {
			log.Printf("%v %v\n", r.Method, r.URL)
			return
		}
		log.Printf("[error] %v %v - %v\n", r.Method, r.URL, err)

		// Render the error template.
		data := &TemplateData{
			Request: r,
			Error:   err.Error(),
		}
		if err := tmpls.ExecuteTemplate(w, errorTemplate, data); err != nil {
			panic(err)
		}
	})
}

func checkForUpdates() {
	// If not enough time has elapsed, skip the update.
	if time.Since(lastUpdate) < updateRateT {
		return
	}
	lastUpdate = time.Now()

	// Check for updates.
	updated, err := src.Update()
	if err != nil {
		log.Println("Update error:", err)
		return
	}
	if updated {
		log.Println("Updated source code. Exiting server..")
		os.Exit(0)
	}
	log.Println("checked for updates (none available)")
}

func init() {
	if err := reloadTemplates(); err != nil {
		log.Fatal(err)
	}

	if *update {
		go func() {
			for {
				time.Sleep(updateRateT)
				checkForUpdates()
			}
		}()
	}
}

func main() {
	flag.Parse()

	// Parse update rate flag.
	var err error
	updateRateT, err = time.ParseDuration(*updateRate)
	if err != nil {
		log.Fatal(err)
	}

	// Static file hosting
	http.Handle("/static/", logHandler(http.StripPrefix(staticDirPrefix, http.FileServer(http.Dir(staticDir)))))

	// App handler.
	http.Handle("/", errorHandler(handler))

	// Start HTTPS server:
	if *tlsAddr != "" {
		go func() {
			if *pemFile == "" {
				log.Fatal("expected -pem-file flag")
			}
			if *keyFile == "" {
				log.Fatal("expected -key-file flag")
			}
			log.Println("Serving on", *tlsAddr)
			log.Fatal(http.ListenAndServeTLS(*tlsAddr, *pemFile, *keyFile, nil))
		}()
	}

	// Start HTTP server:
	log.Println("Serving on", *addr)
	log.Fatal(http.ListenAndServe(*addr, nil))
}
<|endoftext|>


<commit_before>package cmd

import (
	"bytes"
	"io"
	"os"
	"regexp"
	"testing"

	"github.com/spf13/cobra"

	"github.com/ironman-project/ironman/pkg/ironman"

	"github.com/ironman-project/ironman/pkg/testutils"
)

type cmdTestCase struct {
	name     string
	args     []string
	flags    []string
	expected string
	err      bool
}

type testCmdFactory func(ironman *ironman.Ironman, out io.Writer) *cobra.Command

func TestGenerateCmd(t *testing.T) {

	tests := []cmdTestCase{
		{
			"successful generate",
			[]string{"valid", "test-gen"},
			[]string{""},
			"",
			false,
		},
	}
	runCmdTests(t, tests, func(client *ironman.Ironman, out io.Writer) *cobra.Command {
		return newGenerateCommand(client, out)
	})

}

func runCmdTests(t *testing.T, tests []cmdTestCase, cmdFactory testCmdFactory) {
	var buf bytes.Buffer
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			tempHome := testutils.CreateTempDir("ihome", t)
			testutils.CopyDir("testing/ihome", tempHome, t)
			client := ironman.New(tempHome)
			defer func() {
				err := os.RemoveAll(tempHome)
				if err != nil {
					t.Fatal("Failed to remove test ironman home", err)
				}

			}()
			cmd := cmdFactory(client, &buf)
			cmd.ParseFlags(tt.flags)
			err := cmd.RunE(cmd, tt.args)
			if (err != nil) != tt.err {
				t.Errorf("expected error, got '%v'", err)
			}
			re := regexp.MustCompile(tt.expected)
			if !re.Match(buf.Bytes()) {
				t.Errorf("expected\n%q\ngot\n%q", tt.expected, buf.String())
			}
			buf.Reset()
		})
	}
}
<commit_msg>fix tests<commit_after>package cmd

import (
	"bytes"
	"io"
	"os"
	"regexp"
	"testing"

	"github.com/spf13/cobra"

	"github.com/ironman-project/ironman/pkg/ironman"

	"github.com/ironman-project/ironman/pkg/testutils"
)

type cmdTestCase struct {
	name     string
	args     []string
	flags    []string
	expected string
	err      bool
}

type testCmdFactory func(ironman *ironman.Ironman, out io.Writer) *cobra.Command

func TestGenerateCmd(t *testing.T) {

	tests := []cmdTestCase{
		{
			"successful generate",
			[]string{"valid", "test-gen"},
			[]string{""},
			"",
			true,
		},
	}
	runCmdTests(t, tests, func(client *ironman.Ironman, out io.Writer) *cobra.Command {
		return newGenerateCommand(client, out)
	})

}

func runCmdTests(t *testing.T, tests []cmdTestCase, cmdFactory testCmdFactory) {
	var buf bytes.Buffer
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			tempHome := testutils.CreateTempDir("ihome", t)
			testutils.CopyDir("testing/ihome", tempHome, t)
			client := ironman.New(tempHome)
			defer func() {
				err := os.RemoveAll(tempHome)
				if err != nil {
					t.Fatal("Failed to remove test ironman home", err)
				}

			}()
			cmd := cmdFactory(client, &buf)
			cmd.ParseFlags(tt.flags)
			err := cmd.RunE(cmd, tt.args)
			if (err != nil) != tt.err {
				t.Errorf("expected error, got '%v'", err)
			}
			re := regexp.MustCompile(tt.expected)
			if !re.Match(buf.Bytes()) {
				t.Errorf("expected\n%q\ngot\n%q", tt.expected, buf.String())
			}
			buf.Reset()
		})
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/codegangsta/cli"
	"github.com/joefitzgerald/inductor/configuration"
	"github.com/joefitzgerald/inductor/renderer"
	"github.com/joefitzgerald/inductor/tpl"
)

// Version of the CLI
var Version = "0.0.0"

func main() {
	app := newApp()
	err := app.Run(os.Args)
	if err != nil {
		die(err)
	}
}

func newApp() *cli.App {
	app := cli.NewApp()
	app.Name = "Inductor"
	app.Usage = "Generate Packer Templates"
	app.Version = Version
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "config, c",
			Value: "inductor.json",
			Usage: "Inductor configuration source file",
		},
		cli.StringFlag{
			Name:  "outdir, o",
			Value: "out",
			Usage: "The root output directory for all rendered templates",
		},
		cli.StringFlag{
			Name:  "edition, e",
			Usage: "The optional operating system edition",
		},
		cli.StringFlag{
			Name:  "productkey, k",
			Usage: "The MS Windows product key if you have one",
		},
		cli.BoolFlag{
			Name:  "skipwindowsupdates, u",
			Usage: "Skips running Windows updates on first boot",
		},
		cli.BoolFlag{
			Name:  "ssh, s",
			Usage: "Uses the Packer SSH communicator instead of the default WinRM",
		},
		cli.BoolFlag{
			Name:  "gui, g",
			Usage: "Run the VM with a GUI",
		},
	}
	app.Action = func(c *cli.Context) {
		// load the inductor configuration
		configFile, err := os.Open(c.String("config"))
		if err != nil {
			die("Couldn't find the requried inductor.json configuration file", err)
		}
		defer configFile.Close()

		config, err := configuration.New(configFile)
		if err != nil {
			die("Couldn't load the inductor.json configuration file", err)
		}

		// get our required windows version/edition string, e.g. 'windows10'
		var osname string
		if len(c.Args()) > 0 {
			osname = c.Args()[0]
		} else {
			fmt.Println("Available Operating Systems:")
			fmt.Println()
			for _, s := range config.List() {
				fmt.Println(fmt.Sprintf("  %s", s))
			}
			fmt.Println()
			die("You must specify an operating system argument")
		}

		// create the default options set based on the inductor config
		var edition string
		if len(c.String("edition")) > 0 {
			edition = c.String("edition")
		}
		opts, err := renderer.NewRenderOptions(osname, edition, config)
		if err != nil {
			die(err)
		}

		// apply any command line overrides to the options set
		opts.WindowsUpdates = !c.Bool("skipwindowsupdates")
		opts.Headless = !c.Bool("gui")
		if len(c.String("productkey")) > 0 {
			opts.ProductKey = c.String("productkey")
		}
		if c.Bool("ssh") {
			opts.Communicator = "ssh"
		}

		outDir := config.OutDir
		if len(c.String("outdir")) > 0 {
			outDir = c.String("outdir")
		}
		outDir, err = filepath.Abs(outDir)
		if err != nil {
			die(err)
		}

		// find all templates
		cwd, err := os.Getwd()
		if err != nil {
			die(err)
		}
		templates := tpl.New(cwd, osname)

		// finally render all the templates to the output directory
		renderer := renderer.New(opts, outDir)
		err = renderer.Render(templates)
		if err != nil {
			die(err)
		}

		// this allows us to do command substitution with Packer
		// TODO: output path to packer.json
	}
	return app
}

func die(vals ...interface{}) {
	if len(vals) > 1 || vals[0] != nil {
		os.Stderr.WriteString(fmt.Sprintln(vals...))
		os.Exit(1)
	}
}
<commit_msg>Fix leaking errors<commit_after>package main

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/codegangsta/cli"
	"github.com/joefitzgerald/inductor/configuration"
	"github.com/joefitzgerald/inductor/renderer"
	"github.com/joefitzgerald/inductor/tpl"
)

// Version of the CLI
var Version = "0.0.0"

func main() {
	app := newApp()
	err := app.Run(os.Args)
	if err != nil {
		die(err)
	}
}

func newApp() *cli.App {
	app := cli.NewApp()
	app.Name = "Inductor"
	app.Usage = "Generate Packer Templates"
	app.Version = Version
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "config, c",
			Value: "inductor.json",
			Usage: "Inductor configuration source file",
		},
		cli.StringFlag{
			Name:  "outdir, o",
			Value: "out",
			Usage: "The root output directory for all rendered templates",
		},
		cli.StringFlag{
			Name:  "edition, e",
			Usage: "The optional operating system edition",
		},
		cli.StringFlag{
			Name:  "productkey, k",
			Usage: "The MS Windows product key if you have one",
		},
		cli.BoolFlag{
			Name:  "skipwindowsupdates, u",
			Usage: "Skips running Windows updates on first boot",
		},
		cli.BoolFlag{
			Name:  "ssh, s",
			Usage: "Uses the Packer SSH communicator instead of the default WinRM",
		},
		cli.BoolFlag{
			Name:  "gui, g",
			Usage: "Run the VM with a GUI",
		},
	}
	app.Action = func(c *cli.Context) {
		// load the inductor configuration
		configFile, err := os.Open(c.String("config"))
		if err != nil {
			die("Couldn't find the requried inductor.json configuration file.", err)
		}
		defer func() {
			if cerr := configFile.Close(); cerr != nil {
				die(cerr)
			}
		}()

		config, err := configuration.New(configFile)
		if err != nil {
			die("Couldn't load the inductor.json configuration file.", err)
		}

		// get our required windows version/edition string, e.g. 'windows10'
		var osname string
		if len(c.Args()) > 0 {
			osname = c.Args()[0]
		} else {
			fmt.Println("Available Operating Systems:")
			fmt.Println()
			for _, s := range config.List() {
				fmt.Println(fmt.Sprintf("  %s", s))
			}
			fmt.Println()
			die("You must specify an operating system argument")
		}

		// create the default options set based on the inductor config
		var edition string
		if len(c.String("edition")) > 0 {
			edition = c.String("edition")
		}
		opts, err := renderer.NewRenderOptions(osname, edition, config)
		if err != nil {
			die(err)
		}

		// apply any command line overrides to the options set
		opts.WindowsUpdates = !c.Bool("skipwindowsupdates")
		opts.Headless = !c.Bool("gui")
		if len(c.String("productkey")) > 0 {
			opts.ProductKey = c.String("productkey")
		}
		if c.Bool("ssh") {
			opts.Communicator = "ssh"
		}

		outDir := config.OutDir
		if len(c.String("outdir")) > 0 {
			outDir = c.String("outdir")
		}
		outDir, err = filepath.Abs(outDir)
		if err != nil {
			die(err)
		}

		// find all templates
		cwd, err := os.Getwd()
		if err != nil {
			die(err)
		}
		templates := tpl.New(cwd, osname)

		// finally render all the templates to the output directory
		renderer := renderer.New(opts, outDir)
		err = renderer.Render(templates)
		if err != nil {
			die(err)
		}
	}
	return app
}

func die(vals ...interface{}) {
	if len(vals) > 1 || vals[0] != nil {
		fmt.Fprintf(os.Stderr, fmt.Sprintln(vals...))
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"os"
	"path/filepath"
)

var pluginCachePath = filepath.Join(AppDir(), "plugin-cache.json")

// UpdatePluginCache updates all the plugins in ~/.heroku/plugin-cache.json
func UpdatePluginCache() {
	cache := FetchPluginCache()
	for _, plugin := range cache {
		cache[plugin.Name] = getPlugin(plugin.Name, false)
	}
	savePluginCache(cache)
}

// AddPluginsToCache adds/updates a set of plugins to ~/.heroku/plugin-cache.json
func AddPluginsToCache(plugins ...*Plugin) {
	cache := FetchPluginCache()
	for _, plugin := range plugins {
		cache[plugin.Name] = plugin
	}
	savePluginCache(cache)
}

func savePluginCache(cache map[string]*Plugin) {
	f, err := os.Create(pluginCachePath)
	if err != nil {
		panic(err)
	}
	if err := json.NewEncoder(f).Encode(cache); err != nil {
		panic(err)
	}
}

// FetchPluginCache returns the plugins from the cache
func FetchPluginCache() map[string]*Plugin {
	plugins := make(map[string]*Plugin, 100)
	if exists, _ := fileExists(pluginCachePath); !exists {
		return plugins
	}
	f, err := os.Open(pluginCachePath)
	if err != nil {
		return plugins
	}
	if err := json.NewDecoder(f).Decode(&plugins); err != nil {
		panic(err)
	}
	return plugins
}
<commit_msg>only add real plugins to cache<commit_after>package main

import (
	"encoding/json"
	"os"
	"path/filepath"
)

var pluginCachePath = filepath.Join(AppDir(), "plugin-cache.json")

// UpdatePluginCache updates all the plugins in ~/.heroku/plugin-cache.json
func UpdatePluginCache() {
	cache := FetchPluginCache()
	for _, plugin := range cache {
		cache[plugin.Name] = getPlugin(plugin.Name, false)
	}
	savePluginCache(cache)
}

// AddPluginsToCache adds/updates a set of plugins to ~/.heroku/plugin-cache.json
func AddPluginsToCache(plugins ...*Plugin) {
	cache := FetchPluginCache()
	for _, plugin := range plugins {
		if plugin != nil {
			cache[plugin.Name] = plugin
		}
	}
	savePluginCache(cache)
}

func savePluginCache(cache map[string]*Plugin) {
	f, err := os.Create(pluginCachePath)
	if err != nil {
		panic(err)
	}
	if err := json.NewEncoder(f).Encode(cache); err != nil {
		panic(err)
	}
}

// FetchPluginCache returns the plugins from the cache
func FetchPluginCache() map[string]*Plugin {
	plugins := make(map[string]*Plugin, 100)
	if exists, _ := fileExists(pluginCachePath); !exists {
		return plugins
	}
	f, err := os.Open(pluginCachePath)
	if err != nil {
		return plugins
	}
	if err := json.NewDecoder(f).Decode(&plugins); err != nil {
		panic(err)
	}
	return plugins
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"http"
	"net"
	"os"

	"junta/paxos"
	"junta/mon"
	"junta/store"
	"junta/util"
	"junta/client"
	"junta/server"
	"junta/web"
)

const (
	alpha = 50
)

// Flags
var (
	listenAddr *string = flag.String("l", "", "The address to bind to. Must correspond to a single public interface.")
	publishAddr *string = flag.String("p", "", "Address to publish in junta for client connections.")
	attachAddr *string = flag.String("a", "", "The address of another node to attach to.")
	webAddr *string = flag.String("w", "", "Serve web requests on this address.")
)

func activate(st *store.Store, self, prefix string, c *client.Client) {
	logger := util.NewLogger("activate")
	ch := make(chan store.Event)
	st.Watch("/junta/slot/*", ch)
	for ev := range ch {
		// TODO ev.IsEmpty()
		if ev.IsSet() && ev.Body == "" {
			_, err := c.Set(prefix+ev.Path, self, ev.Cas)
			if err == nil {
				return
			}
			logger.Log(err)
		}
	}
}

func Usage() {
	fmt.Fprintf(os.Stderr, "Usage: %s [OPTIONS] <cluster-name>\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "\nOptions:\n")
	flag.PrintDefaults()
}

func main() {
	util.LogWriter = os.Stderr
	logger := util.NewLogger("main")

	flag.Parse()
	flag.Usage = Usage

	if len(flag.Args()) < 1 {
		logger.Log("require a cluster name")
		flag.Usage()
		os.Exit(1)
	}

	clusterName := flag.Arg(0)
	prefix := "/j/" + clusterName

	if *listenAddr == "" {
		logger.Log("require a listen address")
		flag.Usage()
		os.Exit(1)
	}

	if *publishAddr == "" {
		*publishAddr = *listenAddr
	}

	var webListener net.Listener
	if *webAddr != "" {
		wl, err := net.Listen("tcp", *webAddr)
		if err != nil {
			panic(err)
		}
		webListener = wl
	}

	listener, err := net.Listen("tcp", *listenAddr)
	if err != nil {
		panic(err)
	}

	outs := make(paxos.ChanPutCloserTo)

	var cl *client.Client
	self := util.RandId()
	st := store.New()
	seqn := uint64(0)
	if *attachAddr == "" { // we are the only node in a new cluster
		seqn = addPublicAddr(st, seqn + 1, self, *publishAddr)
		seqn = addHostname(st, seqn + 1, self, os.Getenv("HOSTNAME"))
		seqn = addMember(st, seqn + 1, self, *listenAddr)
		seqn = claimSlot(st, seqn + 1, "1", self)
		seqn = claimLeader(st, seqn + 1, self)
		seqn = claimSlot(st, seqn + 1, "2", "")
		seqn = claimSlot(st, seqn + 1, "3", "")
		seqn = claimSlot(st, seqn + 1, "4", "")
		seqn = claimSlot(st, seqn + 1, "5", "")

		cl, err = client.Dial(*listenAddr)
		if err != nil {
			panic(err)
		}
	} else {
		cl, err = client.Dial(*attachAddr)
		if err != nil {
			panic(err)
		}

		path := prefix + "/junta/info/"+ self +"/public-addr"
		_, err = cl.Set(path, *publishAddr, store.Clobber)
		if err != nil {
			panic(err)
		}

		path = prefix + "/junta/info/"+ self +"/hostname"
		_, err = cl.Set(path, os.Getenv("HOSTNAME"), store.Clobber)
		if err != nil {
			panic(err)
		}

		var snap string
		seqn, snap, err = cl.Join(self, *listenAddr)
		if err != nil {
			panic(err)
		}

		ch := make(chan store.Event)
		st.Wait(seqn + alpha, ch)
		st.Apply(1, snap)

		go func() {
			<-ch
			activate(st, self, prefix, cl)
		}()

		// TODO sink needs a way to pick up missing values if there are any
		// gaps in its sequence
	}
	mg := paxos.NewManager(self, seqn, alpha, st, outs)

	if *attachAddr == "" {
		// Skip ahead alpha steps so that the registrar can provide a
		// meaningful cluster.
		for i := seqn + 1; i < seqn + alpha; i++ {
			go st.Apply(i, store.Nop)
		}
	}

	sv := &server.Server{*listenAddr, st, mg, self, prefix}

	go func() {
		panic(mon.Monitor(self, prefix, st, cl))
	}()

	go func() {
		panic(sv.Serve(listener))
	}()

	go func() {
		panic(sv.ListenAndServeUdp(outs))
	}()

	if webListener != nil {
		web.Store = st
		web.MainInfo.ClusterName = clusterName
		// http handlers are installed in the init function of junta/web.
		go http.Serve(webListener, nil)
	}

	for {
		st.Apply(mg.Recv())
	}
}

func addPublicAddr(st *store.Store, seqn uint64, self, addr string) uint64 {
	// TODO pull out path as a const
	path := "/junta/info/"+ self +"/public-addr"
	mx, err := store.EncodeSet(path, addr, store.Missing)
	if err != nil {
		panic(err)
	}
	st.Apply(seqn, mx)
	return seqn
}

func addHostname(st *store.Store, seqn uint64, self, addr string) uint64 {
	// TODO pull out path as a const
	path := "/junta/info/"+ self +"/hostname"
	mx, err := store.EncodeSet(path, addr, store.Missing)
	if err != nil {
		panic(err)
	}
	st.Apply(seqn, mx)
	return seqn
}

func addMember(st *store.Store, seqn uint64, self, addr string) uint64 {
	// TODO pull out path as a const
	mx, err := store.EncodeSet("/junta/members/"+self, addr, store.Missing)
	if err != nil {
		panic(err)
	}
	st.Apply(seqn, mx)
	return seqn
}

func claimSlot(st *store.Store, seqn uint64, slot, self string) uint64 {
	// TODO pull out path as a const
	mx, err := store.EncodeSet("/junta/slot/"+slot, self, store.Missing)
	if err != nil {
		panic(err)
	}
	st.Apply(seqn, mx)
	return seqn
}

func claimLeader(st *store.Store, seqn uint64, self string) uint64 {
	// TODO pull out path as a const
	mx, err := store.EncodeSet("/junta/leader", self, store.Missing)
	if err != nil {
		panic(err)
	}
	st.Apply(seqn, mx)
	return seqn
}
<commit_msg>for health checks<commit_after>package main

import (
	"flag"
	"fmt"
	"http"
	"net"
	"os"

	"junta/paxos"
	"junta/mon"
	"junta/store"
	"junta/util"
	"junta/client"
	"junta/server"
	"junta/web"
)

const (
	alpha = 50
)

// Flags
var (
	listenAddr *string = flag.String("l", "", "The address to bind to. Must correspond to a single public interface.")
	publishAddr *string = flag.String("p", "", "Address to publish in junta for client connections.")
	attachAddr *string = flag.String("a", "", "The address of another node to attach to.")
	webAddr *string = flag.String("w", "", "Serve web requests on this address.")
)

func activate(st *store.Store, self, prefix string, c *client.Client) {
	logger := util.NewLogger("activate")
	ch := make(chan store.Event)
	st.Watch("/junta/slot/*", ch)
	for ev := range ch {
		// TODO ev.IsEmpty()
		if ev.IsSet() && ev.Body == "" {
			_, err := c.Set(prefix+ev.Path, self, ev.Cas)
			if err == nil {
				return
			}
			logger.Log(err)
		}
	}
}

func Usage() {
	fmt.Fprintf(os.Stderr, "Usage: %s [OPTIONS] <cluster-name>\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "\nOptions:\n")
	flag.PrintDefaults()
}

func main() {
	util.LogWriter = os.Stderr
	logger := util.NewLogger("main")

	flag.Parse()
	flag.Usage = Usage

	if len(flag.Args()) < 1 {
		logger.Log("require a cluster name")
		flag.Usage()
		os.Exit(1)
	}

	clusterName := flag.Arg(0)
	prefix := "/j/" + clusterName

	if *listenAddr == "" {
		logger.Log("require a listen address")
		flag.Usage()
		os.Exit(1)
	}

	if *publishAddr == "" {
		*publishAddr = *listenAddr
	}

	var webListener net.Listener
	if *webAddr != "" {
		wl, err := net.Listen("tcp", *webAddr)
		if err != nil {
			panic(err)
		}
		webListener = wl
	}

	listener, err := net.Listen("tcp", *listenAddr)
	if err != nil {
		panic(err)
	}

	outs := make(paxos.ChanPutCloserTo)

	var cl *client.Client
	self := util.RandId()
	st := store.New()
	seqn := uint64(0)
	if *attachAddr == "" { // we are the only node in a new cluster
		seqn = addPublicAddr(st, seqn + 1, self, *publishAddr)
		seqn = addHostname(st, seqn + 1, self, os.Getenv("HOSTNAME"))
		seqn = addMember(st, seqn + 1, self, *listenAddr)
		seqn = claimSlot(st, seqn + 1, "1", self)
		seqn = claimLeader(st, seqn + 1, self)
		seqn = claimSlot(st, seqn + 1, "2", "")
		seqn = claimSlot(st, seqn + 1, "3", "")
		seqn = claimSlot(st, seqn + 1, "4", "")
		seqn = claimSlot(st, seqn + 1, "5", "")
		seqn = addPing(st, seqn + 1, "pong")

		cl, err = client.Dial(*listenAddr)
		if err != nil {
			panic(err)
		}
	} else {
		cl, err = client.Dial(*attachAddr)
		if err != nil {
			panic(err)
		}

		path := prefix + "/junta/info/"+ self +"/public-addr"
		_, err = cl.Set(path, *publishAddr, store.Clobber)
		if err != nil {
			panic(err)
		}

		path = prefix + "/junta/info/"+ self +"/hostname"
		_, err = cl.Set(path, os.Getenv("HOSTNAME"), store.Clobber)
		if err != nil {
			panic(err)
		}

		var snap string
		seqn, snap, err = cl.Join(self, *listenAddr)
		if err != nil {
			panic(err)
		}

		ch := make(chan store.Event)
		st.Wait(seqn + alpha, ch)
		st.Apply(1, snap)

		go func() {
			<-ch
			activate(st, self, prefix, cl)
		}()

		// TODO sink needs a way to pick up missing values if there are any
		// gaps in its sequence
	}
	mg := paxos.NewManager(self, seqn, alpha, st, outs)

	if *attachAddr == "" {
		// Skip ahead alpha steps so that the registrar can provide a
		// meaningful cluster.
		for i := seqn + 1; i < seqn + alpha; i++ {
			go st.Apply(i, store.Nop)
		}
	}

	sv := &server.Server{*listenAddr, st, mg, self, prefix}

	go func() {
		panic(mon.Monitor(self, prefix, st, cl))
	}()

	go func() {
		panic(sv.Serve(listener))
	}()

	go func() {
		panic(sv.ListenAndServeUdp(outs))
	}()

	if webListener != nil {
		web.Store = st
		web.MainInfo.ClusterName = clusterName
		// http handlers are installed in the init function of junta/web.
		go http.Serve(webListener, nil)
	}

	for {
		st.Apply(mg.Recv())
	}
}

func addPublicAddr(st *store.Store, seqn uint64, self, addr string) uint64 {
	// TODO pull out path as a const
	path := "/junta/info/"+ self +"/public-addr"
	mx, err := store.EncodeSet(path, addr, store.Missing)
	if err != nil {
		panic(err)
	}
	st.Apply(seqn, mx)
	return seqn
}

func addHostname(st *store.Store, seqn uint64, self, addr string) uint64 {
	// TODO pull out path as a const
	path := "/junta/info/"+ self +"/hostname"
	mx, err := store.EncodeSet(path, addr, store.Missing)
	if err != nil {
		panic(err)
	}
	st.Apply(seqn, mx)
	return seqn
}

func addMember(st *store.Store, seqn uint64, self, addr string) uint64 {
	// TODO pull out path as a const
	mx, err := store.EncodeSet("/junta/members/"+self, addr, store.Missing)
	if err != nil {
		panic(err)
	}
	st.Apply(seqn, mx)
	return seqn
}

func claimSlot(st *store.Store, seqn uint64, slot, self string) uint64 {
	// TODO pull out path as a const
	mx, err := store.EncodeSet("/junta/slot/"+slot, self, store.Missing)
	if err != nil {
		panic(err)
	}
	st.Apply(seqn, mx)
	return seqn
}

func claimLeader(st *store.Store, seqn uint64, self string) uint64 {
	// TODO pull out path as a const
	mx, err := store.EncodeSet("/junta/leader", self, store.Missing)
	if err != nil {
		panic(err)
	}
	st.Apply(seqn, mx)
	return seqn
}

func addPing(st *store.Store, seqn uint64, v string) uint64 {
	// TODO pull out path as a const
	mx, err := store.EncodeSet("/ping", v, store.Missing)
	if err != nil {
		panic(err)
	}
	st.Apply(seqn, mx)
	return seqn
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"strings"

	"github.com/digitalocean/logtalez"
)

func main() {

	endpointsPtr := flag.String("endpoints", "", "comma delimited list of zeromq endpoints")
	topicsPtr := flag.String("topics", "", "comma delimited list of topics to subscribe to")
	serverCertPathPtr := flag.String("servercertpath", "", "path to server public cert")
	clientCertPathPtr := flag.String("clientcertpath", "", "path to client public cert")
	jSONPtr := flag.Bool("json", false, "restrict output to valid JSON")

	flag.Parse()

	if *endpointsPtr == "" {
		log.Fatal("--endpoints is mandatory")
	}

	if *serverCertPathPtr == "" {
		log.Fatal("--servercertpath is mandatory")
	}

	if *clientCertPathPtr == "" {
		log.Fatal("--clientcertpath is mandatory")
	}

	if _, err := os.Stat(*serverCertPathPtr); err != nil {
		log.Fatalf("error reading server certificate %q: %s", *serverCertPathPtr, err)
	}

	if _, err := os.Stat(*clientCertPathPtr); err != nil {
		log.Fatalf("error reading client certificate %q: %s", *clientCertPathPtr, err)
	}

	topicList := logtalez.MakeList(*topicPtr)
	endpointList := logtalez.MakeList(*endpointsPtr)

	lt, err := logtalez.New(endpointList, topicList, *serverCertPathPtr, *clientCertPathPtr)
	if err != nil {
		log.Fatal(err)
	}
	defer lt.Destroy()

	buf := make([]byte, 65536)
	for {
		n, err := lt.Read(buf)
		if err != nil && err != io.EOF {
			panic(err)
		}

		output := string(buf[:n])
		if *jSONPtr {
			output = strings.SplitAfterN(output, "@cee:", 2)[1]
		}

		fmt.Println(output)
	}
}
<commit_msg>modify main to work with library changes<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"strings"

	"github.com/digitalocean/logtalez"
)

func main() {

	endpointsPtr := flag.String("endpoints", "", "comma delimited list of zeromq endpoints")
	topicsPtr := flag.String("topics", "", "comma delimited list of topics to subscribe to")
	serverCertPathPtr := flag.String("servercertpath", "", "path to server public cert")
	clientCertPathPtr := flag.String("clientcertpath", "", "path to client public cert")
	jSONPtr := flag.Bool("json", false, "restrict output to valid JSON")

	flag.Parse()

	if *endpointsPtr == "" {
		log.Fatal("--endpoints is mandatory")
	}

	if *serverCertPathPtr == "" {
		log.Fatal("--servercertpath is mandatory")
	}

	if *clientCertPathPtr == "" {
		log.Fatal("--clientcertpath is mandatory")
	}

	if _, err := os.Stat(*serverCertPathPtr); err != nil {
		log.Fatalf("error reading server certificate %q: %s", *serverCertPathPtr, err)
	}

	if _, err := os.Stat(*clientCertPathPtr); err != nil {
		log.Fatalf("error reading client certificate %q: %s", *clientCertPathPtr, err)
	}

	topicList := make([]string, 0)
	for _, t := range strings.Split(*topicsPtr, ",") {
		topicList = append(topicList, t)
	}

	endpointList := make([]string, 0)
	for _, e := range strings.Split(*endpointsPtr, ",") {
		endpointList = append(endpointList, e)
	}

	lt, err := logtalez.New(endpointList, topicList, *serverCertPathPtr, *clientCertPathPtr)
	if err != nil {
		log.Fatal(err)
	}
	defer lt.Destroy()

	buf := make([]byte, 65536)
	for {
		n, err := lt.Read(buf)
		if err != nil && err != io.EOF {
			panic(err)
		}

		output := string(buf[:n])
		if *jSONPtr {
			output = strings.SplitAfterN(output, "@cee:", 2)[1]
		}

		fmt.Println(output)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"text/template"
	"time"

	"github.com/client9/misspell"
)

var (
	defaultWrite *template.Template
	defaultRead  *template.Template

	stdout *log.Logger
	debug  *log.Logger
)

const (
	defaultWriteTmpl = `{{ .Filename }}:{{ .Line }}:{{ .Column }}:corrected "{{ js .Original }}" to "{{ js .Corrected }}"`
	defaultReadTmpl  = `{{ .Filename }}:{{ .Line }}:{{ .Column }}:"{{ js .Original }}" is a misspelling of "{{ js .Corrected }}"`
	csvTmpl          = `{{ printf "%q" .Filename }},{{ .Line }},{{ .Column }},{{ .Original }},{{ .Corrected }}`
	csvHeader        = `file,line,column,typo,corrected`
	sqliteTmpl       = `INSERT INTO misspell VALUES({{ printf "%q" .Filename }},{{ .Line }},{{ .Column }},{{ printf "%q" .Original }},{{ printf "%q" .Corrected }});`
	sqliteHeader     = `PRAGMA foreign_keys=OFF;
BEGIN TRANSACTION;
CREATE TABLE misspell(
	"file" TEXT, "line" INTEGER, "column" INTEGER, "typo" TEXT, "corrected" TEXT
);`
	sqliteFooter = "COMMIT;"
)

func worker(writeit bool, r *misspell.Replacer, mode string, files <-chan string, results chan<- int) {
	count := 0
	for filename := range files {
		orig, err := misspell.ReadTextFile(filename)
		if err != nil {
			log.Println(err)
			continue
		}
		if len(orig) == 0 {
			continue
		}

		debug.Printf("Processing %s", filename)

		updated, changes := r.Replace(orig)

		if len(changes) == 0 {
			continue
		}
		count += len(changes)
		for _, diff := range changes {
			// add in filename
			diff.Filename = filename

			// output can be done by doing multiple goroutines
			// and can clobber os.Stdout.
			//
			// the log package can be used simultaneously from multiple goroutines
			var output bytes.Buffer
			if writeit {
				defaultWrite.Execute(&output, diff)
			} else {
				defaultRead.Execute(&output, diff)
			}

			// goroutine-safe print to os.Stdout
			stdout.Println(output.String())
		}

		if writeit {
			ioutil.WriteFile(filename, []byte(updated), 0)
		}
	}
	results <- count
}

func main() {
	t := time.Now()
	var (
		workers   = flag.Int("j", 0, "Number of workers, 0 = number of CPUs")
		writeit   = flag.Bool("w", false, "Overwrite file with corrections (default is just to display)")
		quietFlag = flag.Bool("q", false, "Do not emit misspelling output")
		outFlag   = flag.String("o", "stdout", "output file or [stderr|stdout|]")
		format    = flag.String("f", "", "'csv', 'sqlite3' or custom Golang template for output")
		ignores   = flag.String("i", "", "ignore the following corrections, comma separated")
		locale    = flag.String("locale", "", "Correct spellings using locale perferances for US or UK.  Default is to use a neutral variety of English.  Setting locale to US will correct the British spelling of 'colour' to 'color'")
		mode      = flag.String("source", "auto", "Source mode: auto=guess, go=golang source, text=plain or markdown-like text")
		debugFlag = flag.Bool("debug", false, "Debug matching, very slow")
		exitError = flag.Bool("error", false, "Exit with 2 if misspelling found")
	)
	flag.Parse()

	if *debugFlag {
		debug = log.New(os.Stderr, "DEBUG ", 0)
	} else {
		debug = log.New(ioutil.Discard, "", 0)
	}

	r := misspell.Replacer{
		Replacements: misspell.DictMain,
		Debug:        *debugFlag,
	}
	//
	// Figure out regional variations
	//
	switch strings.ToUpper(*locale) {
	case "":
		// nothing
	case "US":
		r.AddRuleList(misspell.DictAmerican)
	case "UK", "GB":
		r.AddRuleList(misspell.DictBritish)
	case "NZ", "AU", "CA":
		log.Fatalf("Help wanted.  https://github.com/client9/misspell/issues/6")
	default:
		log.Fatalf("Unknow locale: %q", *locale)
	}

	//
	// Stuff to ignore
	//
	if len(*ignores) > 0 {
		r.RemoveRule(strings.Split(*ignores, ","))
	}

	//
	// Source input mode
	//
	switch *mode {
	case "auto":
	case "go":
	case "text":
	default:
		log.Fatalf("Mode must be one of auto=guess, go=golang source, text=plain or markdown-like text")
	}

	//
	// Custom output
	//
	switch {
	case *format == "csv":
		tmpl := template.Must(template.New("csv").Parse(csvTmpl))
		defaultWrite = tmpl
		defaultRead = tmpl
		stdout.Println(csvHeader)
	case *format == "sqlite" || *format == "sqlite3":
		tmpl := template.Must(template.New("sqlite3").Parse(sqliteTmpl))
		defaultWrite = tmpl
		defaultRead = tmpl
		stdout.Println(sqliteHeader)
	case len(*format) > 0:
		t, err := template.New("custom").Parse(*format)
		if err != nil {
			log.Fatalf("Unable to compile log format: %s", err)
		}
		defaultWrite = t
		defaultRead = t
	default: // format == ""
		defaultWrite = template.Must(template.New("defaultWrite").Parse(defaultWriteTmpl))
		defaultRead = template.Must(template.New("defaultRead").Parse(defaultReadTmpl))
	}

	// we cant't just write to os.Stdout directly since we have multiple goroutine
	// all writing at the same time causing broken output.  Log is routine safe.
	// we see it so it doesn't use a prefix or include a time stamp.
	switch {
	case *quietFlag || *outFlag == "/dev/null":
		stdout = log.New(ioutil.Discard, "", 0)
	case *outFlag == "/dev/stderr" || *outFlag == "stderr":
		stdout = log.New(os.Stderr, "", 0)
	case *outFlag == "/dev/stdout" || *outFlag == "stdout":
		stdout = log.New(os.Stdout, "", 0)
	case *outFlag == "" || *outFlag == "-":
		stdout = log.New(os.Stdout, "", 0)
	default:
		fo, err := os.Create(*outFlag)
		if err != nil {
			log.Fatalf("unable to create outfile %q: %s", *outFlag, err)
		}
		defer fo.Close()
		stdout = log.New(fo, "", 0)
	}

	//
	// Number of Workers / CPU to use
	//
	if *workers < 0 {
		log.Fatalf("-j must >= 0")
	}
	if *workers == 0 {
		*workers = runtime.NumCPU()
	}
	if *debugFlag {
		*workers = 1
	}

	//
	// Done with Flags.
	//  Compile the Replacer and process files
	//
	r.Compile()

	args := flag.Args()
	debug.Printf("initialization complete in %v", time.Since(t))

	// stdin/stdout
	if len(args) == 0 {
		// if we are working with pipes/stdin/stdout
		// there is no concurrency, so we can directly
		// send data to the writers
		var fileout io.Writer
		var errout io.Writer
		switch *writeit {
		case true:
			// if we ARE writing the corrected stream
			// the the corrected stream goes to stdout
			// and the misspelling errors goes to stderr
			// so we can do something like this:
			// curl something | misspell -w | gzip > afile.gz
			fileout = os.Stdout
			errout = os.Stderr
		case false:
			// if we are not writing out the corrected stream
			// then work just like files.  Misspelling errors
			// are sent to stdout
			fileout = ioutil.Discard
			errout = os.Stdout
		}
		count := 0
		next := func(diff misspell.Diff) {
			count++

			// don't even evaluate the output templates
			if *quietFlag {
				return
			}
			diff.Filename = "stdin"
			if *writeit {
				defaultWrite.Execute(errout, diff)
			} else {
				defaultRead.Execute(errout, diff)
			}
			errout.Write([]byte{'\n'})

		}
		err := r.ReplaceReader(os.Stdin, fileout, next)
		if err != nil {
			os.Exit(1)
		}
		switch *format {
		case "sqlite", "sqlite3":
			fileout.Write([]byte(sqliteFooter))
		}
		if count != 0 && *exitError {
			// error
			os.Exit(2)
		}
		return
	}

	c := make(chan string, 64)
	results := make(chan int, *workers)

	for i := 0; i < *workers; i++ {
		go worker(*writeit, &r, *mode, c, results)
	}

	for _, filename := range args {
		filepath.Walk(filename, func(path string, info os.FileInfo, err error) error {
			if err == nil && !info.IsDir() {
				c <- path
			}
			return nil
		})
	}
	close(c)

	count := 0
	for i := 0; i < *workers; i++ {
		changed := <-results
		count += changed
	}

	switch *format {
	case "sqlite", "sqlite3":
		stdout.Println(sqliteFooter)
	}

	if count != 0 && *exitError {
		os.Exit(2)
	}
}
<commit_msg>no need to json escape log lines<commit_after>package main

import (
	"bytes"
	"flag"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"text/template"
	"time"

	"github.com/client9/misspell"
)

var (
	defaultWrite *template.Template
	defaultRead  *template.Template

	stdout *log.Logger
	debug  *log.Logger
)

const (
	defaultWriteTmpl = `{{ .Filename }}:{{ .Line }}:{{ .Column }}:corrected "{{ .Original }}" to "{{ .Corrected }}"`
	defaultReadTmpl  = `{{ .Filename }}:{{ .Line }}:{{ .Column }}:"{{ .Original }}" is a misspelling of "{{ .Corrected }}"`
	csvTmpl          = `{{ printf "%q" .Filename }},{{ .Line }},{{ .Column }},{{ .Original }},{{ .Corrected }}`
	csvHeader        = `file,line,column,typo,corrected`
	sqliteTmpl       = `INSERT INTO misspell VALUES({{ printf "%q" .Filename }},{{ .Line }},{{ .Column }},{{ printf "%q" .Original }},{{ printf "%q" .Corrected }});`
	sqliteHeader     = `PRAGMA foreign_keys=OFF;
BEGIN TRANSACTION;
CREATE TABLE misspell(
	"file" TEXT, "line" INTEGER, "column" INTEGER, "typo" TEXT, "corrected" TEXT
);`
	sqliteFooter = "COMMIT;"
)

func worker(writeit bool, r *misspell.Replacer, mode string, files <-chan string, results chan<- int) {
	count := 0
	for filename := range files {
		orig, err := misspell.ReadTextFile(filename)
		if err != nil {
			log.Println(err)
			continue
		}
		if len(orig) == 0 {
			continue
		}

		debug.Printf("Processing %s", filename)

		updated, changes := r.Replace(orig)

		if len(changes) == 0 {
			continue
		}
		count += len(changes)
		for _, diff := range changes {
			// add in filename
			diff.Filename = filename

			// output can be done by doing multiple goroutines
			// and can clobber os.Stdout.
			//
			// the log package can be used simultaneously from multiple goroutines
			var output bytes.Buffer
			if writeit {
				defaultWrite.Execute(&output, diff)
			} else {
				defaultRead.Execute(&output, diff)
			}

			// goroutine-safe print to os.Stdout
			stdout.Println(output.String())
		}

		if writeit {
			ioutil.WriteFile(filename, []byte(updated), 0)
		}
	}
	results <- count
}

func main() {
	t := time.Now()
	var (
		workers   = flag.Int("j", 0, "Number of workers, 0 = number of CPUs")
		writeit   = flag.Bool("w", false, "Overwrite file with corrections (default is just to display)")
		quietFlag = flag.Bool("q", false, "Do not emit misspelling output")
		outFlag   = flag.String("o", "stdout", "output file or [stderr|stdout|]")
		format    = flag.String("f", "", "'csv', 'sqlite3' or custom Golang template for output")
		ignores   = flag.String("i", "", "ignore the following corrections, comma separated")
		locale    = flag.String("locale", "", "Correct spellings using locale perferances for US or UK.  Default is to use a neutral variety of English.  Setting locale to US will correct the British spelling of 'colour' to 'color'")
		mode      = flag.String("source", "auto", "Source mode: auto=guess, go=golang source, text=plain or markdown-like text")
		debugFlag = flag.Bool("debug", false, "Debug matching, very slow")
		exitError = flag.Bool("error", false, "Exit with 2 if misspelling found")
	)
	flag.Parse()

	if *debugFlag {
		debug = log.New(os.Stderr, "DEBUG ", 0)
	} else {
		debug = log.New(ioutil.Discard, "", 0)
	}

	r := misspell.Replacer{
		Replacements: misspell.DictMain,
		Debug:        *debugFlag,
	}
	//
	// Figure out regional variations
	//
	switch strings.ToUpper(*locale) {
	case "":
		// nothing
	case "US":
		r.AddRuleList(misspell.DictAmerican)
	case "UK", "GB":
		r.AddRuleList(misspell.DictBritish)
	case "NZ", "AU", "CA":
		log.Fatalf("Help wanted.  https://github.com/client9/misspell/issues/6")
	default:
		log.Fatalf("Unknow locale: %q", *locale)
	}

	//
	// Stuff to ignore
	//
	if len(*ignores) > 0 {
		r.RemoveRule(strings.Split(*ignores, ","))
	}

	//
	// Source input mode
	//
	switch *mode {
	case "auto":
	case "go":
	case "text":
	default:
		log.Fatalf("Mode must be one of auto=guess, go=golang source, text=plain or markdown-like text")
	}

	//
	// Custom output
	//
	switch {
	case *format == "csv":
		tmpl := template.Must(template.New("csv").Parse(csvTmpl))
		defaultWrite = tmpl
		defaultRead = tmpl
		stdout.Println(csvHeader)
	case *format == "sqlite" || *format == "sqlite3":
		tmpl := template.Must(template.New("sqlite3").Parse(sqliteTmpl))
		defaultWrite = tmpl
		defaultRead = tmpl
		stdout.Println(sqliteHeader)
	case len(*format) > 0:
		t, err := template.New("custom").Parse(*format)
		if err != nil {
			log.Fatalf("Unable to compile log format: %s", err)
		}
		defaultWrite = t
		defaultRead = t
	default: // format == ""
		defaultWrite = template.Must(template.New("defaultWrite").Parse(defaultWriteTmpl))
		defaultRead = template.Must(template.New("defaultRead").Parse(defaultReadTmpl))
	}

	// we cant't just write to os.Stdout directly since we have multiple goroutine
	// all writing at the same time causing broken output.  Log is routine safe.
	// we see it so it doesn't use a prefix or include a time stamp.
	switch {
	case *quietFlag || *outFlag == "/dev/null":
		stdout = log.New(ioutil.Discard, "", 0)
	case *outFlag == "/dev/stderr" || *outFlag == "stderr":
		stdout = log.New(os.Stderr, "", 0)
	case *outFlag == "/dev/stdout" || *outFlag == "stdout":
		stdout = log.New(os.Stdout, "", 0)
	case *outFlag == "" || *outFlag == "-":
		stdout = log.New(os.Stdout, "", 0)
	default:
		fo, err := os.Create(*outFlag)
		if err != nil {
			log.Fatalf("unable to create outfile %q: %s", *outFlag, err)
		}
		defer fo.Close()
		stdout = log.New(fo, "", 0)
	}

	//
	// Number of Workers / CPU to use
	//
	if *workers < 0 {
		log.Fatalf("-j must >= 0")
	}
	if *workers == 0 {
		*workers = runtime.NumCPU()
	}
	if *debugFlag {
		*workers = 1
	}

	//
	// Done with Flags.
	//  Compile the Replacer and process files
	//
	r.Compile()

	args := flag.Args()
	debug.Printf("initialization complete in %v", time.Since(t))

	// stdin/stdout
	if len(args) == 0 {
		// if we are working with pipes/stdin/stdout
		// there is no concurrency, so we can directly
		// send data to the writers
		var fileout io.Writer
		var errout io.Writer
		switch *writeit {
		case true:
			// if we ARE writing the corrected stream
			// the the corrected stream goes to stdout
			// and the misspelling errors goes to stderr
			// so we can do something like this:
			// curl something | misspell -w | gzip > afile.gz
			fileout = os.Stdout
			errout = os.Stderr
		case false:
			// if we are not writing out the corrected stream
			// then work just like files.  Misspelling errors
			// are sent to stdout
			fileout = ioutil.Discard
			errout = os.Stdout
		}
		count := 0
		next := func(diff misspell.Diff) {
			count++

			// don't even evaluate the output templates
			if *quietFlag {
				return
			}
			diff.Filename = "stdin"
			if *writeit {
				defaultWrite.Execute(errout, diff)
			} else {
				defaultRead.Execute(errout, diff)
			}
			errout.Write([]byte{'\n'})

		}
		err := r.ReplaceReader(os.Stdin, fileout, next)
		if err != nil {
			os.Exit(1)
		}
		switch *format {
		case "sqlite", "sqlite3":
			fileout.Write([]byte(sqliteFooter))
		}
		if count != 0 && *exitError {
			// error
			os.Exit(2)
		}
		return
	}

	c := make(chan string, 64)
	results := make(chan int, *workers)

	for i := 0; i < *workers; i++ {
		go worker(*writeit, &r, *mode, c, results)
	}

	for _, filename := range args {
		filepath.Walk(filename, func(path string, info os.FileInfo, err error) error {
			if err == nil && !info.IsDir() {
				c <- path
			}
			return nil
		})
	}
	close(c)

	count := 0
	for i := 0; i < *workers; i++ {
		changed := <-results
		count += changed
	}

	switch *format {
	case "sqlite", "sqlite3":
		stdout.Println(sqliteFooter)
	}

	if count != 0 && *exitError {
		os.Exit(2)
	}
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"flag"
	"fmt"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/GoogleCloudPlatform/cloud-run-release-operator/internal/metrics"
	"github.com/GoogleCloudPlatform/cloud-run-release-operator/internal/metrics/sheets"
	"github.com/GoogleCloudPlatform/cloud-run-release-operator/internal/stackdriver"
	"github.com/GoogleCloudPlatform/cloud-run-release-operator/pkg/config"
	sdlog "github.com/TV4/logrus-stackdriver-formatter"
	isatty "github.com/mattn/go-isatty"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

type stepFlags []int64

func (steps *stepFlags) Set(step string) error {
	value, err := strconv.ParseInt(step, 10, 64)
	if err != nil {
		return errors.Wrap(err, "failed to parse step")
	}
	*steps = append(*steps, value)
	return nil
}

func (steps stepFlags) String() string {
	var value string
	for _, step := range steps {
		value += " " + strconv.FormatInt(step, 10)
	}

	return value
}

var (
	flLoggingLevel       string
	flCLI                bool
	flCLILoopIntervalSec int
	flHTTPAddr           string
	flProject            string
	flLabelSelector      string

	// Empty array means all regions.
	flRegions       []string
	flRegionsString string

	// Rollout strategy-related flags.
	flSteps              stepFlags
	flStepsString        string
	flHealthOffsetMinute int
	flTimeBeweenRollouts time.Duration
	flMinRequestCount    int
	flErrorRate          float64
	flLatencyP99         float64
	flLatencyP95         float64
	flLatencyP50         float64

	// Metrics provider flags.
	flGoogleSheetsID string
)

func init() {
	defaultAddr := ":8080"
	if v := os.Getenv("PORT"); v != "" {
		defaultAddr = fmt.Sprintf(":%s", v)
	}

	flag.StringVar(&flLoggingLevel, "verbosity", "info", "the logging level (e.g. debug)")
	flag.BoolVar(&flCLI, "cli", false, "run as CLI application to manage rollout in intervals")
	flag.IntVar(&flCLILoopIntervalSec, "cli-run-interval", 60, "the time between each rollout process (in seconds)")
	flag.StringVar(&flHTTPAddr, "http-addr", defaultAddr, "address where to listen to http requests (e.g. :8080)")
	flag.StringVar(&flProject, "project", "", "project in which the service is deployed")
	flag.StringVar(&flLabelSelector, "label", "rollout-strategy=gradual", "filter services based on a label (e.g. team=backend)")
	flag.StringVar(&flRegionsString, "regions", "", "the Cloud Run regions where the services should be looked at")
	flag.Var(&flSteps, "step", "a percentage in traffic the candidate should go through")
	flag.StringVar(&flStepsString, "steps", "5,20,50,80", "define steps in one flag separated by commas (e.g. 5,30,60)")
	flag.IntVar(&flHealthOffsetMinute, "healthcheck-offset", 30, "use metrics from the last N minutes relative to current rollout process")
	flag.DurationVar(&flTimeBeweenRollouts, "min-wait", 30*time.Minute, "minimum time to wait between rollout stages (in minutes), use 0 to disable")
	flag.IntVar(&flMinRequestCount, "min-requests", 100, "expected minimum requests before determining candidate's health")
	flag.Float64Var(&flErrorRate, "max-error-rate", 1.0, "expected max server error rate (in percent)")
	flag.Float64Var(&flLatencyP99, "latency-p99", 0, "expected max latency for 99th percentile of requests (set 0 to ignore)")
	flag.Float64Var(&flLatencyP95, "latency-p95", 0, "expected max latency for 95th percentile of requests (set 0 to ignore)")
	flag.Float64Var(&flLatencyP50, "latency-p50", 0, "expected max latency for 50th percentile of requests (set 0 to ignore)")
	flag.StringVar(&flGoogleSheetsID, "google-sheets", "", "ID of public Google sheets document to use as metrics provider")
	flag.Parse()

	if flRegionsString != "" {
		flRegions = strings.Split(flRegionsString, ",")
	}
}

func main() {
	logger := logrus.New()
	loggingLevel, err := logrus.ParseLevel(flLoggingLevel)
	if err != nil {
		logger.Fatalf("invalid logging level: %v", err)
	}
	logger.SetLevel(loggingLevel)

	if !isatty.IsTerminal(os.Stdout.Fd()) {
		serviceName := os.Getenv("K_SERVICE")
		if serviceName == "" {
			serviceName = "cloud-run-release-operator"
		}
		logger.Formatter = sdlog.NewFormatter(
			sdlog.WithService(serviceName),
		)
	}

	valid, err := flagsAreValid()
	if !valid {
		logger.Fatalf("invalid flags: %v", err)
	}

	// Configuration.
	target := config.NewTarget(flProject, flRegions, flLabelSelector)
	healthCriteria := healthCriteriaFromFlags(flMinRequestCount, flErrorRate, flLatencyP99, flLatencyP95, flLatencyP50)
	printHealthCriteria(logger, healthCriteria)
	strategy := config.NewStrategy(target, flSteps, flHealthOffsetMinute, flTimeBeweenRollouts, healthCriteria)
	cfg := &config.Config{Strategies: []config.Strategy{strategy}}
	if err := cfg.Validate(); err != nil {
		logger.Fatalf("invalid rollout configuration: %v", err)
	}

	ctx := context.Background()
	if flCLI {
		runDaemon(ctx, logger, cfg)
	} else {
		http.HandleFunc("/rollout", makeRolloutHandler(logger, cfg))
		logger.WithField("addr", flHTTPAddr).Infof("starting server")
		logger.Fatal(http.ListenAndServe(flHTTPAddr, nil))
	}
}

func runDaemon(ctx context.Context, logger *logrus.Logger, cfg *config.Config) {
	for {
		// TODO(gvso): Handle all the strategies.
		errs := runRollouts(ctx, logger, cfg.Strategies[0])
		errsStr := rolloutErrsToString(errs)
		if len(errs) != 0 {
			logger.Warnf("there were %d errors: \n%s", len(errs), errsStr)
		}

		duration := time.Duration(flCLILoopIntervalSec)
		time.Sleep(duration * time.Second)
	}
}

func flagsAreValid() (bool, error) {
	// -steps flag has precedence over the list of -step flags.
	if flStepsString != "" {
		steps := strings.Split(flStepsString, ",")
		flSteps = []int64{}
		for _, step := range steps {
			value, err := strconv.ParseInt(step, 10, 64)
			if err != nil {
				return false, errors.Wrap(err, "invalid step value")
			}

			flSteps = append(flSteps, value)
		}
	}

	for _, region := range flRegions {
		if region == "" {
			return false, errors.New("region cannot be empty")
		}
	}

	return true, nil
}

// chooseMetricsProvider checks the CLI flags and determine which metrics
// provider should be used for the rollout.
func chooseMetricsProvider(ctx context.Context, logger *logrus.Entry, project, region, svcName string) (metrics.Provider, error) {
	if flGoogleSheetsID != "" {
		logger.Debug("using Google Sheets as metrics provider")
		return sheets.NewProvider(ctx, flGoogleSheetsID, "", region, svcName)
	}
	logger.Debug("using Cloud Monitoring (Stackdriver) as metrics provider")
	return stackdriver.NewProvider(ctx, project, region, svcName)
}

// healthCriteriaFromFlags checks the metrics-related flags and return an array
// of config.Metric based on them.
func healthCriteriaFromFlags(requestCount int, errorRate, latencyP99, latencyP95, latencyP50 float64) []config.HealthCriterion {
	metrics := []config.HealthCriterion{
		{Metric: config.ErrorRateMetricsCheck, Threshold: errorRate},
		{Metric: config.RequestCountMetricsCheck, Threshold: float64(requestCount)},
	}

	if latencyP99 > 0 {
		metrics = append(metrics, config.HealthCriterion{Metric: config.LatencyMetricsCheck, Percentile: 99, Threshold: latencyP99})
	}
	if latencyP95 > 0 {
		metrics = append(metrics, config.HealthCriterion{Metric: config.LatencyMetricsCheck, Percentile: 95, Threshold: latencyP95})
	}
	if latencyP50 > 0 {
		metrics = append(metrics, config.HealthCriterion{Metric: config.LatencyMetricsCheck, Percentile: 50, Threshold: latencyP50})
	}

	return metrics
}

func printHealthCriteria(logger *logrus.Logger, healthCriteria []config.HealthCriterion) {
	for _, criteria := range healthCriteria {
		lg := logger.WithFields(logrus.Fields{
			"metricsType": criteria.Metric,
			"threshold":   criteria.Threshold,
		})

		switch criteria.Metric {
		case config.ErrorRateMetricsCheck:
			lg.Debug("found health criterion")
		case config.LatencyMetricsCheck:
			lg.WithField("percentile", criteria.Percentile).Debug("found health criterion")
		default:
			lg.Debug("invalid health criterion")
		}
	}
}
<commit_msg>main: Print all health criteria (#69)<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"flag"
	"fmt"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/GoogleCloudPlatform/cloud-run-release-operator/internal/metrics"
	"github.com/GoogleCloudPlatform/cloud-run-release-operator/internal/metrics/sheets"
	"github.com/GoogleCloudPlatform/cloud-run-release-operator/internal/stackdriver"
	"github.com/GoogleCloudPlatform/cloud-run-release-operator/pkg/config"
	sdlog "github.com/TV4/logrus-stackdriver-formatter"
	isatty "github.com/mattn/go-isatty"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

type stepFlags []int64

func (steps *stepFlags) Set(step string) error {
	value, err := strconv.ParseInt(step, 10, 64)
	if err != nil {
		return errors.Wrap(err, "failed to parse step")
	}
	*steps = append(*steps, value)
	return nil
}

func (steps stepFlags) String() string {
	var value string
	for _, step := range steps {
		value += " " + strconv.FormatInt(step, 10)
	}

	return value
}

var (
	flLoggingLevel       string
	flCLI                bool
	flCLILoopIntervalSec int
	flHTTPAddr           string
	flProject            string
	flLabelSelector      string

	// Empty array means all regions.
	flRegions       []string
	flRegionsString string

	// Rollout strategy-related flags.
	flSteps              stepFlags
	flStepsString        string
	flHealthOffsetMinute int
	flTimeBeweenRollouts time.Duration
	flMinRequestCount    int
	flErrorRate          float64
	flLatencyP99         float64
	flLatencyP95         float64
	flLatencyP50         float64

	// Metrics provider flags.
	flGoogleSheetsID string
)

func init() {
	defaultAddr := ":8080"
	if v := os.Getenv("PORT"); v != "" {
		defaultAddr = fmt.Sprintf(":%s", v)
	}

	flag.StringVar(&flLoggingLevel, "verbosity", "info", "the logging level (e.g. debug)")
	flag.BoolVar(&flCLI, "cli", false, "run as CLI application to manage rollout in intervals")
	flag.IntVar(&flCLILoopIntervalSec, "cli-run-interval", 60, "the time between each rollout process (in seconds)")
	flag.StringVar(&flHTTPAddr, "http-addr", defaultAddr, "address where to listen to http requests (e.g. :8080)")
	flag.StringVar(&flProject, "project", "", "project in which the service is deployed")
	flag.StringVar(&flLabelSelector, "label", "rollout-strategy=gradual", "filter services based on a label (e.g. team=backend)")
	flag.StringVar(&flRegionsString, "regions", "", "the Cloud Run regions where the services should be looked at")
	flag.Var(&flSteps, "step", "a percentage in traffic the candidate should go through")
	flag.StringVar(&flStepsString, "steps", "5,20,50,80", "define steps in one flag separated by commas (e.g. 5,30,60)")
	flag.IntVar(&flHealthOffsetMinute, "healthcheck-offset", 30, "use metrics from the last N minutes relative to current rollout process")
	flag.DurationVar(&flTimeBeweenRollouts, "min-wait", 30*time.Minute, "minimum time to wait between rollout stages (in minutes), use 0 to disable")
	flag.IntVar(&flMinRequestCount, "min-requests", 100, "expected minimum requests before determining candidate's health")
	flag.Float64Var(&flErrorRate, "max-error-rate", 1.0, "expected max server error rate (in percent)")
	flag.Float64Var(&flLatencyP99, "latency-p99", 0, "expected max latency for 99th percentile of requests (set 0 to ignore)")
	flag.Float64Var(&flLatencyP95, "latency-p95", 0, "expected max latency for 95th percentile of requests (set 0 to ignore)")
	flag.Float64Var(&flLatencyP50, "latency-p50", 0, "expected max latency for 50th percentile of requests (set 0 to ignore)")
	flag.StringVar(&flGoogleSheetsID, "google-sheets", "", "ID of public Google sheets document to use as metrics provider")
	flag.Parse()

	if flRegionsString != "" {
		flRegions = strings.Split(flRegionsString, ",")
	}
}

func main() {
	logger := logrus.New()
	loggingLevel, err := logrus.ParseLevel(flLoggingLevel)
	if err != nil {
		logger.Fatalf("invalid logging level: %v", err)
	}
	logger.SetLevel(loggingLevel)

	if !isatty.IsTerminal(os.Stdout.Fd()) {
		serviceName := os.Getenv("K_SERVICE")
		if serviceName == "" {
			serviceName = "cloud-run-release-operator"
		}
		logger.Formatter = sdlog.NewFormatter(
			sdlog.WithService(serviceName),
		)
	}

	valid, err := flagsAreValid()
	if !valid {
		logger.Fatalf("invalid flags: %v", err)
	}

	// Configuration.
	target := config.NewTarget(flProject, flRegions, flLabelSelector)
	healthCriteria := healthCriteriaFromFlags(flMinRequestCount, flErrorRate, flLatencyP99, flLatencyP95, flLatencyP50)
	printHealthCriteria(logger, healthCriteria)
	strategy := config.NewStrategy(target, flSteps, flHealthOffsetMinute, flTimeBeweenRollouts, healthCriteria)
	cfg := &config.Config{Strategies: []config.Strategy{strategy}}
	if err := cfg.Validate(); err != nil {
		logger.Fatalf("invalid rollout configuration: %v", err)
	}

	ctx := context.Background()
	if flCLI {
		runDaemon(ctx, logger, cfg)
	} else {
		http.HandleFunc("/rollout", makeRolloutHandler(logger, cfg))
		logger.WithField("addr", flHTTPAddr).Infof("starting server")
		logger.Fatal(http.ListenAndServe(flHTTPAddr, nil))
	}
}

func runDaemon(ctx context.Context, logger *logrus.Logger, cfg *config.Config) {
	for {
		// TODO(gvso): Handle all the strategies.
		errs := runRollouts(ctx, logger, cfg.Strategies[0])
		errsStr := rolloutErrsToString(errs)
		if len(errs) != 0 {
			logger.Warnf("there were %d errors: \n%s", len(errs), errsStr)
		}

		duration := time.Duration(flCLILoopIntervalSec)
		time.Sleep(duration * time.Second)
	}
}

func flagsAreValid() (bool, error) {
	// -steps flag has precedence over the list of -step flags.
	if flStepsString != "" {
		steps := strings.Split(flStepsString, ",")
		flSteps = []int64{}
		for _, step := range steps {
			value, err := strconv.ParseInt(step, 10, 64)
			if err != nil {
				return false, errors.Wrap(err, "invalid step value")
			}

			flSteps = append(flSteps, value)
		}
	}

	for _, region := range flRegions {
		if region == "" {
			return false, errors.New("region cannot be empty")
		}
	}

	return true, nil
}

// chooseMetricsProvider checks the CLI flags and determine which metrics
// provider should be used for the rollout.
func chooseMetricsProvider(ctx context.Context, logger *logrus.Entry, project, region, svcName string) (metrics.Provider, error) {
	if flGoogleSheetsID != "" {
		logger.Debug("using Google Sheets as metrics provider")
		return sheets.NewProvider(ctx, flGoogleSheetsID, "", region, svcName)
	}
	logger.Debug("using Cloud Monitoring (Stackdriver) as metrics provider")
	return stackdriver.NewProvider(ctx, project, region, svcName)
}

// healthCriteriaFromFlags checks the metrics-related flags and return an array
// of config.Metric based on them.
func healthCriteriaFromFlags(requestCount int, errorRate, latencyP99, latencyP95, latencyP50 float64) []config.HealthCriterion {
	metrics := []config.HealthCriterion{
		{Metric: config.ErrorRateMetricsCheck, Threshold: errorRate},
		{Metric: config.RequestCountMetricsCheck, Threshold: float64(requestCount)},
	}

	if latencyP99 > 0 {
		metrics = append(metrics, config.HealthCriterion{Metric: config.LatencyMetricsCheck, Percentile: 99, Threshold: latencyP99})
	}
	if latencyP95 > 0 {
		metrics = append(metrics, config.HealthCriterion{Metric: config.LatencyMetricsCheck, Percentile: 95, Threshold: latencyP95})
	}
	if latencyP50 > 0 {
		metrics = append(metrics, config.HealthCriterion{Metric: config.LatencyMetricsCheck, Percentile: 50, Threshold: latencyP50})
	}

	return metrics
}

func printHealthCriteria(logger *logrus.Logger, healthCriteria []config.HealthCriterion) {
	for _, criteria := range healthCriteria {
		lg := logger.WithFields(logrus.Fields{
			"metricsType": criteria.Metric,
			"threshold":   criteria.Threshold,
		})

		if criteria.Metric == config.LatencyMetricsCheck {
			lg = lg.WithField("percentile", criteria.Percentile)
		}
		lg.Debug("found health criterion")
	}
}
<|endoftext|>


<commit_before>package backup

import (
	"fmt"

	log "github.com/Sirupsen/logrus"
	"github.com/spf13/cobra"

	"github.com/camptocamp/bivac/cmd"
	"github.com/camptocamp/bivac/pkg/client"
)

var (
	remoteAddress string
	psk           string
	volumeID      string
)

var envs = make(map[string]string)

var resticCmd = &cobra.Command{
	Use:   "restic --volume [VOLUME_ID] [COMMAND]",
	Short: "Run Restic command on a volume's repository",
	Args:  cobra.ArbitraryArgs,
	PreRun: func(cmd *cobra.Command, args []string) {
		if volumeID == "" {
			log.Fatal("You must provide a volume ID.")
			return
		}
	},
	Run: func(cmd *cobra.Command, args []string) {
		c, err := client.NewClient(remoteAddress, psk)
		if err != nil {
			log.Errorf("failed to create new client: %s", err)
			return
		}

		output, err := c.RunRawCommand(volumeID, args)
		if err != nil {
			log.Errorf("failed to run command: %s", err)
			return
		}

		fmt.Println(output)
	},
}

func init() {
	resticCmd.Flags().StringVarP(&remoteAddress, "remote.address", "", "http://127.0.0.1:8182", "Address of the remote Bivac server.")
	envs["BIVAC_REMOTE_ADDRESS"] = "remote.address"

	resticCmd.Flags().StringVarP(&psk, "server.psk", "", "", "Pre-shared key.")
	envs["BIVAC_SERVER_PSK"] = "server.psk"

	resticCmd.Flags().StringVarP(&volumeID, "volume", "", "", "Volume ID")

	cmd.SetValuesFromEnv(envs, resticCmd.Flags())
	cmd.RootCmd.AddCommand(resticCmd)
}
<commit_msg>improve restic cmd usage msg<commit_after>package backup

import (
	"fmt"

	log "github.com/Sirupsen/logrus"
	"github.com/spf13/cobra"

	"github.com/camptocamp/bivac/cmd"
	"github.com/camptocamp/bivac/pkg/client"
)

var (
	remoteAddress string
	psk           string
	volumeID      string
)

var envs = make(map[string]string)

var resticCmd = &cobra.Command{
	Use:   "restic --volume [VOLUME_ID] -- [RESTIC_COMMAND]",
	Short: "Run Restic command on a volume's repository",
	Args:  cobra.ArbitraryArgs,
	PreRun: func(cmd *cobra.Command, args []string) {
		if volumeID == "" {
			log.Fatal("You must provide a volume ID.")
			return
		}
	},
	Run: func(cmd *cobra.Command, args []string) {
		c, err := client.NewClient(remoteAddress, psk)
		if err != nil {
			log.Errorf("failed to create new client: %s", err)
			return
		}

		output, err := c.RunRawCommand(volumeID, args)
		if err != nil {
			log.Errorf("failed to run command: %s", err)
			return
		}

		fmt.Println(output)
	},
}

func init() {
	resticCmd.Flags().StringVarP(&remoteAddress, "remote.address", "", "http://127.0.0.1:8182", "Address of the remote Bivac server.")
	envs["BIVAC_REMOTE_ADDRESS"] = "remote.address"

	resticCmd.Flags().StringVarP(&psk, "server.psk", "", "", "Pre-shared key.")
	envs["BIVAC_SERVER_PSK"] = "server.psk"

	resticCmd.Flags().StringVarP(&volumeID, "volume", "", "", "Volume ID")

	cmd.SetValuesFromEnv(envs, resticCmd.Flags())
	cmd.RootCmd.AddCommand(resticCmd)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/metakeule/scaffold"
	"io/ioutil"
	"os"
	"path/filepath"
)

func main() {

	var (
		err            error
		workingDir     string
		templateRaw    []byte
		help, template string
	)

steps:
	for jump := 1; err == nil; jump++ {
		switch jump - 1 {
		default:
			break steps
		case 0:
			if len(os.Args) < 2 {
				err = fmt.Errorf("missing template argument")
			}
		case 1:
			workingDir, err = os.Getwd()
		case 2:
			workingDir, err = filepath.Abs(workingDir)
		case 3:
			templateRaw, err = ioutil.ReadFile(os.Args[1])
		case 4:
			help, template = scaffold.SplitTemplate(string(templateRaw))
			if len(os.Args) > 2 && os.Args[2] == "help" {
				fmt.Fprintln(os.Stdout, help)
				os.Exit(0)
			}
		case 5:
			err = scaffold.Run(workingDir, template, os.Stdin, os.Stdout)
		}
	}
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		fmt.Fprintln(os.Stderr, help)
		os.Exit(1)
	}
}
<commit_msg>improve command<commit_after>package main

import (
	"fmt"
	"gopkg.in/metakeule/scaffold.v0"
	"gopkg.in/metakeule/config.v1"
	"io/ioutil"
	"os"
	"path/filepath"
)

var (
	cfg = config.MustNew("scaffold", "1.0",
		`scaffold creates files and directories based on a template and json input.
		Complete documentation at http://godoc.org/github.com/metakeule/scaffold.`)

	templateArg = cfg.NewString("template", "the file where the template resides", config.Required, config.Shortflag('t'))
	dirArg      = cfg.NewString("dir", "directory that is the target/root of the file creations", config.Default("."))
	exampleCmd  = cfg.MustCommand("example", "shows the example section of the given template")
)

func main() {

	var (
		err               error
		dir               string
		templateRaw       []byte
		example, template string
	)

steps:
	for jump := 1; err == nil; jump++ {
		switch jump - 1 {
		default:
			break steps
		case 0:
			err = cfg.Run()
		case 1:
			dir = dirArg.Get()
		case 2:
			dir, err = filepath.Abs(dir)
		case 3:
			templateRaw, err = ioutil.ReadFile(templateArg.Get())
		case 4:
			example, template = scaffold.SplitTemplate(string(templateRaw))
		case 5:
			switch cfg.ActiveCommand() {
			case nil:
				err = scaffold.Run(dir, template, os.Stdin, os.Stdout)
			case exampleCmd:
				fmt.Fprintln(os.Stdout, example)
			}
		}
	}

	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}
<|endoftext|>