<commit_before>// Copyright 2016 DigitalOcean
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"os"
	"runtime"
	"time"

	"github.com/digitalocean/do-agent/bootstrap"
	"github.com/digitalocean/do-agent/collector"
	"github.com/digitalocean/do-agent/config"
	"github.com/digitalocean/do-agent/log"
	"github.com/digitalocean/do-agent/monitoringclient"
	"github.com/digitalocean/do-agent/plugins"
	"github.com/digitalocean/do-agent/procfs"
	"github.com/digitalocean/do-agent/update"

	"github.com/digitalocean/go-metadata"
	"github.com/ianschenck/envflag"
	"github.com/jpillora/backoff"
)

var (
	defaultPluginPath = "/var/lib/do-agent/plugins"

	forceUpdate        = flag.Bool("force_update", false, "Update the version of do-agent.")
	logToSyslog        = flag.Bool("log_syslog", false, "Log to syslog.")
	logLevel           = flag.String("log_level", "INFO", "Log level to log: ERROR, INFO, DEBUG")
	debugAuthURL       = envflag.String("DO_AGENT_AUTHENTICATION_URL", monitoringclient.AuthURL, "Override authentication URL")
	debugAppKey        = envflag.String("DO_AGENT_APPKEY", "", "Override AppKey")
	debugMetricsURL    = envflag.String("DO_AGENT_METRICS_URL", "", "Override metrics URL")
	debugDropletID     = envflag.Int64("DO_AGENT_DROPLET_ID", 0, "Override Droplet ID")
	debugAuthToken     = envflag.String("DO_AGENT_AUTHTOKEN", "", "Override AuthToken")
	debugUpdateURL     = envflag.String("DO_AGENT_UPDATE_URL", update.RepoURL, "Override Update URL")
	debugLocalRepoPath = envflag.String("DO_AGENT_REPO_PATH", update.RepoLocalStore, "Override Local repository path")
	pluginPath         = envflag.String("DO_AGENT_PLUGIN_PATH", defaultPluginPath, "Override plugin path")
)

func main() {
	envflag.Parse()
	flag.Parse()

	if err := log.SetLogger(*logLevel, *logToSyslog); err != nil {
		log.Fatal(err)
	}

	log.Info("Do-Agent version ", config.Version())
	log.Info("Do-Agent build ", config.Build())
	log.Info("Architecture: ", runtime.GOARCH)
	log.Info("Operating System: ", runtime.GOOS)

	if *debugAuthURL != monitoringclient.AuthURL {
		log.Info("Authentication URL Override: ", *debugAuthURL)
	}
	if *debugMetricsURL != "" {
		log.Info("Metrics URL Override: ", *debugMetricsURL)
	}
	if *debugAppKey != "" {
		log.Info("AppKey Override: ", *debugAppKey)
	}
	if *debugDropletID != 0 {
		log.Info("DropletID Override: ", *debugDropletID)
	}
	if *debugAuthToken != "" {
		log.Info("AuthToken Override: ", *debugAuthToken)
	}
	if *debugUpdateURL != update.RepoURL {
		log.Info("Update URL Override: ", debugUpdateURL)
	}
	if *debugLocalRepoPath != update.RepoLocalStore {
		log.Info("Local Repository Path Override: ", *debugLocalRepoPath)
	}
	if *pluginPath != defaultPluginPath {
		log.Info("Plugin path Override: ", *pluginPath)
	}
	updater := update.NewUpdate(*debugLocalRepoPath, *debugUpdateURL)

	if *forceUpdate {
		log.Info("Checking for newer version of do-agent")

		if err := updater.FetchLatest(true); err != nil {
			if err != update.ErrUpdateNotAvailable {
				log.Info(err)
				os.Exit(0)
			}
			log.Info("No update available")
			os.Exit(0)
		}
		log.Info("Updated successfully")
		os.Exit(0)
	}

	metadataClient := metadata.NewClient()
	monitoringClient := monitoringclient.NewClient(*debugAuthURL)

	errorBackoffTimer := backoff.Backoff{
		Min:    500 * time.Millisecond,
		Max:    5 * time.Minute,
		Factor: 2,
		Jitter: true,
	}

	var err error
	var credentials *bootstrap.Credentials
	for {
		credentials, err = bootstrap.InitCredentials(metadataClient, monitoringClient, *debugAppKey, *debugDropletID, *debugAuthToken)
		if err == nil {
			break
		}
		log.Info("Unable to read credentials: ", err)

		if _, err = metadataClient.AuthToken(); err != nil {
			log.Fatal("do-agent requires a DigitalOcean host")
		}
		time.Sleep(errorBackoffTimer.Duration())
	}

	if credentials.AppKey == "" {
		log.Fatal("No Appkey is configured. do-agent requires a DigitalOcean host")
	}

	smc, err := monitoringclient.CreateMetricsClient(credentials.AppKey, credentials.DropletID, credentials.Region, *debugMetricsURL)
	if err != nil {
		log.Fatal("Error creating monitoring client: ", err)
	}

	updateAgent(updater)
	lastUpdate := time.Now()

	r := smc.Registry()
	collector.RegisterCPUMetrics(r, procfs.NewStat)
	collector.RegisterDiskMetrics(r, procfs.NewDisk)
	collector.RegisterFSMetrics(r, procfs.NewMount)
	collector.RegisterLoadMetrics(r, procfs.NewLoad)
	collector.RegisterMemoryMetrics(r, procfs.NewMemory)
	collector.RegisterNetworkMetrics(r, procfs.NewNetwork)
	collector.RegisterNodeMetrics(r, procfs.NewOSRelease)
	collector.RegisterProcessMetrics(r, procfs.NewProcProc)
	plugins.RegisterPluginDir(r, *pluginPath)

	for {
		log.Debug("Transmitting metrics to DigitalOcean.")
		pushInterval, err := smc.SendMetrics()
		if err != nil {
			log.Error("Sending metrics to DigitalOcean: ", err)
		}

		log.Debug(fmt.Sprintf("sleeping for %d seconds", pushInterval))
		time.Sleep(time.Duration(pushInterval) * time.Second)

		if time.Now().After(lastUpdate.Add(1 * time.Hour)) {
			lastUpdate = time.Now()
			updateAgent(updater)
		}
	}
}

func updateAgent(updater update.Updater) {
	log.Info("Checking for newer version of do-agent")

	if err := updater.FetchLatestAndExec(false); err != nil {
		if err == update.ErrUpdateNotAvailable {
			log.Info("No update available")
			return
		}
		log.Error(err)
		return
	}
}
<commit_msg>Remove tuf target expiration errors from the logs.<commit_after>// Copyright 2016 DigitalOcean
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"os"
	"runtime"
	"time"

	"github.com/digitalocean/do-agent/bootstrap"
	"github.com/digitalocean/do-agent/collector"
	"github.com/digitalocean/do-agent/config"
	"github.com/digitalocean/do-agent/log"
	"github.com/digitalocean/do-agent/monitoringclient"
	"github.com/digitalocean/do-agent/plugins"
	"github.com/digitalocean/do-agent/procfs"
	"github.com/digitalocean/do-agent/update"

	"github.com/digitalocean/go-metadata"
	"github.com/ianschenck/envflag"
	"github.com/jpillora/backoff"
)

var (
	defaultPluginPath = "/var/lib/do-agent/plugins"

	forceUpdate        = flag.Bool("force_update", false, "Update the version of do-agent.")
	logToSyslog        = flag.Bool("log_syslog", false, "Log to syslog.")
	logLevel           = flag.String("log_level", "INFO", "Log level to log: ERROR, INFO, DEBUG")
	debugAuthURL       = envflag.String("DO_AGENT_AUTHENTICATION_URL", monitoringclient.AuthURL, "Override authentication URL")
	debugAppKey        = envflag.String("DO_AGENT_APPKEY", "", "Override AppKey")
	debugMetricsURL    = envflag.String("DO_AGENT_METRICS_URL", "", "Override metrics URL")
	debugDropletID     = envflag.Int64("DO_AGENT_DROPLET_ID", 0, "Override Droplet ID")
	debugAuthToken     = envflag.String("DO_AGENT_AUTHTOKEN", "", "Override AuthToken")
	debugUpdateURL     = envflag.String("DO_AGENT_UPDATE_URL", update.RepoURL, "Override Update URL")
	debugLocalRepoPath = envflag.String("DO_AGENT_REPO_PATH", update.RepoLocalStore, "Override Local repository path")
	pluginPath         = envflag.String("DO_AGENT_PLUGIN_PATH", defaultPluginPath, "Override plugin path")
)

func main() {
	envflag.Parse()
	flag.Parse()

	if err := log.SetLogger(*logLevel, *logToSyslog); err != nil {
		log.Fatal(err)
	}

	log.Info("Do-Agent version ", config.Version())
	log.Info("Do-Agent build ", config.Build())
	log.Info("Architecture: ", runtime.GOARCH)
	log.Info("Operating System: ", runtime.GOOS)

	if *debugAuthURL != monitoringclient.AuthURL {
		log.Info("Authentication URL Override: ", *debugAuthURL)
	}
	if *debugMetricsURL != "" {
		log.Info("Metrics URL Override: ", *debugMetricsURL)
	}
	if *debugAppKey != "" {
		log.Info("AppKey Override: ", *debugAppKey)
	}
	if *debugDropletID != 0 {
		log.Info("DropletID Override: ", *debugDropletID)
	}
	if *debugAuthToken != "" {
		log.Info("AuthToken Override: ", *debugAuthToken)
	}
	if *debugUpdateURL != update.RepoURL {
		log.Info("Update URL Override: ", debugUpdateURL)
	}
	if *debugLocalRepoPath != update.RepoLocalStore {
		log.Info("Local Repository Path Override: ", *debugLocalRepoPath)
	}
	if *pluginPath != defaultPluginPath {
		log.Info("Plugin path Override: ", *pluginPath)
	}
	updater := update.NewUpdate(*debugLocalRepoPath, *debugUpdateURL)

	if *forceUpdate {
		updateAgentWithExit(updater)
	}

	metadataClient := metadata.NewClient()
	monitoringClient := monitoringclient.NewClient(*debugAuthURL)

	errorBackoffTimer := backoff.Backoff{
		Min:    500 * time.Millisecond,
		Max:    5 * time.Minute,
		Factor: 2,
		Jitter: true,
	}

	var err error
	var credentials *bootstrap.Credentials
	for {
		credentials, err = bootstrap.InitCredentials(metadataClient, monitoringClient, *debugAppKey, *debugDropletID, *debugAuthToken)
		if err == nil {
			break
		}
		log.Info("Unable to read credentials: ", err)

		if _, err = metadataClient.AuthToken(); err != nil {
			log.Fatal("do-agent requires a DigitalOcean host")
		}
		time.Sleep(errorBackoffTimer.Duration())
	}

	if credentials.AppKey == "" {
		log.Fatal("No Appkey is configured. do-agent requires a DigitalOcean host")
	}

	smc, err := monitoringclient.CreateMetricsClient(credentials.AppKey, credentials.DropletID, credentials.Region, *debugMetricsURL)
	if err != nil {
		log.Fatal("Error creating monitoring client: ", err)
	}

	updateAgentWithRestart(updater)
	lastUpdate := time.Now()

	r := smc.Registry()
	collector.RegisterCPUMetrics(r, procfs.NewStat)
	collector.RegisterDiskMetrics(r, procfs.NewDisk)
	collector.RegisterFSMetrics(r, procfs.NewMount)
	collector.RegisterLoadMetrics(r, procfs.NewLoad)
	collector.RegisterMemoryMetrics(r, procfs.NewMemory)
	collector.RegisterNetworkMetrics(r, procfs.NewNetwork)
	collector.RegisterNodeMetrics(r, procfs.NewOSRelease)
	collector.RegisterProcessMetrics(r, procfs.NewProcProc)
	plugins.RegisterPluginDir(r, *pluginPath)

	for {
		log.Debug("Transmitting metrics to DigitalOcean.")
		pushInterval, err := smc.SendMetrics()
		if err != nil {
			log.Error("Sending metrics to DigitalOcean: ", err)
		}

		log.Debug(fmt.Sprintf("sleeping for %d seconds", pushInterval))
		time.Sleep(time.Duration(pushInterval) * time.Second)

		if time.Now().After(lastUpdate.Add(1 * time.Hour)) {
			lastUpdate = time.Now()
			updateAgentWithRestart(updater)
		}
	}
}

// updateAgentWithRestart looks for any available updates to the agent. If an update is found, it will
// update the agent binary and reinitialize itself. If an update isn't found or fails, it will
// only log the results of its attempt.
func updateAgentWithRestart(updater update.Updater) {
	log.Info("Checking for newer version of do-agent")

	if err := updater.FetchLatestAndExec(false); err != nil {
		if err == update.ErrUpdateNotAvailable {
			log.Info("No update available")
			return
		}

		if err == update.ErrUnableToRetrieveTargets {
			log.Info("No target available for update")
			return
		}

		log.Errorf("Unable to update do-agent: %s\n", err)
	}
}

// updateAgentWithExit looks for any available updates to the agent. After attempting to update
// the agent it will gracefully terminate execution.
func updateAgentWithExit(updater update.Updater) {
	log.Info("Checking for newer version of do-agent")

	if err := updater.FetchLatest(true); err != nil {
		if err == update.ErrUpdateNotAvailable {
			log.Info("No update available")
			os.Exit(0)
		}

		if err == update.ErrUnableToRetrieveTargets {
			log.Info("No target available for update")
			os.Exit(0)
		}

		log.Errorf("Unable to update do-agent: %s\n", err)
		os.Exit(1)
	}

	log.Info("Updated successfully")
	os.Exit(0)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/ha/doozer"
	"os"
	"reflect"
	"sort"
	"strconv"
)

var (
	addr        = flag.String("a", "127.0.0.1:8046", "the address to bind to")
	rrev        = flag.Int64("r", -1, "request rev")
	showHelp    = flag.Bool("h", false, "show help")
	showVersion = flag.Bool("v", false, "print version string")
)

type cmd struct {
	f interface{}
	a string // args
	d string // short description
}

var (
	self    = os.Args[0]
	cmds    = map[string]cmd{}
	cmdHelp = map[string]string{}
)

const (
	usage1 = `
Each command takes zero or more options and zero or more arguments.
In addition, there are some global options that can be used with any command.
The exit status is 0 on success, 1 for a rev mismatch, and 2 otherwise.

Global Options:
`
	usage2 = `
Environment:

    DOOZER_TOKEN

        Set the capability token for access to the server if needed.

Commands:
`
)

func usage() {
	fmt.Fprintf(os.Stderr, "Use: %s [options] <command> [options] [args]\n", self)
	fmt.Fprint(os.Stderr, usage1)
	flag.PrintDefaults()
	fmt.Fprintln(os.Stderr)

	fmt.Fprint(os.Stderr, usage2)
	var max int
	var names []string
	us := make(map[string]string)
	for k := range cmds {
		u := k + " " + cmds[k].a
		if len(u) > max {
			max = len(u)
		}
		names = append(names, k)
		us[k] = u
	}
	sort.SortStrings(names)
	for _, k := range names {
		fmt.Fprintf(os.Stderr, "  %-*s - %s\n", max, us[k], cmds[k].d)
	}

}

func bail(e os.Error) {
	fmt.Fprintln(os.Stderr, "Error:", e)
	if e == doozer.ErrOldRev {
		os.Exit(1)
	}
	os.Exit(2)
}

func mustAtoi64(arg string) int64 {
	n, err := strconv.Atoi64(arg)
	if err != nil {
		bail(err)
	}
	return n
}

func dial() *doozer.Conn {
	c, err := doozer.Dial(*addr)
	if err != nil {
		bail(err)
	}

	if token := os.Getenv("DOOZER_TOKEN"); token != "" {
		err := c.Access(token)
		if err != nil {
			bail(err)
		}
	}
	return c
}

func main() {
	flag.Usage = usage
	flag.Parse()

	if *showHelp {
		usage()
		return
	}

	if *showVersion {
		fmt.Println("doozer", version)
		return
	}

	if flag.NArg() < 1 {
		fmt.Fprintf(os.Stderr, "%s: missing command\n", os.Args[0])
		usage()
		os.Exit(127)
	}

	cmd := flag.Arg(0)

	c, ok := cmds[cmd]
	if !ok {
		fmt.Fprintln(os.Stderr, "Unknown command:", cmd)
		usage()
		os.Exit(127)
	}

	os.Args = flag.Args()
	flag.Parse()

	if *showHelp {
		help(cmd)
		return
	}

	args := flag.Args()
	ft := reflect.Typeof(c.f).(*reflect.FuncType)
	if len(args) != ft.NumIn() {
		fmt.Fprintf(os.Stderr, "%s: wrong number of arguments\n", cmd)
		help(cmd)
		os.Exit(127)
	}

	vals := make([]reflect.Value, len(args))
	for i, s := range args {
		vals[i] = reflect.NewValue(s)
	}
	fv := reflect.NewValue(c.f).(*reflect.FuncValue)
	fv.Call(vals)
}
<commit_msg>rename DOOZER_TOKEN -> DOOZER_SECRET<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/ha/doozer"
	"os"
	"reflect"
	"sort"
	"strconv"
)

var (
	addr        = flag.String("a", "127.0.0.1:8046", "the address to bind to")
	rrev        = flag.Int64("r", -1, "request rev")
	showHelp    = flag.Bool("h", false, "show help")
	showVersion = flag.Bool("v", false, "print version string")
)

type cmd struct {
	f interface{}
	a string // args
	d string // short description
}

var (
	self    = os.Args[0]
	cmds    = map[string]cmd{}
	cmdHelp = map[string]string{}
)

const (
	usage1 = `
Each command takes zero or more options and zero or more arguments.
In addition, there are some global options that can be used with any command.
The exit status is 0 on success, 1 for a rev mismatch, and 2 otherwise.

Global Options:
`
	usage2 = `
Environment:

    DOOZER_SECRET

        Set the capability token for access to the server if needed.

Commands:
`
)

func usage() {
	fmt.Fprintf(os.Stderr, "Use: %s [options] <command> [options] [args]\n", self)
	fmt.Fprint(os.Stderr, usage1)
	flag.PrintDefaults()
	fmt.Fprintln(os.Stderr)

	fmt.Fprint(os.Stderr, usage2)
	var max int
	var names []string
	us := make(map[string]string)
	for k := range cmds {
		u := k + " " + cmds[k].a
		if len(u) > max {
			max = len(u)
		}
		names = append(names, k)
		us[k] = u
	}
	sort.SortStrings(names)
	for _, k := range names {
		fmt.Fprintf(os.Stderr, "  %-*s - %s\n", max, us[k], cmds[k].d)
	}

}

func bail(e os.Error) {
	fmt.Fprintln(os.Stderr, "Error:", e)
	if e == doozer.ErrOldRev {
		os.Exit(1)
	}
	os.Exit(2)
}

func mustAtoi64(arg string) int64 {
	n, err := strconv.Atoi64(arg)
	if err != nil {
		bail(err)
	}
	return n
}

func dial() *doozer.Conn {
	c, err := doozer.Dial(*addr)
	if err != nil {
		bail(err)
	}

	if token := os.Getenv("DOOZER_SECRET"); token != "" {
		err := c.Access(token)
		if err != nil {
			bail(err)
		}
	}
	return c
}

func main() {
	flag.Usage = usage
	flag.Parse()

	if *showHelp {
		usage()
		return
	}

	if *showVersion {
		fmt.Println("doozer", version)
		return
	}

	if flag.NArg() < 1 {
		fmt.Fprintf(os.Stderr, "%s: missing command\n", os.Args[0])
		usage()
		os.Exit(127)
	}

	cmd := flag.Arg(0)

	c, ok := cmds[cmd]
	if !ok {
		fmt.Fprintln(os.Stderr, "Unknown command:", cmd)
		usage()
		os.Exit(127)
	}

	os.Args = flag.Args()
	flag.Parse()

	if *showHelp {
		help(cmd)
		return
	}

	args := flag.Args()
	ft := reflect.Typeof(c.f).(*reflect.FuncType)
	if len(args) != ft.NumIn() {
		fmt.Fprintf(os.Stderr, "%s: wrong number of arguments\n", cmd)
		help(cmd)
		os.Exit(127)
	}

	vals := make([]reflect.Value, len(args))
	for i, s := range args {
		vals[i] = reflect.NewValue(s)
	}
	fv := reflect.NewValue(c.f).(*reflect.FuncValue)
	fv.Call(vals)
}
<|endoftext|>


<commit_before>package main

import "fmt"

func main() {
	fmt.Println("doof")
}
<commit_msg>moved<commit_after><|endoftext|>


<commit_before>package main

import (
	"flag"
	"os"
	"runtime"
	//"strconv"
	"time"

	"github.com/Sirupsen/logrus"

	"github.com/mozilla/tls-observatory/config"
	"github.com/mozilla/tls-observatory/connection"
	pg "github.com/mozilla/tls-observatory/database"
	"github.com/mozilla/tls-observatory/logger"
	"github.com/mozilla/tls-observatory/worker"
)

var db *pg.DB
var log = logger.GetLogger()

func main() {
	var (
		cfgFile, cipherscan string
		debug               bool
	)
	flag.StringVar(&cfgFile, "c", "/etc/tls-observatory/scanner.cfg", "Input file csv format")
	flag.StringVar(&cipherscan, "b", "/opt/cipherscan/cipherscan", "Cipherscan binary location")
	flag.BoolVar(&debug, "debug", false, "Set debug logging")
	flag.Parse()

	if debug {
		logger.SetLevelToDebug()
	}

	conf, err := config.Load(cfgFile)
	if err != nil {
		log.Fatal("Failed to load configuration: %v", err)
	}
	if !conf.General.Enable && os.Getenv("TLSOBS_SCANNER_ENABLE") != "on" {
		log.Fatal("Scanner is disabled in configuration")
	}

	_, err = os.Stat(cipherscan)
	if err != nil {
		log.WithFields(logrus.Fields{
			"error": err.Error(),
		}).Error("Could not locate cipherscan executable. TLS connection capabilities will not be available.")
	}

	cores := runtime.NumCPU()
	runtime.GOMAXPROCS(cores * conf.General.GoRoutines)

	db, err = pg.RegisterConnection(conf.General.PostgresDB, conf.General.PostgresUser, conf.General.PostgresPass, conf.General.Postgres, "disable")
	if err != nil {
		log.WithFields(logrus.Fields{
			"error": err.Error(),
		}).Fatal("Failed to connect to database")
	}

	incomingScans := db.RegisterScanListener(conf.General.PostgresDB, conf.General.PostgresUser, conf.General.PostgresPass, conf.General.Postgres, "disable")
	Setup(conf)

	for scanId := range incomingScans {
		go scan(scanId, cipherscan)
	}
}

func scan(scanId int64, cipherscan string) {

	log.WithFields(logrus.Fields{
		"scan_id": scanId,
	}).Debug("Received new scan ")

	scan, err := db.GetScanByID(scanId)
	if err != nil {
		log.WithFields(logrus.Fields{
			"scan_id": scanId,
			"error":   err.Error(),
		}).Error("Could not find/decode scan")
		return
	}
	var completion int = 0

	// Retrieve the certificate from the target
	certID, trustID, err := handleCert(scan.Target)
	if err != nil {
		err, ok := err.(NoTLSCertsErr)
		if ok {
			//nil cert, does not implement TLS
			db.Exec("UPDATE scans SET has_tls=FALSE WHERE id=$1", scanId)
			return
		} else {
			log.WithFields(logrus.Fields{
				"scan_id":     scanId,
				"scan_Target": scan.Target,
				"error":       err.Error(),
			}).Error("Could not get certificate info")
			return
		}
	}
	log.WithFields(logrus.Fields{
		"scan_id":  scanId,
		"cert_id":  certID,
		"trust_id": trustID,
	}).Debug("Retrieved certs")

	isTrustValid, err := db.IsTrustValid(trustID)
	if err != nil {
		log.WithFields(logrus.Fields{
			"scan_id": scanId,
			"cert_id": certID,
			"error":   err.Error(),
		}).Error("Could not get if trust is valid")
		return
	}

	_, err = db.Exec("UPDATE scans SET cert_id=$1,trust_id=$2,has_tls=TRUE,is_valid=$3 WHERE id=$4", certID, trustID, isTrustValid, scanId)
	if err != nil {
		log.WithFields(logrus.Fields{
			"scan_id": scanId,
			"cert_id": certID,
			"error":   err.Error(),
		}).Error("Could not update scans for cert")
	}

	completion += 20
	err = db.UpdateScanCompletionPercentage(scanId, completion)
	if err != nil {
		log.WithFields(logrus.Fields{
			"scan_id": scanId,
			"error":   err.Error(),
		}).Error("Could not update completion percentage for scan")
	}

	// Cipherscan the target
	js, err := connection.Connect(scan.Target, cipherscan)
	if err != nil {
		err, ok := err.(connection.NoTLSConnErr)
		if ok {
			//does not implement TLS
			db.Exec("UPDATE scans SET has_tls=FALSE WHERE id=$1", scanId)
			return
		} else {
			log.WithFields(logrus.Fields{
				"scan_id": scanId,
				"error":   err.Error(),
			}).Error("Could not get TLS connection info")
			return
		}
	} else {
		db.Exec("UPDATE scans SET conn_info=$1 WHERE id=$2", js, scanId)
		completion += 20
		err = db.UpdateScanCompletionPercentage(scanId, completion)
		if err != nil {
			log.WithFields(logrus.Fields{
				"scan_id": scanId,
				"error":   err.Error(),
			}).Error("Could not update completion percentage for scan")
		}
	}

	// launch workers that evaluate the results
	resChan := make(chan worker.WorkerResult)
	for _, wrkInfo := range worker.AvailableWorkers {
		go wrkInfo.Runner.(worker.Worker).Run(js, resChan)
	}

	totalWorkers := len(worker.AvailableWorkers)

	log.WithFields(logrus.Fields{
		"scan_id": scanId,
		"workers": totalWorkers,
	}).Debug("Running workers")

	if totalWorkers > 0 {
		endedWorkers := 0
		select {
		case <-time.After(10 * time.Second):

			log.WithFields(logrus.Fields{
				"scan_id": scanId,
			}).Debug("Scanners timed out")

			if err != nil {
				log.Println(err)
			}
			return

		case res := <-resChan:
			endedWorkers += endedWorkers
			completion = ((endedWorkers/totalWorkers)*60 + completion)

			log.WithFields(logrus.Fields{
				"scan_id":        scanId,
				"worker_name":    res.WorkerName,
				"result_success": res.Success,
				"result_data":    string(res.Result),
			}).Debug("Received results from worker")

			err = db.UpdateScanCompletionPercentage(scanId, completion)
			if err != nil {
				log.WithFields(logrus.Fields{
					"scan_id": scanId,
					"error":   err.Error(),
				}).Error("Could not update completion percentage")
			}

			if res.Success {

				log.WithFields(logrus.Fields{
					"scan_id":     scanId,
					"worker_name": res.WorkerName,
					"result_data": string(res.Result),
				}).Debug("Storing results from worker")
				db.Exec("INSERT INTO analysis(scan_id,worker_name,output) VALUES($1,$2,$3)", scanId, res.WorkerName, res.Result)
			} else {
				log.WithFields(logrus.Fields{
					"worker_name": res.WorkerName,
					"errors":      res.Errors,
				}).Error("Worker returned with errors")
			}
		}
	} else {
		err = db.UpdateScanCompletionPercentage(scanId, 100)
	}
}
<commit_msg>Set completion to 100 when a target doesn't support TLS<commit_after>package main

import (
	"flag"
	"os"
	"runtime"
	//"strconv"
	"time"

	"github.com/Sirupsen/logrus"

	"github.com/mozilla/tls-observatory/config"
	"github.com/mozilla/tls-observatory/connection"
	pg "github.com/mozilla/tls-observatory/database"
	"github.com/mozilla/tls-observatory/logger"
	"github.com/mozilla/tls-observatory/worker"
)

var db *pg.DB
var log = logger.GetLogger()

func main() {
	var (
		cfgFile, cipherscan string
		debug               bool
	)
	flag.StringVar(&cfgFile, "c", "/etc/tls-observatory/scanner.cfg", "Input file csv format")
	flag.StringVar(&cipherscan, "b", "/opt/cipherscan/cipherscan", "Cipherscan binary location")
	flag.BoolVar(&debug, "debug", false, "Set debug logging")
	flag.Parse()

	if debug {
		logger.SetLevelToDebug()
	}

	conf, err := config.Load(cfgFile)
	if err != nil {
		log.Fatal("Failed to load configuration: %v", err)
	}
	if !conf.General.Enable && os.Getenv("TLSOBS_SCANNER_ENABLE") != "on" {
		log.Fatal("Scanner is disabled in configuration")
	}

	_, err = os.Stat(cipherscan)
	if err != nil {
		log.WithFields(logrus.Fields{
			"error": err.Error(),
		}).Error("Could not locate cipherscan executable. TLS connection capabilities will not be available.")
	}

	cores := runtime.NumCPU()
	runtime.GOMAXPROCS(cores * conf.General.GoRoutines)

	db, err = pg.RegisterConnection(conf.General.PostgresDB, conf.General.PostgresUser, conf.General.PostgresPass, conf.General.Postgres, "disable")
	if err != nil {
		log.WithFields(logrus.Fields{
			"error": err.Error(),
		}).Fatal("Failed to connect to database")
	}

	incomingScans := db.RegisterScanListener(conf.General.PostgresDB, conf.General.PostgresUser, conf.General.PostgresPass, conf.General.Postgres, "disable")
	Setup(conf)

	for scanId := range incomingScans {
		go scan(scanId, cipherscan)
	}
}

func scan(scanId int64, cipherscan string) {

	log.WithFields(logrus.Fields{
		"scan_id": scanId,
	}).Debug("Received new scan ")

	scan, err := db.GetScanByID(scanId)
	if err != nil {
		log.WithFields(logrus.Fields{
			"scan_id": scanId,
			"error":   err.Error(),
		}).Error("Could not find/decode scan")
		return
	}
	var completion int = 0

	// Retrieve the certificate from the target
	certID, trustID, err := handleCert(scan.Target)
	if err != nil {
		err, ok := err.(NoTLSCertsErr)
		if ok {
			//nil cert, does not implement TLS
			db.Exec("UPDATE scans SET has_tls=FALSE, completion_perc=100 WHERE id=$1", scanId)
			return
		} else {
			log.WithFields(logrus.Fields{
				"scan_id":     scanId,
				"scan_Target": scan.Target,
				"error":       err.Error(),
			}).Error("Could not get certificate info")
			return
		}
	}
	log.WithFields(logrus.Fields{
		"scan_id":  scanId,
		"cert_id":  certID,
		"trust_id": trustID,
	}).Debug("Retrieved certs")

	isTrustValid, err := db.IsTrustValid(trustID)
	if err != nil {
		log.WithFields(logrus.Fields{
			"scan_id": scanId,
			"cert_id": certID,
			"error":   err.Error(),
		}).Error("Could not get if trust is valid")
		return
	}
	completion += 20
	_, err = db.Exec(`UPDATE scans
			SET cert_id=$1, trust_id=$2, has_tls=TRUE, is_valid=$3, completion_perc=$4
			WHERE id=$5`, certID, trustID, isTrustValid, completion, scanId)
	if err != nil {
		log.WithFields(logrus.Fields{
			"scan_id": scanId,
			"cert_id": certID,
			"error":   err.Error(),
		}).Error("Could not update scans for cert")
		return
	}

	// Cipherscan the target
	js, err := connection.Connect(scan.Target, cipherscan)
	if err != nil {
		err, ok := err.(connection.NoTLSConnErr)
		if ok {
			//does not implement TLS
			db.Exec("UPDATE scans SET has_tls=FALSE, completion_perc=100 WHERE id=$1", scanId)
		} else {
			log.WithFields(logrus.Fields{
				"scan_id": scanId,
				"error":   err.Error(),
			}).Error("Could not get TLS connection info")
		}
		return
	}
	completion += 20
	_, err = db.Exec("UPDATE scans SET conn_info=$1, completion_perc=$2 WHERE id=$3",
		js, completion, scanId)
	if err != nil {
		log.WithFields(logrus.Fields{
			"scan_id": scanId,
			"error":   err.Error(),
		}).Error("Could not update connection information for scan")
	}

	// launch workers that evaluate the results
	resChan := make(chan worker.WorkerResult)
	for _, wrkInfo := range worker.AvailableWorkers {
		go wrkInfo.Runner.(worker.Worker).Run(js, resChan)
	}

	totalWorkers := len(worker.AvailableWorkers)

	log.WithFields(logrus.Fields{
		"scan_id": scanId,
		"workers": totalWorkers,
	}).Debug("Running workers")

	if totalWorkers > 0 {
		endedWorkers := 0
		select {
		case <-time.After(10 * time.Second):

			log.WithFields(logrus.Fields{
				"scan_id": scanId,
			}).Debug("Scanners timed out")

			if err != nil {
				log.Println(err)
			}
			return

		case res := <-resChan:
			endedWorkers += endedWorkers
			completion = ((endedWorkers/totalWorkers)*60 + completion)

			log.WithFields(logrus.Fields{
				"scan_id":        scanId,
				"worker_name":    res.WorkerName,
				"result_success": res.Success,
				"result_data":    string(res.Result),
			}).Debug("Received results from worker")

			err = db.UpdateScanCompletionPercentage(scanId, completion)
			if err != nil {
				log.WithFields(logrus.Fields{
					"scan_id": scanId,
					"error":   err.Error(),
				}).Error("Could not update completion percentage")
			}

			if res.Success {

				log.WithFields(logrus.Fields{
					"scan_id":     scanId,
					"worker_name": res.WorkerName,
					"result_data": string(res.Result),
				}).Debug("Storing results from worker")
				db.Exec("INSERT INTO analysis(scan_id,worker_name,output) VALUES($1,$2,$3)", scanId, res.WorkerName, res.Result)
			} else {
				log.WithFields(logrus.Fields{
					"worker_name": res.WorkerName,
					"errors":      res.Errors,
				}).Error("Worker returned with errors")
			}
		}
	} else {
		err = db.UpdateScanCompletionPercentage(scanId, 100)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"runtime"
	"sync"
	"time"

	"github.com/bradfitz/gomemcache/memcache"
	"github.com/ubleipzig/marctools"
)

const (
	version = "1.2.0"
	backoff = 50 * time.Millisecond
)

var errSetFailed = errors.New("cache set failed")

type work struct {
	blob []byte
	id   string
}

type options struct {
	hostport string
	retry    uint
	verbose  bool
}

func worker(queue chan []work, opts options, wg *sync.WaitGroup) {
	defer wg.Done()
	mc := memcache.New(opts.hostport)
	for batch := range queue {
		for _, work := range batch {
			ok := false
			var i uint

			for i = 1; i <= opts.retry; i++ {
				err := mc.Set(&memcache.Item{Key: work.id, Value: work.blob})
				if err != nil {
					pause := 2 << i * backoff
					if opts.verbose {
						log.Printf("retry %d for %s in %s ...", i, work.id, pause)
					}
					time.Sleep(pause)
				} else {
					ok = true
					break
				}
			}
			if !ok {
				log.Fatal(errSetFailed)
			}
		}
	}
}

func main() {

	hostport := flag.String("addr", "127.0.0.1:11211", "hostport of memcache")
	retry := flag.Int("retry", 10, "retry set operation this many times")
	numWorker := flag.Int("w", runtime.NumCPU(), "number of workers")
	size := flag.Int("b", 10000, "batch size")
	verbose := flag.Bool("verbose", false, "be verbose")
	showVersion := flag.Bool("v", false, "prints current program version")

	flag.Parse()

	runtime.GOMAXPROCS(*numWorker)

	if *showVersion {
		fmt.Println(version)
		os.Exit(0)
	}

	if flag.NArg() < 1 {
		log.Fatal("input file or files required")
	}

	opts := options{
		hostport: *hostport,
		retry:    uint(*retry),
		verbose:  *verbose,
	}

	queue := make(chan []work)
	var wg sync.WaitGroup

	for i := 0; i < *numWorker; i++ {
		wg.Add(1)
		go worker(queue, opts, &wg)
	}

	var batch []work
	var counter int

	for _, filename := range flag.Args() {

		var offset int64
		var i int

		file, err := os.Open(filename)
		if err != nil {
			log.Fatal(err)
		}

		ids := marctools.IDList(file.Name())

		for {
			length, err := marctools.RecordLength(file)
			if err == io.EOF {
				break
			}
			if err != nil {
				log.Fatal(err)
			}

			file.Seek(offset, 0)
			buf := make([]byte, length)
			_, err = file.Read(buf)
			if err != nil {
				log.Fatal(err)
			}

			batch = append(batch, work{id: ids[i], blob: buf})

			if i%*size == 0 {
				b := make([]work, len(batch))
				copy(b, batch)
				queue <- b
				if *verbose {
					log.Printf("@%d", i)
				}
				batch = batch[:0]
			}

			offset = offset + length
			i++
			counter++
		}
	}

	b := make([]work, len(batch))
	copy(b, batch)
	queue <- b
	if *verbose {
		log.Printf("@%d", counter)
	}

	close(queue)
	wg.Wait()
}
<commit_msg>bail out early on invalid keys<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"runtime"
	"sync"
	"time"

	"github.com/bradfitz/gomemcache/memcache"
	"github.com/ubleipzig/marctools"
)

const (
	version = "1.2.0"
	backoff = 50 * time.Millisecond
)

var errSetFailed = errors.New("cache set failed")

type work struct {
	blob []byte
	id   string
}

type options struct {
	hostport string
	retry    uint
	verbose  bool
}

func worker(queue chan []work, opts options, wg *sync.WaitGroup) {
	defer wg.Done()
	mc := memcache.New(opts.hostport)
	for batch := range queue {
		for _, work := range batch {
			ok := false
			var i uint

			for i = 1; i <= opts.retry; i++ {
				err := mc.Set(&memcache.Item{Key: work.id, Value: work.blob})
				if err != nil {
					pause := 2 << i * backoff
					if opts.verbose {
						log.Printf("retry %d for %s in %s ...", i, work.id, pause)
					}
					time.Sleep(pause)
				} else {
					ok = true
					break
				}
			}
			if !ok {
				log.Fatal(errSetFailed)
			}
		}
	}
}

func main() {

	hostport := flag.String("addr", "127.0.0.1:11211", "hostport of memcache")
	retry := flag.Int("retry", 10, "retry set operation this many times")
	numWorker := flag.Int("w", runtime.NumCPU(), "number of workers")
	size := flag.Int("b", 10000, "batch size")
	verbose := flag.Bool("verbose", false, "be verbose")
	showVersion := flag.Bool("v", false, "prints current program version")

	flag.Parse()

	runtime.GOMAXPROCS(*numWorker)

	if *showVersion {
		fmt.Println(version)
		os.Exit(0)
	}

	if flag.NArg() < 1 {
		log.Fatal("input file or files required")
	}

	opts := options{
		hostport: *hostport,
		retry:    uint(*retry),
		verbose:  *verbose,
	}

	queue := make(chan []work)
	var wg sync.WaitGroup

	for i := 0; i < *numWorker; i++ {
		wg.Add(1)
		go worker(queue, opts, &wg)
	}

	var batch []work
	var counter int

	for _, filename := range flag.Args() {

		var offset int64
		var i int

		file, err := os.Open(filename)
		if err != nil {
			log.Fatal(err)
		}

		ids := marctools.IDList(file.Name())

		for {
			length, err := marctools.RecordLength(file)
			if err == io.EOF {
				break
			}
			if err != nil {
				log.Fatal(err)
			}
			if len(ids[i]) > 250 {
				log.Fatal("memcache maximum key length (250) exceeded")
			}

			file.Seek(offset, 0)
			buf := make([]byte, length)
			_, err = file.Read(buf)
			if err != nil {
				log.Fatal(err)
			}

			batch = append(batch, work{id: ids[i], blob: buf})

			if i%*size == 0 {
				b := make([]work, len(batch))
				copy(b, batch)
				queue <- b
				if *verbose {
					log.Printf("@%d", i)
				}
				batch = batch[:0]
			}

			offset = offset + length
			i++
			counter++
		}
	}

	b := make([]work, len(batch))
	copy(b, batch)
	queue <- b
	if *verbose {
		log.Printf("@%d", counter)
	}

	close(queue)
	wg.Wait()
}
<|endoftext|>


<commit_before>// Copyright 2016 The etcd-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"flag"
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"runtime"
	"time"

	"github.com/coreos/etcd-operator/pkg/analytics"
	"github.com/coreos/etcd-operator/pkg/backup/s3/s3config"
	"github.com/coreos/etcd-operator/pkg/chaos"
	"github.com/coreos/etcd-operator/pkg/client"
	"github.com/coreos/etcd-operator/pkg/controller"
	"github.com/coreos/etcd-operator/pkg/debug"
	"github.com/coreos/etcd-operator/pkg/garbagecollection"
	"github.com/coreos/etcd-operator/pkg/util/constants"
	"github.com/coreos/etcd-operator/pkg/util/k8sutil"
	"github.com/coreos/etcd-operator/pkg/util/probe"
	"github.com/coreos/etcd-operator/pkg/util/retryutil"
	"github.com/coreos/etcd-operator/version"

	"github.com/Sirupsen/logrus"
	"golang.org/x/time/rate"
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/scheme"
	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
	"k8s.io/client-go/tools/leaderelection"
	"k8s.io/client-go/tools/leaderelection/resourcelock"
	"k8s.io/client-go/tools/record"
)

var (
	analyticsEnabled bool
	pvProvisioner    string
	namespace        string
	name             string
	awsSecret        string
	awsConfig        string
	s3Bucket         string
	listenAddr       string
	gcInterval       time.Duration

	chaosLevel int

	printVersion bool
)

var (
	leaseDuration = 15 * time.Second
	renewDuration = 5 * time.Second
	retryPeriod   = 3 * time.Second
)

func init() {
	flag.BoolVar(&analyticsEnabled, "analytics", true, "Send analytical event (Cluster Created/Deleted etc.) to Google Analytics")
	flag.StringVar(&debug.DebugFilePath, "debug-logfile-path", "", "only for a self hosted cluster, the path where the debug logfile will be written, recommended to be under: /var/tmp/etcd-operator/debug/ to avoid any issue with lack of write permissions")

	flag.StringVar(&pvProvisioner, "pv-provisioner", constants.PVProvisionerGCEPD, "persistent volume provisioner type")
	flag.StringVar(&awsSecret, "backup-aws-secret", "",
		"The name of the kube secret object that stores the AWS credential file. The file name must be 'credentials'.")
	flag.StringVar(&awsConfig, "backup-aws-config", "",
		"The name of the kube configmap object that stores the AWS config file. The file name must be 'config'.")
	flag.StringVar(&s3Bucket, "backup-s3-bucket", "", "The name of the AWS S3 bucket to store backups in.")
	flag.StringVar(&listenAddr, "listen-addr", "0.0.0.0:8080", "The address on which the HTTP server will listen to")
	// chaos level will be removed once we have a formal tool to inject failures.
	flag.IntVar(&chaosLevel, "chaos-level", -1, "DO NOT USE IN PRODUCTION - level of chaos injected into the etcd clusters created by the operator.")
	flag.BoolVar(&printVersion, "version", false, "Show version and quit")
	flag.DurationVar(&gcInterval, "gc-interval", 10*time.Minute, "GC interval")
	flag.Parse()

	// TODO: remove this and use CR client
	restCfg, err := k8sutil.InClusterConfig()
	if err != nil {
		panic(err)
	}
	controller.MasterHost = restCfg.Host
	restcli, _, err := client.New(restCfg)
	if err != nil {
		panic(err)
	}
	controller.KubeHttpCli = restcli.Client
}

func main() {
	namespace = os.Getenv("MY_POD_NAMESPACE")
	if len(namespace) == 0 {
		logrus.Fatalf("must set env MY_POD_NAMESPACE")
	}
	name = os.Getenv("MY_POD_NAME")
	if len(name) == 0 {
		logrus.Fatalf("must set env MY_POD_NAME")
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c)
	go func() {
		logrus.Infof("received signal: %v", <-c)
		os.Exit(1)
	}()

	if printVersion {
		fmt.Println("etcd-operator Version:", version.Version)
		fmt.Println("Git SHA:", version.GitSHA)
		fmt.Println("Go Version:", runtime.Version())
		fmt.Printf("Go OS/Arch: %s/%s\n", runtime.GOOS, runtime.GOARCH)
		os.Exit(0)
	}

	logrus.Infof("etcd-operator Version: %v", version.Version)
	logrus.Infof("Git SHA: %s", version.GitSHA)
	logrus.Infof("Go Version: %s", runtime.Version())
	logrus.Infof("Go OS/Arch: %s/%s", runtime.GOOS, runtime.GOARCH)

	if analyticsEnabled {
		analytics.Enable()
	}

	analytics.OperatorStarted()

	id, err := os.Hostname()
	if err != nil {
		logrus.Fatalf("failed to get hostname: %v", err)
	}

	kubecli := k8sutil.MustNewKubeClient()

	http.HandleFunc(probe.HTTPReadyzEndpoint, probe.ReadyzHandler)
	go http.ListenAndServe(listenAddr, nil)

	rl, err := resourcelock.New(resourcelock.EndpointsResourceLock,
		namespace,
		"etcd-operator",
		kubecli.(*kubernetes.Clientset),
		resourcelock.ResourceLockConfig{
			Identity:      id,
			EventRecorder: createRecorder(kubecli, name, namespace),
		})
	if err != nil {
		logrus.Fatalf("error creating lock: %v", err)
	}

	leaderelection.RunOrDie(leaderelection.LeaderElectionConfig{
		Lock:          rl,
		LeaseDuration: 15 * time.Second,
		RenewDeadline: 10 * time.Second,
		RetryPeriod:   2 * time.Second,
		Callbacks: leaderelection.LeaderCallbacks{
			OnStartedLeading: run,
			OnStoppedLeading: func() {
				logrus.Fatalf("leader election lost")
			},
		},
	})

	panic("unreachable")
}

func run(stop <-chan struct{}) {
	cfg := newControllerConfig()
	if err := cfg.Validate(); err != nil {
		logrus.Fatalf("invalid operator config: %v", err)
	}

	go periodicFullGC(cfg.KubeCli, cfg.Namespace, gcInterval)

	startChaos(context.Background(), cfg.KubeCli, cfg.Namespace, chaosLevel)

	for {
		c := controller.New(cfg)
		err := c.Run()
		switch err {
		case controller.ErrVersionOutdated:
		default:
			logrus.Fatalf("controller Run() ended with failure: %v", err)
		}
	}
}

func newControllerConfig() controller.Config {
	kubecli := k8sutil.MustNewKubeClient()

	serviceAccount, err := getMyPodServiceAccount(kubecli)
	if err != nil {
		logrus.Fatalf("fail to get my pod's service account: %v", err)
	}

	cfg := controller.Config{
		Namespace:      namespace,
		ServiceAccount: serviceAccount,
		PVProvisioner:  pvProvisioner,
		S3Context: s3config.S3Context{
			AWSSecret: awsSecret,
			AWSConfig: awsConfig,
			S3Bucket:  s3Bucket,
		},
		KubeCli:    kubecli,
		KubeExtCli: k8sutil.MustNewKubeExtClient(),
	}

	return cfg
}

func getMyPodServiceAccount(kubecli kubernetes.Interface) (string, error) {
	var sa string
	err := retryutil.Retry(5*time.Second, 100, func() (bool, error) {
		pod, err := kubecli.CoreV1().Pods(namespace).Get(name, metav1.GetOptions{})
		if err != nil {
			logrus.Errorf("fail to get operator pod (%s): %v", name, err)
			return false, nil
		}
		sa = pod.Spec.ServiceAccountName
		return true, nil
	})
	return sa, err
}

func periodicFullGC(kubecli kubernetes.Interface, ns string, d time.Duration) {
	gc := garbagecollection.New(kubecli, ns)
	timer := time.NewTimer(d)
	defer timer.Stop()
	for {
		<-timer.C
		err := gc.FullyCollect()
		if err != nil {
			logrus.Warningf("failed to cleanup resources: %v", err)
		}
	}
}

func startChaos(ctx context.Context, kubecli kubernetes.Interface, ns string, chaosLevel int) {
	m := chaos.NewMonkeys(kubecli)
	ls := labels.SelectorFromSet(map[string]string{"app": "etcd"})

	switch chaosLevel {
	case 1:
		logrus.Info("chaos level = 1: randomly kill one etcd pod every 30 seconds at 50%")
		c := &chaos.CrashConfig{
			Namespace: ns,
			Selector:  ls,

			KillRate:        rate.Every(30 * time.Second),
			KillProbability: 0.5,
			KillMax:         1,
		}
		go func() {
			time.Sleep(60 * time.Second) // don't start until quorum up
			m.CrushPods(ctx, c)
		}()

	case 2:
		logrus.Info("chaos level = 2: randomly kill at most five etcd pods every 30 seconds at 50%")
		c := &chaos.CrashConfig{
			Namespace: ns,
			Selector:  ls,

			KillRate:        rate.Every(30 * time.Second),
			KillProbability: 0.5,
			KillMax:         5,
		}

		go m.CrushPods(ctx, c)

	default:
	}
}

func createRecorder(kubecli kubernetes.Interface, name, namespace string) record.EventRecorder {
	eventBroadcaster := record.NewBroadcaster()
	eventBroadcaster.StartLogging(logrus.Infof)
	eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: v1core.New(kubecli.Core().RESTClient()).Events(namespace)})
	return eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: name})
}
<commit_msg>main: remove unused vars<commit_after>// Copyright 2016 The etcd-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"flag"
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"runtime"
	"time"

	"github.com/coreos/etcd-operator/pkg/analytics"
	"github.com/coreos/etcd-operator/pkg/backup/s3/s3config"
	"github.com/coreos/etcd-operator/pkg/chaos"
	"github.com/coreos/etcd-operator/pkg/client"
	"github.com/coreos/etcd-operator/pkg/controller"
	"github.com/coreos/etcd-operator/pkg/debug"
	"github.com/coreos/etcd-operator/pkg/garbagecollection"
	"github.com/coreos/etcd-operator/pkg/util/constants"
	"github.com/coreos/etcd-operator/pkg/util/k8sutil"
	"github.com/coreos/etcd-operator/pkg/util/probe"
	"github.com/coreos/etcd-operator/pkg/util/retryutil"
	"github.com/coreos/etcd-operator/version"

	"github.com/Sirupsen/logrus"
	"golang.org/x/time/rate"
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/scheme"
	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
	"k8s.io/client-go/tools/leaderelection"
	"k8s.io/client-go/tools/leaderelection/resourcelock"
	"k8s.io/client-go/tools/record"
)

var (
	analyticsEnabled bool
	pvProvisioner    string
	namespace        string
	name             string
	awsSecret        string
	awsConfig        string
	s3Bucket         string
	listenAddr       string
	gcInterval       time.Duration

	chaosLevel int

	printVersion bool
)

func init() {
	flag.BoolVar(&analyticsEnabled, "analytics", true, "Send analytical event (Cluster Created/Deleted etc.) to Google Analytics")
	flag.StringVar(&debug.DebugFilePath, "debug-logfile-path", "", "only for a self hosted cluster, the path where the debug logfile will be written, recommended to be under: /var/tmp/etcd-operator/debug/ to avoid any issue with lack of write permissions")

	flag.StringVar(&pvProvisioner, "pv-provisioner", constants.PVProvisionerGCEPD, "persistent volume provisioner type")
	flag.StringVar(&awsSecret, "backup-aws-secret", "",
		"The name of the kube secret object that stores the AWS credential file. The file name must be 'credentials'.")
	flag.StringVar(&awsConfig, "backup-aws-config", "",
		"The name of the kube configmap object that stores the AWS config file. The file name must be 'config'.")
	flag.StringVar(&s3Bucket, "backup-s3-bucket", "", "The name of the AWS S3 bucket to store backups in.")
	flag.StringVar(&listenAddr, "listen-addr", "0.0.0.0:8080", "The address on which the HTTP server will listen to")
	// chaos level will be removed once we have a formal tool to inject failures.
	flag.IntVar(&chaosLevel, "chaos-level", -1, "DO NOT USE IN PRODUCTION - level of chaos injected into the etcd clusters created by the operator.")
	flag.BoolVar(&printVersion, "version", false, "Show version and quit")
	flag.DurationVar(&gcInterval, "gc-interval", 10*time.Minute, "GC interval")
	flag.Parse()

	// TODO: remove this and use CR client
	restCfg, err := k8sutil.InClusterConfig()
	if err != nil {
		panic(err)
	}
	controller.MasterHost = restCfg.Host
	restcli, _, err := client.New(restCfg)
	if err != nil {
		panic(err)
	}
	controller.KubeHttpCli = restcli.Client
}

func main() {
	namespace = os.Getenv("MY_POD_NAMESPACE")
	if len(namespace) == 0 {
		logrus.Fatalf("must set env MY_POD_NAMESPACE")
	}
	name = os.Getenv("MY_POD_NAME")
	if len(name) == 0 {
		logrus.Fatalf("must set env MY_POD_NAME")
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c)
	go func() {
		logrus.Infof("received signal: %v", <-c)
		os.Exit(1)
	}()

	if printVersion {
		fmt.Println("etcd-operator Version:", version.Version)
		fmt.Println("Git SHA:", version.GitSHA)
		fmt.Println("Go Version:", runtime.Version())
		fmt.Printf("Go OS/Arch: %s/%s\n", runtime.GOOS, runtime.GOARCH)
		os.Exit(0)
	}

	logrus.Infof("etcd-operator Version: %v", version.Version)
	logrus.Infof("Git SHA: %s", version.GitSHA)
	logrus.Infof("Go Version: %s", runtime.Version())
	logrus.Infof("Go OS/Arch: %s/%s", runtime.GOOS, runtime.GOARCH)

	if analyticsEnabled {
		analytics.Enable()
	}

	analytics.OperatorStarted()

	id, err := os.Hostname()
	if err != nil {
		logrus.Fatalf("failed to get hostname: %v", err)
	}

	kubecli := k8sutil.MustNewKubeClient()

	http.HandleFunc(probe.HTTPReadyzEndpoint, probe.ReadyzHandler)
	go http.ListenAndServe(listenAddr, nil)

	rl, err := resourcelock.New(resourcelock.EndpointsResourceLock,
		namespace,
		"etcd-operator",
		kubecli.(*kubernetes.Clientset),
		resourcelock.ResourceLockConfig{
			Identity:      id,
			EventRecorder: createRecorder(kubecli, name, namespace),
		})
	if err != nil {
		logrus.Fatalf("error creating lock: %v", err)
	}

	leaderelection.RunOrDie(leaderelection.LeaderElectionConfig{
		Lock:          rl,
		LeaseDuration: 15 * time.Second,
		RenewDeadline: 10 * time.Second,
		RetryPeriod:   2 * time.Second,
		Callbacks: leaderelection.LeaderCallbacks{
			OnStartedLeading: run,
			OnStoppedLeading: func() {
				logrus.Fatalf("leader election lost")
			},
		},
	})

	panic("unreachable")
}

func run(stop <-chan struct{}) {
	cfg := newControllerConfig()
	if err := cfg.Validate(); err != nil {
		logrus.Fatalf("invalid operator config: %v", err)
	}

	go periodicFullGC(cfg.KubeCli, cfg.Namespace, gcInterval)

	startChaos(context.Background(), cfg.KubeCli, cfg.Namespace, chaosLevel)

	for {
		c := controller.New(cfg)
		err := c.Run()
		switch err {
		case controller.ErrVersionOutdated:
		default:
			logrus.Fatalf("controller Run() ended with failure: %v", err)
		}
	}
}

func newControllerConfig() controller.Config {
	kubecli := k8sutil.MustNewKubeClient()

	serviceAccount, err := getMyPodServiceAccount(kubecli)
	if err != nil {
		logrus.Fatalf("fail to get my pod's service account: %v", err)
	}

	cfg := controller.Config{
		Namespace:      namespace,
		ServiceAccount: serviceAccount,
		PVProvisioner:  pvProvisioner,
		S3Context: s3config.S3Context{
			AWSSecret: awsSecret,
			AWSConfig: awsConfig,
			S3Bucket:  s3Bucket,
		},
		KubeCli:    kubecli,
		KubeExtCli: k8sutil.MustNewKubeExtClient(),
	}

	return cfg
}

func getMyPodServiceAccount(kubecli kubernetes.Interface) (string, error) {
	var sa string
	err := retryutil.Retry(5*time.Second, 100, func() (bool, error) {
		pod, err := kubecli.CoreV1().Pods(namespace).Get(name, metav1.GetOptions{})
		if err != nil {
			logrus.Errorf("fail to get operator pod (%s): %v", name, err)
			return false, nil
		}
		sa = pod.Spec.ServiceAccountName
		return true, nil
	})
	return sa, err
}

func periodicFullGC(kubecli kubernetes.Interface, ns string, d time.Duration) {
	gc := garbagecollection.New(kubecli, ns)
	timer := time.NewTimer(d)
	defer timer.Stop()
	for {
		<-timer.C
		err := gc.FullyCollect()
		if err != nil {
			logrus.Warningf("failed to cleanup resources: %v", err)
		}
	}
}

func startChaos(ctx context.Context, kubecli kubernetes.Interface, ns string, chaosLevel int) {
	m := chaos.NewMonkeys(kubecli)
	ls := labels.SelectorFromSet(map[string]string{"app": "etcd"})

	switch chaosLevel {
	case 1:
		logrus.Info("chaos level = 1: randomly kill one etcd pod every 30 seconds at 50%")
		c := &chaos.CrashConfig{
			Namespace: ns,
			Selector:  ls,

			KillRate:        rate.Every(30 * time.Second),
			KillProbability: 0.5,
			KillMax:         1,
		}
		go func() {
			time.Sleep(60 * time.Second) // don't start until quorum up
			m.CrushPods(ctx, c)
		}()

	case 2:
		logrus.Info("chaos level = 2: randomly kill at most five etcd pods every 30 seconds at 50%")
		c := &chaos.CrashConfig{
			Namespace: ns,
			Selector:  ls,

			KillRate:        rate.Every(30 * time.Second),
			KillProbability: 0.5,
			KillMax:         5,
		}

		go m.CrushPods(ctx, c)

	default:
	}
}

func createRecorder(kubecli kubernetes.Interface, name, namespace string) record.EventRecorder {
	eventBroadcaster := record.NewBroadcaster()
	eventBroadcaster.StartLogging(logrus.Infof)
	eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: v1core.New(kubecli.Core().RESTClient()).Events(namespace)})
	return eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: name})
}
<|endoftext|>


<commit_before>// Copyright 2015 Matthew Collins
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"io"
	"os"
	"os/exec"
	"runtime"

	"github.com/thinkofdeath/steven"
)

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU() * 2)

	// Can't use flags as we need to support a weird flag
	// format
	var username, uuid, accessToken string
	var noFork bool

	for i, arg := range os.Args {
		switch arg {
		case "--username":
			username = os.Args[i+1]
		case "--uuid":
			uuid = os.Args[i+1]
		case "--accessToken":
			accessToken = os.Args[i+1]
		case "--no-fork":
			noFork = true
		}
	}
	if noFork {
		steven.Main(username, uuid, accessToken)
		return
	}

	cmd := exec.Command(os.Args[0], append(os.Args[1:], "--no-fork")...)
	l, err := os.Create("steven.log")
	if err != nil {
		panic(err)
	}
	defer l.Close()
	cmd.Stdout = io.MultiWriter(os.Stdout, l)
	f, err := os.Create("steven-error.log")
	if err != nil {
		panic(err)
	}
	defer f.Close()
	cmd.Stderr = io.MultiWriter(f, os.Stderr)
	cmd.Run()
}
<commit_msg>cmd/steven: don't manually set GOMAXPROCS<commit_after>// Copyright 2015 Matthew Collins
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"io"
	"os"
	"os/exec"

	"github.com/thinkofdeath/steven"
)

func main() {
	// Can't use flags as we need to support a weird flag
	// format
	var username, uuid, accessToken string
	var noFork bool

	for i, arg := range os.Args {
		switch arg {
		case "--username":
			username = os.Args[i+1]
		case "--uuid":
			uuid = os.Args[i+1]
		case "--accessToken":
			accessToken = os.Args[i+1]
		case "--no-fork":
			noFork = true
		}
	}
	if noFork {
		steven.Main(username, uuid, accessToken)
		return
	}

	cmd := exec.Command(os.Args[0], append(os.Args[1:], "--no-fork")...)
	l, err := os.Create("steven.log")
	if err != nil {
		panic(err)
	}
	defer l.Close()
	cmd.Stdout = io.MultiWriter(os.Stdout, l)
	f, err := os.Create("steven-error.log")
	if err != nil {
		panic(err)
	}
	defer f.Close()
	cmd.Stderr = io.MultiWriter(f, os.Stderr)
	cmd.Run()
}
<|endoftext|>


<commit_before>// tinystat is used to compare two or more sets of measurements (e.g., runs of a
// multiple runs of benchmarks of two possible implementations) and determine if
// they are statistically different. It's inspired largely by FreeBSD's ministat
// (written by Poul-Henning Kamp).
//
// Imagine we have the results of different animals' SAT scores. Each animal
// took the SATs multiple times, and we're assuming that differences between
// each animal's attempts are measurement error (i.e., normally distributed). We
// can test for differences as follows:
//
//     $ tinystat iguana chameleon leopard
//
//      1.5 k  +
//             |
//             |
//             |
//             |
//             |
//       1000  +                                             |
//             |                              |              |
//             |                        +-----------+  +-----------+
//             |              |         |           |  |           |
//             |              |         |           |  +-----*-----+
//             |              |         |     *     |  |           |
//        500  +              |         +-----------+  +-----------+
//             |        +-----------+   |           |        |
//             |        |     *     |   +-----------+
//             |        +-----------+         |
//             |        +-----------+         |
//          0  +--------------|-----------------------------------------------
//                         iguana         chameleon       leopard
//
//     Experiment  Results
//     chameleon   No difference proven at 95% confidence.
//     leopard     Difference at 95% confidence!
//                   343.5 +/- 292.63453863922877
//                   114.5% +/- 97.54484621307626%
//                   (Student's t, pooled s = 238.9799344943192)
//
// As you can see, despite the superficial differences between the iguana's
// scores and the chameleon's scores, there is no statistically significant
// difference between the two at a 95% confidence level. The leopard, on the
// other hand, has statistically significantly different scores.
package main

import (
	"encoding/csv"
	"flag"
	"fmt"
	"os"
	"path"
	"strconv"
	"text/tabwriter"

	"github.com/codahale/tinystat"
	"github.com/vdobler/chart"
	"github.com/vdobler/chart/txtg"
)

//nolint:gochecknoglobals // main package
var (
	confidence = flag.Float64("confidence", 95, "confidence level for analysis (0,100)")
	column     = flag.Int("column", 0, "the CSV column to analyze")
	delimiter  = flag.String("delimiter", ",", "the CSV delimiter to use")
	noChart    = flag.Bool("no_chart", false, "don't display the box chart")
	width      = flag.Int("width", 74, "width of box chart, in chars")
	height     = flag.Int("height", 20, "height of box chart, in chars")
	v          = flag.Bool("version", false, "display the version number")
	version    = "dev" // injected by goreleaser
)

func main() {
	flag.Usage = func() {
		_, _ = fmt.Fprintf(os.Stderr, "Usage: tinystat <control.csv> [<experiment.csv>...] [options] \n\n")

		flag.PrintDefaults()
	}
	flag.Parse()

	if *v {
		fmt.Println(version)
		return
	}

	if len(flag.Args()) < 1 {
		flag.Usage()
		os.Exit(-1)
	}

	controlFilename := flag.Arg(0)
	experimentFilenames := flag.Args()[1:]

	// read the data
	controlData, experimentData, err := readData(controlFilename, experimentFilenames)
	if err != nil {
		_, _ = fmt.Fprintln(os.Stderr, err)
		os.Exit(-1)
	}

	// chart the data
	if !*noChart {
		printChart(experimentFilenames, controlFilename, controlData, experimentData)
	}

	// compare the data
	if len(experimentFilenames) > 0 {
		printComparison(controlData, experimentFilenames, experimentData)
	}
}

func printComparison(
	controlData []float64, experimentFilenames []string, experimentData map[string][]float64,
) {
	control := tinystat.Summarize(controlData)
	table := &tabwriter.Writer{}
	table.Init(os.Stdout, 2, 0, 2, ' ', 0)
	_, _ = fmt.Fprintln(table, "Experiment\tResults\t")

	for _, filename := range experimentFilenames {
		experimental := tinystat.Summarize(experimentData[filename])
		d := tinystat.Compare(control, experimental, *confidence)

		if d.Significant() {
			_, _ = fmt.Fprintf(table, "%s\tDifference at %v%% confidence!\t\n", filename, *confidence)
			_, _ = fmt.Fprintf(table, "\t  %v +/- %v\t\n", d.Delta, d.Error)
			_, _ = fmt.Fprintf(table, "\t  %v%% +/- %v%%\t\n", d.RelDelta*100, d.RelError*100)
			_, _ = fmt.Fprintf(table, "\t  (Student's t, pooled s = %v)\t\n", d.StdDev)
		} else {
			_, _ = fmt.Fprintf(table, "%s\tNo difference proven at %v%% confidence.\t\n", filename, *confidence)
		}
	}

	_ = table.Flush()
}

func readData(controlFilename string, experimentFilenames []string) ([]float64, map[string][]float64, error) {
	controlData, err := readFile(controlFilename, *column, *delimiter)
	if err != nil {
		return nil, nil, err
	}

	experimentData := make(map[string][]float64)

	for _, filename := range experimentFilenames {
		expData, err := readFile(filename, *column, *delimiter)
		if err != nil {
			return nil, nil, err
		}

		experimentData[filename] = expData
	}

	return controlData, experimentData, nil
}

func printChart(
	experimentFilenames []string, controlFilename string, controlData []float64,
	experimentData map[string][]float64,
) {
	c := chart.BoxChart{}
	c.XRange.Fixed(-1, float64(len(experimentFilenames))+1, 1)
	c.XRange.Category = make([]string, len(experimentFilenames)+1)
	c.XRange.Category[0] = path.Base(controlFilename)

	for i, file := range experimentFilenames {
		c.XRange.Category[i+1] = path.Base(file)
	}

	c.AddSet(0, controlData, true)

	for i, filename := range experimentFilenames {
		c.AddSet(float64(i+1), experimentData[filename], true)
	}

	txt := txtg.New(*width, *height)
	c.Plot(txt)
	fmt.Println(txt)
}

func readFile(filename string, col int, del string) ([]float64, error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}

	defer func() { _ = f.Close() }()

	r := csv.NewReader(f)
	r.Comma = []rune(del)[0]

	records, err := r.ReadAll()
	if err != nil {
		return nil, err
	}

	data := make([]float64, len(records))

	for i, s := range records {
		n, err := strconv.ParseFloat(s[col], 64)
		if err != nil {
			return nil, err
		}

		data[i] = n
	}

	return data, nil
}
<commit_msg>Tidy tidy.<commit_after>// tinystat is used to compare two or more sets of measurements (e.g., runs of a
// multiple runs of benchmarks of two possible implementations) and determine if
// they are statistically different. It's inspired largely by FreeBSD's ministat
// (written by Poul-Henning Kamp).
//
// Imagine we have the results of different animals' SAT scores. Each animal
// took the SATs multiple times, and we're assuming that differences between
// each animal's attempts are measurement error (i.e., normally distributed). We
// can test for differences as follows:
//
//     $ tinystat iguana chameleon leopard
//
//      1.5 k  +
//             |
//             |
//             |
//             |
//             |
//       1000  +                                             |
//             |                              |              |
//             |                        +-----------+  +-----------+
//             |              |         |           |  |           |
//             |              |         |           |  +-----*-----+
//             |              |         |     *     |  |           |
//        500  +              |         +-----------+  +-----------+
//             |        +-----------+   |           |        |
//             |        |     *     |   +-----------+
//             |        +-----------+         |
//             |        +-----------+         |
//          0  +--------------|-----------------------------------------------
//                         iguana         chameleon       leopard
//
//     Experiment  Results
//     chameleon   No difference proven at 95% confidence.
//     leopard     Difference at 95% confidence!
//                   343.5 +/- 292.63453863922877
//                   114.5% +/- 97.54484621307626%
//                   (Student's t, pooled s = 238.9799344943192)
//
// As you can see, despite the superficial differences between the iguana's
// scores and the chameleon's scores, there is no statistically significant
// difference between the two at a 95% confidence level. The leopard, on the
// other hand, has statistically significantly different scores.
package main

import (
	"encoding/csv"
	"flag"
	"fmt"
	"os"
	"path"
	"strconv"
	"text/tabwriter"

	"github.com/codahale/tinystat"
	"github.com/vdobler/chart"
	"github.com/vdobler/chart/txtg"
)

//nolint:gochecknoglobals // main package
var (
	confidence = flag.Float64("confidence", 95, "confidence level for analysis (0,100)")
	column     = flag.Int("column", 0, "the CSV column to analyze")
	delimiter  = flag.String("delimiter", ",", "the CSV delimiter to use")
	noChart    = flag.Bool("no_chart", false, "don't display the box chart")
	width      = flag.Int("width", 74, "width of box chart, in chars")
	height     = flag.Int("height", 20, "height of box chart, in chars")
	v          = flag.Bool("version", false, "display the version number")
	version    = "dev" // injected by goreleaser
)

func main() {
	flag.Usage = func() {
		_, _ = fmt.Fprintf(os.Stderr, "Usage: tinystat <control.csv> [<experiment.csv>...] [options] \n\n")

		flag.PrintDefaults()
	}
	flag.Parse()

	if *v {
		fmt.Println(version)
		return
	}

	args := flag.Args()
	if len(args) < 1 {
		flag.Usage()
		os.Exit(-1)
	}

	controlFilename := args[0]
	experimentFilenames := args[1:]

	// read the data
	controlData, experimentData, err := readData(controlFilename, experimentFilenames)
	if err != nil {
		_, _ = fmt.Fprintln(os.Stderr, err)
		os.Exit(-1)
	}

	// chart the data
	if !*noChart {
		printChart(experimentFilenames, controlFilename, controlData, experimentData)
	}

	// compare the data
	if len(experimentFilenames) > 0 {
		printComparison(controlData, experimentFilenames, experimentData)
	}
}

func printComparison(
	controlData []float64, experimentFilenames []string, experimentData map[string][]float64,
) {
	control := tinystat.Summarize(controlData)
	table := &tabwriter.Writer{}
	table.Init(os.Stdout, 2, 0, 2, ' ', 0)
	_, _ = fmt.Fprintln(table, "Experiment\tResults\t")

	for _, filename := range experimentFilenames {
		experimental := tinystat.Summarize(experimentData[filename])
		d := tinystat.Compare(control, experimental, *confidence)

		if d.Significant() {
			_, _ = fmt.Fprintf(table, "%s\tDifference at %v%% confidence!\t\n", filename, *confidence)
			_, _ = fmt.Fprintf(table, "\t  %v +/- %v\t\n", d.Delta, d.Error)
			_, _ = fmt.Fprintf(table, "\t  %v%% +/- %v%%\t\n", d.RelDelta*100, d.RelError*100)
			_, _ = fmt.Fprintf(table, "\t  (Student's t, pooled s = %v)\t\n", d.StdDev)
		} else {
			_, _ = fmt.Fprintf(table, "%s\tNo difference proven at %v%% confidence.\t\n", filename, *confidence)
		}
	}

	_ = table.Flush()
}

func readData(controlFilename string, experimentFilenames []string) ([]float64, map[string][]float64, error) {
	controlData, err := readFile(controlFilename, *column, *delimiter)
	if err != nil {
		return nil, nil, err
	}

	experimentData := make(map[string][]float64)

	for _, filename := range experimentFilenames {
		expData, err := readFile(filename, *column, *delimiter)
		if err != nil {
			return nil, nil, err
		}

		experimentData[filename] = expData
	}

	return controlData, experimentData, nil
}

func printChart(
	experimentFilenames []string, controlFilename string, controlData []float64,
	experimentData map[string][]float64,
) {
	c := chart.BoxChart{}
	c.XRange.Fixed(-1, float64(len(experimentFilenames))+1, 1)
	c.XRange.Category = make([]string, len(experimentFilenames)+1)
	c.XRange.Category[0] = path.Base(controlFilename)

	for i, file := range experimentFilenames {
		c.XRange.Category[i+1] = path.Base(file)
	}

	c.AddSet(0, controlData, true)

	for i, filename := range experimentFilenames {
		c.AddSet(float64(i+1), experimentData[filename], true)
	}

	txt := txtg.New(*width, *height)
	c.Plot(txt)
	fmt.Println(txt)
}

func readFile(filename string, col int, del string) ([]float64, error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}

	defer func() { _ = f.Close() }()

	r := csv.NewReader(f)
	r.Comma = []rune(del)[0]

	records, err := r.ReadAll()
	if err != nil {
		return nil, err
	}

	data := make([]float64, len(records))

	for i, s := range records {
		n, err := strconv.ParseFloat(s[col], 64)
		if err != nil {
			return nil, err
		}

		data[i] = n
	}

	return data, nil
}
<|endoftext|>


<commit_before>// Package colormap implements a basic color mapping library.
package colormap

import "image/color"

// Scale returns red, green, blue values represented as float64 in range [0..1].
func Scale(val, vmin, vmax float64) (float64, float64, float64) {
	var (
		vrange float64

		red   float64 = 1
		green float64 = 1
		blue  float64 = 1
	)

	if val < vmin {
		val = vmin
	}
	if val > vmax {
		val = vmax
	}
	vrange = vmax - vmin

	if val < (vmin + 0.25*vrange) {
		red = 0
		green = 4 * (val - vmin) / vrange
	} else if val < (vmin + 0.5*vrange) {
		red = 0
		blue = 1 + 4*(vmin+0.25*vrange)/vrange
	} else if val < (vmin+0.75*vrange)/vrange {
		blue = 0
		red = 4 * (val - vmin - 0.5*vrange) / vrange
	} else {
		blue = 0
		green = 1 + 4*(vmin+0.75*vrange-val)/vrange
	}
	return red, green, blue
}

// RGBA transforms val from given range to a non-alpha color.Color.
// Depending on val resulting color belongs to range from blue(vmin) to red(v max).
func RGBA(val, vmin, vmax float64) color.Color {
	// Note: alpha value is always 255.
	var m float64 = 255
	r, g, b := Scale(val, vmin, vmax)
	return color.RGBA{uint8(r * m), uint8(g * m), uint8(b * m), uint8(m)}
}
<commit_msg>Added doc string for package colormap.<commit_after>// Package colormap implements a basic color mapping library.
// With givin value, value minimum and value maximum (given range) returns
// RGB color from Blue for minimum value to Red for maximum value.
// Intermediate colors such as green, yellow and more might also be returned for
// values between minimum and maximum.
package colormap

import "image/color"

// Scale returns red, green, blue values represented as float64 in range [0..1].
func Scale(val, vmin, vmax float64) (float64, float64, float64) {
	var (
		vrange float64

		red   float64 = 1
		green float64 = 1
		blue  float64 = 1
	)

	if val < vmin {
		val = vmin
	}
	if val > vmax {
		val = vmax
	}
	vrange = vmax - vmin

	if val < (vmin + 0.25*vrange) {
		red = 0
		green = 4 * (val - vmin) / vrange
	} else if val < (vmin + 0.5*vrange) {
		red = 0
		blue = 1 + 4*(vmin+0.25*vrange)/vrange
	} else if val < (vmin+0.75*vrange)/vrange {
		blue = 0
		red = 4 * (val - vmin - 0.5*vrange) / vrange
	} else {
		blue = 0
		green = 1 + 4*(vmin+0.75*vrange-val)/vrange
	}
	return red, green, blue
}

// Colorize returns color.RGBA as interface color.Color where alpha set to 0xFF.
func Colorize(val, vmin, vmax float64) color.Color {
	// Note: alpha value is always 255.
	var m float64 = 255
	r, g, b := Scale(val, vmin, vmax)
	return color.RGBA{uint8(r * m), uint8(g * m), uint8(b * m), uint8(m)}
}
<|endoftext|>


<commit_before>// Copyright 2016 Albert Nigmatzianov. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package commands

import (
	"os"
	"runtime"
	"strings"

	"github.com/bogem/nehm/applescript"
	"github.com/bogem/nehm/config"
	"github.com/bogem/nehm/ui"
	"github.com/bogem/nehm/util"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
)

var RootCmd = listCommand

// Variables used in flags
var (
	limit, offset                       uint
	dlFolder, itunesPlaylist, permalink string
)

func Execute() {
	RootCmd.AddCommand(getCommand)
	RootCmd.AddCommand(searchCommand)
	RootCmd.AddCommand(versionCommand)
	RootCmd.Execute()
}

// addCommonFlags adds common flags related to download tracks.
func addCommonFlags(cmd *cobra.Command) {
	cmd.Flags().StringVarP(&dlFolder, "dlFolder", "f", "", "filesystem path to download folder")

	if runtime.GOOS == "darwin" {
		cmd.Flags().StringVarP(&itunesPlaylist, "itunesPlaylist", "i", "", "name of iTunes playlist")
	}
}

func addLimitFlag(cmd *cobra.Command) {
	cmd.Flags().UintVarP(&limit, "limit", "l", 10, "count of tracks on each page")
}

func addOffsetFlag(cmd *cobra.Command) {
	cmd.Flags().UintVarP(&offset, "offset", "o", 0, "offset relative to first like")
}

func addPermalinkFlag(cmd *cobra.Command) {
	cmd.Flags().StringVarP(&permalink, "permalink", "p", "", "user's permalink")
}

// initializeConfig initializes a config with flags.
func initializeConfig(cmd *cobra.Command) {
	err := config.ReadInConfig()
	if err == config.ErrNotExist {
		ui.Warning("There is no config file. Read README to configure nehm")
	} else {
		ui.Term("", err)
	}

	loadDefaultSettings()

	initializeDlFolder(cmd)
	initializePermalink(cmd)
	if runtime.GOOS == "darwin" {
		initializeItunesPlaylist(cmd)
	}
}

func loadDefaultSettings() {
	config.SetDefault("dlFolder", os.Getenv("HOME"))
	config.SetDefault("itunesPlaylist", "")
}

func flagChanged(fs *pflag.FlagSet, key string) bool {
	flag := fs.Lookup(key)
	if flag == nil {
		return false
	}
	return flag.Changed
}

// initializeDlFolder initializes dlFolder value. If there is no dlFolder
// set up, then dlFolder is set to HOME env variable.
func initializeDlFolder(cmd *cobra.Command) {
	var df string
	if flagChanged(cmd.Flags(), "dlFolder") {
		df = dlFolder
	} else {
		df = config.Get("dlFolder")
	}
	config.Set("dlFolder", util.SanitizePath(df))
}

// initializePermalink initializes permalink value. If there is no permalink
// set up, then program is terminating.
func initializePermalink(cmd *cobra.Command) {
	if flagChanged(cmd.Flags(), "permalink") {
		config.Set("permalink", permalink)
	} else if config.Get("permalink") == "" {
		ui.Term("You didn't set a permalink. Use flag '-p' or set permalink in config file.\nTo know, what is permalink, read FAQ.", nil)
	}
}

// initializeItunesPlaylist initializes itunesPlaylist value. If there is no
// itunesPlaylist set up, then itunesPlaylist set up to blank string. Blank
// string is the sign, what tracks should not to be added to iTunes.
func initializeItunesPlaylist(cmd *cobra.Command) {
	var playlist string
	if flagChanged(cmd.Flags(), "itunesPlaylist") {
		playlist = itunesPlaylist
	} else {
		playlist = config.Get("itunesPlaylist")
	}
	if playlist != "" {
		playlistsList, err := applescript.ListOfPlaylists()
		if err != nil {
			ui.Term("Couldn't get list of playlists", err)
		}
		if !strings.Contains(playlistsList, playlist) {
			ui.Term("Playlist "+playlist+" doesn't exist. Please enter correct name.", nil)
		}
	}
}
<commit_msg>Fix iTunes playlist initialization<commit_after>// Copyright 2016 Albert Nigmatzianov. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package commands

import (
	"os"
	"runtime"
	"strings"

	"github.com/bogem/nehm/applescript"
	"github.com/bogem/nehm/config"
	"github.com/bogem/nehm/ui"
	"github.com/bogem/nehm/util"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
)

var RootCmd = listCommand

// Variables used in flags
var (
	limit, offset                       uint
	dlFolder, itunesPlaylist, permalink string
)

func Execute() {
	RootCmd.AddCommand(getCommand)
	RootCmd.AddCommand(searchCommand)
	RootCmd.AddCommand(versionCommand)
	RootCmd.Execute()
}

// addCommonFlags adds common flags related to download tracks.
func addCommonFlags(cmd *cobra.Command) {
	cmd.Flags().StringVarP(&dlFolder, "dlFolder", "f", "", "filesystem path to download folder")

	if runtime.GOOS == "darwin" {
		cmd.Flags().StringVarP(&itunesPlaylist, "itunesPlaylist", "i", "", "name of iTunes playlist")
	}
}

func addLimitFlag(cmd *cobra.Command) {
	cmd.Flags().UintVarP(&limit, "limit", "l", 10, "count of tracks on each page")
}

func addOffsetFlag(cmd *cobra.Command) {
	cmd.Flags().UintVarP(&offset, "offset", "o", 0, "offset relative to first like")
}

func addPermalinkFlag(cmd *cobra.Command) {
	cmd.Flags().StringVarP(&permalink, "permalink", "p", "", "user's permalink")
}

// initializeConfig initializes a config with flags.
func initializeConfig(cmd *cobra.Command) {
	err := config.ReadInConfig()
	if err == config.ErrNotExist {
		ui.Warning("There is no config file. Read README to configure nehm")
	} else {
		ui.Term("", err)
	}

	loadDefaultSettings()

	initializeDlFolder(cmd)
	initializePermalink(cmd)
	if runtime.GOOS == "darwin" {
		initializeItunesPlaylist(cmd)
	}
}

func loadDefaultSettings() {
	config.SetDefault("dlFolder", os.Getenv("HOME"))
	config.SetDefault("itunesPlaylist", "")
}

func flagChanged(fs *pflag.FlagSet, key string) bool {
	flag := fs.Lookup(key)
	if flag == nil {
		return false
	}
	return flag.Changed
}

// initializeDlFolder initializes dlFolder value. If there is no dlFolder
// set up, then dlFolder is set to HOME env variable.
func initializeDlFolder(cmd *cobra.Command) {
	var df string
	if flagChanged(cmd.Flags(), "dlFolder") {
		df = dlFolder
	} else {
		df = config.Get("dlFolder")
	}
	config.Set("dlFolder", util.SanitizePath(df))
}

// initializePermalink initializes permalink value. If there is no permalink
// set up, then program is terminating.
func initializePermalink(cmd *cobra.Command) {
	if flagChanged(cmd.Flags(), "permalink") {
		config.Set("permalink", permalink)
	} else if config.Get("permalink") == "" {
		ui.Term("You didn't set a permalink. Use flag '-p' or set permalink in config file.\nTo know, what is permalink, read FAQ.", nil)
	}
}

// initializeItunesPlaylist initializes itunesPlaylist value. If there is no
// itunesPlaylist set up, then itunesPlaylist set up to blank string. Blank
// string is the sign, what tracks should not to be added to iTunes.
func initializeItunesPlaylist(cmd *cobra.Command) {
	var playlist string
	if flagChanged(cmd.Flags(), "itunesPlaylist") {
		playlist = itunesPlaylist
	} else {
		playlist = config.Get("itunesPlaylist")
	}
	if playlist != "" {
		playlistsList, err := applescript.ListOfPlaylists()
		if err != nil {
			ui.Term("Couldn't get list of playlists", err)
		}
		if !strings.Contains(playlistsList, playlist) {
			ui.Term("Playlist "+playlist+" doesn't exist. Please enter correct name.", nil)
		}
	}
	config.Set("itunesPlaylist", playlist)
}
<|endoftext|>


<commit_before>package commands

import "os"
import "os/exec"
import "strings"
import "fmt"
import "bytes"
import "path/filepath"
import "bufio"

import alias "../alias/table"
import "../conio"
import "../history"
import "../interpreter"
import "../ls"

import "github.com/shiena/ansicolor"

func cmd_exit(cmd *exec.Cmd) (interpreter.NextT, error) {
	return interpreter.SHUTDOWN, nil
}

func getHome() string {
	home := os.Getenv("HOME")
	if home != "" {
		return home
	}
	homeDrive := os.Getenv("HOMEDRIVE")
	if homeDrive != "" {
		homePath := os.Getenv("HOMEPATH")
		if homePath != "" {
			return homeDrive + homePath
		}
	}
	return ""
}

func cmd_pwd(cmd *exec.Cmd) (interpreter.NextT, error) {
	wd, _ := os.Getwd()
	fmt.Fprintln(cmd.Stdout, wd)
	return interpreter.CONTINUE, nil
}

func cmd_cd(cmd *exec.Cmd) (interpreter.NextT, error) {
	if len(cmd.Args) >= 2 {
		err := os.Chdir(cmd.Args[1])
		return interpreter.CONTINUE, err
	}
	home := getHome()
	if home != "" {
		err := os.Chdir(home)
		return interpreter.CONTINUE, err
	}
	return cmd_pwd(cmd)
}

func cmd_ls(cmd *exec.Cmd) (interpreter.NextT, error) {
	err := ls.Main(cmd.Args[1:], ansicolor.NewAnsiColorWriter(cmd.Stdout))
	return interpreter.CONTINUE, err
}

func cmd_set(cmd *exec.Cmd) (interpreter.NextT, error) {
	if len(cmd.Args) <= 1 {
		for _, val := range os.Environ() {
			fmt.Fprintln(cmd.Stdout, val)
		}
		return interpreter.CONTINUE, nil
	}
	for _, arg := range cmd.Args[1:] {
		eqlPos := strings.Index(arg, "=")
		if eqlPos < 0 {
			fmt.Fprintf(cmd.Stdout, "%s=%s\n", arg, os.Getenv(arg))
		} else {
			os.Setenv(arg[:eqlPos], arg[eqlPos+1:])
		}
	}
	return interpreter.CONTINUE, nil
}

func cmd_source(cmd *exec.Cmd) (interpreter.NextT, error) {
	envTxtPath := filepath.Join(
		os.TempDir(),
		fmt.Sprintf("nyagos-%d.tmp", os.Getpid()))

	var buffer bytes.Buffer

	buffer.WriteString(cmd.Args[1])
	buffer.WriteString(" & set > ")
	buffer.WriteString(envTxtPath)

	args := make([]string, 3)
	args[0] = os.Getenv("COMSPEC")
	args[1] = "/c"
	args[2] = buffer.String()
	var cmd2 exec.Cmd
	cmd2.Path = args[0]
	cmd2.Args = args
	cmd2.Env = nil
	cmd2.Dir = ""
	if err := cmd2.Run(); err != nil {
		return interpreter.CONTINUE, err
	}
	fp, err := os.Open(envTxtPath)
	if err != nil {
		return interpreter.CONTINUE, nil
	}
	defer os.Remove(envTxtPath)
	defer fp.Close()

	scr := bufio.NewScanner(fp)
	for scr.Scan() {
		line := scr.Text()
		eqlPos := strings.Index(line, "=")
		if eqlPos > 0 {
			os.Setenv(line[:eqlPos], line[eqlPos+1:])
		}
	}
	return interpreter.CONTINUE, nil
}

func cmd_echo(cmd *exec.Cmd) (interpreter.NextT, error) {
	fmt.Fprintln(cmd.Stdout, strings.Join(cmd.Args[1:], " "))
	return interpreter.CONTINUE, nil
}

func cmd_cls(cmd *exec.Cmd) (interpreter.NextT, error) {
	conio.Cls()
	return interpreter.CONTINUE, nil
}

func cmd_alias(cmd *exec.Cmd) (interpreter.NextT, error) {
	if len(cmd.Args) <= 1 {
		for key, val := range alias.Table {
			fmt.Fprintf(cmd.Stdout, "%s=%s\n", key, val)
		}
		return interpreter.CONTINUE, nil
	}
	for _, args := range cmd.Args[1:] {
		if eqlPos := strings.IndexRune(args, '='); eqlPos >= 0 {
			key := args[0:eqlPos]
			val := args[eqlPos+1:]
			if len(val) > 0 {
				alias.Table[strings.ToLower(key)] = val
			} else {
				delete(alias.Table, strings.ToLower(key))
			}
		} else {
			key := strings.ToLower(args)
			val := alias.Table[key]

			fmt.Printf("%s=%s\n", key, val)
		}
	}
	return interpreter.CONTINUE, nil
}

var buildInCmd = map[string]func(cmd *exec.Cmd) (interpreter.NextT, error){
	".":       cmd_source,
	"alias":   cmd_alias,
	"cd":      cmd_cd,
	"cls":     cmd_cls,
	"echo":    cmd_echo,
	"exit":    cmd_exit,
	"history": history.CmdHistory,
	"ls":      cmd_ls,
	"pwd":     cmd_pwd,
	"set":     cmd_set,
	"source":  cmd_source,
}

func Exec(cmd *exec.Cmd, IsBackground bool) (interpreter.NextT, error) {
	name := strings.ToLower(cmd.Args[0])
	if len(name) == 2 && strings.HasSuffix(name, ":") {
		os.Chdir(name + ".")
		return interpreter.CONTINUE, nil
	}
	function, ok := buildInCmd[name]
	if ok {
		newArgs := make([]string, 0)
		for _, arg1 := range cmd.Args {
			matches, _ := filepath.Glob(arg1)
			if matches == nil {
				newArgs = append(newArgs, arg1)
			} else {
				for _, s := range matches {
					newArgs = append(newArgs, s)
				}
			}
		}
		cmd.Args = newArgs
		if IsBackground {
			go function(cmd)
			return interpreter.CONTINUE, nil
		} else {
			next, err := function(cmd)
			return next, err
		}
	} else {
		return interpreter.THROUGH, nil
	}
}
<commit_msg>存在しないドライブへの移動がエラーになっていなかったのを修正<commit_after>package commands

import "os"
import "os/exec"
import "strings"
import "fmt"
import "bytes"
import "path/filepath"
import "bufio"

import alias "../alias/table"
import "../conio"
import "../history"
import "../interpreter"
import "../ls"

import "github.com/shiena/ansicolor"

func cmd_exit(cmd *exec.Cmd) (interpreter.NextT, error) {
	return interpreter.SHUTDOWN, nil
}

func getHome() string {
	home := os.Getenv("HOME")
	if home != "" {
		return home
	}
	homeDrive := os.Getenv("HOMEDRIVE")
	if homeDrive != "" {
		homePath := os.Getenv("HOMEPATH")
		if homePath != "" {
			return homeDrive + homePath
		}
	}
	return ""
}

func cmd_pwd(cmd *exec.Cmd) (interpreter.NextT, error) {
	wd, _ := os.Getwd()
	fmt.Fprintln(cmd.Stdout, wd)
	return interpreter.CONTINUE, nil
}

func cmd_cd(cmd *exec.Cmd) (interpreter.NextT, error) {
	if len(cmd.Args) >= 2 {
		err := os.Chdir(cmd.Args[1])
		return interpreter.CONTINUE, err
	}
	home := getHome()
	if home != "" {
		err := os.Chdir(home)
		return interpreter.CONTINUE, err
	}
	return cmd_pwd(cmd)
}

func cmd_ls(cmd *exec.Cmd) (interpreter.NextT, error) {
	err := ls.Main(cmd.Args[1:], ansicolor.NewAnsiColorWriter(cmd.Stdout))
	return interpreter.CONTINUE, err
}

func cmd_set(cmd *exec.Cmd) (interpreter.NextT, error) {
	if len(cmd.Args) <= 1 {
		for _, val := range os.Environ() {
			fmt.Fprintln(cmd.Stdout, val)
		}
		return interpreter.CONTINUE, nil
	}
	for _, arg := range cmd.Args[1:] {
		eqlPos := strings.Index(arg, "=")
		if eqlPos < 0 {
			fmt.Fprintf(cmd.Stdout, "%s=%s\n", arg, os.Getenv(arg))
		} else {
			os.Setenv(arg[:eqlPos], arg[eqlPos+1:])
		}
	}
	return interpreter.CONTINUE, nil
}

func cmd_source(cmd *exec.Cmd) (interpreter.NextT, error) {
	envTxtPath := filepath.Join(
		os.TempDir(),
		fmt.Sprintf("nyagos-%d.tmp", os.Getpid()))

	var buffer bytes.Buffer

	buffer.WriteString(cmd.Args[1])
	buffer.WriteString(" & set > ")
	buffer.WriteString(envTxtPath)

	args := make([]string, 3)
	args[0] = os.Getenv("COMSPEC")
	args[1] = "/c"
	args[2] = buffer.String()
	var cmd2 exec.Cmd
	cmd2.Path = args[0]
	cmd2.Args = args
	cmd2.Env = nil
	cmd2.Dir = ""
	if err := cmd2.Run(); err != nil {
		return interpreter.CONTINUE, err
	}
	fp, err := os.Open(envTxtPath)
	if err != nil {
		return interpreter.CONTINUE, nil
	}
	defer os.Remove(envTxtPath)
	defer fp.Close()

	scr := bufio.NewScanner(fp)
	for scr.Scan() {
		line := scr.Text()
		eqlPos := strings.Index(line, "=")
		if eqlPos > 0 {
			os.Setenv(line[:eqlPos], line[eqlPos+1:])
		}
	}
	return interpreter.CONTINUE, nil
}

func cmd_echo(cmd *exec.Cmd) (interpreter.NextT, error) {
	fmt.Fprintln(cmd.Stdout, strings.Join(cmd.Args[1:], " "))
	return interpreter.CONTINUE, nil
}

func cmd_cls(cmd *exec.Cmd) (interpreter.NextT, error) {
	conio.Cls()
	return interpreter.CONTINUE, nil
}

func cmd_alias(cmd *exec.Cmd) (interpreter.NextT, error) {
	if len(cmd.Args) <= 1 {
		for key, val := range alias.Table {
			fmt.Fprintf(cmd.Stdout, "%s=%s\n", key, val)
		}
		return interpreter.CONTINUE, nil
	}
	for _, args := range cmd.Args[1:] {
		if eqlPos := strings.IndexRune(args, '='); eqlPos >= 0 {
			key := args[0:eqlPos]
			val := args[eqlPos+1:]
			if len(val) > 0 {
				alias.Table[strings.ToLower(key)] = val
			} else {
				delete(alias.Table, strings.ToLower(key))
			}
		} else {
			key := strings.ToLower(args)
			val := alias.Table[key]

			fmt.Printf("%s=%s\n", key, val)
		}
	}
	return interpreter.CONTINUE, nil
}

var buildInCmd = map[string]func(cmd *exec.Cmd) (interpreter.NextT, error){
	".":       cmd_source,
	"alias":   cmd_alias,
	"cd":      cmd_cd,
	"cls":     cmd_cls,
	"echo":    cmd_echo,
	"exit":    cmd_exit,
	"history": history.CmdHistory,
	"ls":      cmd_ls,
	"pwd":     cmd_pwd,
	"set":     cmd_set,
	"source":  cmd_source,
}

func Exec(cmd *exec.Cmd, IsBackground bool) (interpreter.NextT, error) {
	name := strings.ToLower(cmd.Args[0])
	if len(name) == 2 && strings.HasSuffix(name, ":") {
		err := os.Chdir(name + ".")
		return interpreter.CONTINUE, err
	}
	function, ok := buildInCmd[name]
	if ok {
		newArgs := make([]string, 0)
		for _, arg1 := range cmd.Args {
			matches, _ := filepath.Glob(arg1)
			if matches == nil {
				newArgs = append(newArgs, arg1)
			} else {
				for _, s := range matches {
					newArgs = append(newArgs, s)
				}
			}
		}
		cmd.Args = newArgs
		if IsBackground {
			go function(cmd)
			return interpreter.CONTINUE, nil
		} else {
			next, err := function(cmd)
			return next, err
		}
	} else {
		return interpreter.THROUGH, nil
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os/exec"
	"strings"

	"github.com/blang/semver"
)

func getTag(match ...string) (string, *semver.PRVersion) {
	args := append([]string{
		"describe", "--tags",
	}, match...)
	var tag []byte
	var err error
	if tag, err = exec.Command("git", args...).Output(); err != nil {
		if _, err := exec.Command("git", "fetch", "--tags").Output(); err != nil {
			return "", nil
		}
		if tag, err = exec.Command("git", args...).Output(); err != nil {
			return "", nil
		}
	}
	tagParts := strings.Split(string(tag), "-")
	if len(tagParts) == 3 {
		if ahead, err := semver.NewPRVersion(tagParts[1]); err == nil {
			return tagParts[0], &ahead
		}
	} else if len(tagParts) == 4 {
		if ahead, err := semver.NewPRVersion(tagParts[2]); err == nil {
			return tagParts[0] + "-" + tagParts[1], &ahead
		}
	}

	return string(tag), nil
}

func main() {
	// Find the last vX.X.X Tag and get how many builds we are ahead of it.
	versionStr, ahead := getTag("--match", "v*")
	version, err := semver.ParseTolerant(versionStr)
	if err != nil {
		// no version tag found so just return what ever we can find.
		fmt.Println("0.0.0-unknown")
		return
	}
	// Get the tag of the current revision.
	tag, _ := getTag("--exact-match")
	if tag == versionStr {
		// Seems that we are going to build a release.
		// So the version number should already be correct.
		fmt.Println(version.String())
		return
	}

	// If we don't have any tag assume "dev"
	if tag == "" {
		tag = "dev"
	}
	// Get the most likely next version:
	if !strings.Contains(version.String(), "rc") {
		version.Patch = version.Patch + 1
	}

	if pr, err := semver.NewPRVersion(tag); err == nil {
		// append the tag as pre-release name
		version.Pre = append(version.Pre, pr)
	}

	if ahead != nil {
		// if we know how many commits we are ahead of the last release, append that too.
		version.Pre = append(version.Pre, *ahead)
	}

	fmt.Println(version.String())
}
<commit_msg>Only fetch tags if no tags are found at all<commit_after>package main

import (
	"fmt"
	"os/exec"
	"strings"

	"github.com/blang/semver"
)

func getTag(match ...string) (string, *semver.PRVersion) {
	args := append([]string{
		"describe", "--tags",
	}, match...)
	if tag, err := exec.Command("git", args...).Output(); err != nil {
		return "", nil
	} else {
		tagParts := strings.Split(string(tag), "-")
		if len(tagParts) == 3 {
			if ahead, err := semver.NewPRVersion(tagParts[1]); err == nil {
				return tagParts[0], &ahead
			}
		} else if len(tagParts) == 4 {
			if ahead, err := semver.NewPRVersion(tagParts[2]); err == nil {
				return tagParts[0] + "-" + tagParts[1], &ahead
			}
		}

		return string(tag), nil
	}
}

func main() {
	if tags, err := exec.Command("git", "tag").Output(); err != nil || len(tags) == 0 {
		// no tags found -- fetch them
		exec.Command("git", "fetch", "--tags").Run()
	}
	// Find the last vX.X.X Tag and get how many builds we are ahead of it.
	versionStr, ahead := getTag("--match", "v*")
	version, err := semver.ParseTolerant(versionStr)
	if err != nil {
		// no version tag found so just return what ever we can find.
		fmt.Println("0.0.0-unknown")
		return
	}
	// Get the tag of the current revision.
	tag, _ := getTag("--exact-match")
	if tag == versionStr {
		// Seems that we are going to build a release.
		// So the version number should already be correct.
		fmt.Println(version.String())
		return
	}

	// If we don't have any tag assume "dev"
	if tag == "" {
		tag = "dev"
	}
	// Get the most likely next version:
	if !strings.Contains(version.String(), "rc") {
		version.Patch = version.Patch + 1
	}

	if pr, err := semver.NewPRVersion(tag); err == nil {
		// append the tag as pre-release name
		version.Pre = append(version.Pre, pr)
	}

	if ahead != nil {
		// if we know how many commits we are ahead of the last release, append that too.
		version.Pre = append(version.Pre, *ahead)
	}

	fmt.Println(version.String())
}
<|endoftext|>


<commit_before>package timeago

import (
	"testing"
	"time"
)

func TestNow(t *testing.T) {
	tm := time.Now()
	s := TimeAgo(tm)
	if s != "right now" {
		t.Error("Expected: 'right now' but got ", s)
	}
}

func TestFewSeconds(t *testing.T) {
	ago := time.Unix(time.Now().Unix()-4, 0)
	s := TimeAgo(ago)
	if s != "few seconds ago" {
		t.Error("Expected: 'few seconds ago' but got ", s)
	}
}
<commit_msg>Add x seconds ago<commit_after>package timeago

import (
	"testing"
	"time"
)

func TestNow(t *testing.T) {
	tm := time.Now()
	s := TimeAgo(tm)
	if s != "right now" {
		t.Error("Expected: 'right now' but got ", s)
	}
}

func TestFewSeconds(t *testing.T) {
	ago := time.Unix(time.Now().Unix()-4, 0)
	s := TimeAgo(ago)
	if s != "few seconds ago" {
		t.Error("Expected: 'few seconds ago' but got ", s)
	}
}

func TestSeconds(t *testing.T) {
	ago := time.Unix(time.Now().Unix()-15, 0)
	s := TimeAgo(ago)
	if s != "15 seconds ago" {
		t.Error("Expected: '15 seconds ago' but got ", s)
	}
}
<|endoftext|>