<commit_before>package main

import (
	"log"
	"net/url"
	"os"
	"time"

	_ "github.com/grafana/loki/pkg/build"
	"github.com/grafana/loki/pkg/logcli/client"
	"github.com/grafana/loki/pkg/logcli/labelquery"
	"github.com/grafana/loki/pkg/logcli/output"
	"github.com/grafana/loki/pkg/logcli/query"
	"github.com/prometheus/common/config"
	"github.com/prometheus/common/version"

	"gopkg.in/alecthomas/kingpin.v2"
)

var (
	app        = kingpin.New("logcli", "A command-line for loki.").Version(version.Print("logcli"))
	quiet      = app.Flag("quiet", "suppress everything but log lines").Default("false").Short('q').Bool()
	statistics = app.Flag("stats", "show query statistics").Default("false").Bool()
	outputMode = app.Flag("output", "specify output mode [default, raw, jsonl]").Default("default").Short('o').Enum("default", "raw", "jsonl")
	timezone   = app.Flag("timezone", "Specify the timezone to use when formatting output timestamps [Local, UTC]").Default("Local").Short('z').Enum("Local", "UTC")

	queryClient = newQueryClient(app)

	queryCmd   = app.Command("query", "Run a LogQL query.")
	rangeQuery = newQuery(false, queryCmd)
	tail       = queryCmd.Flag("tail", "Tail the logs").Short('t').Default("false").Bool()
	delayFor   = queryCmd.Flag("delay-for", "Delay in tailing by number of seconds to accumulate logs for re-ordering").Default("0").Int()

	instantQueryCmd = app.Command("instant-query", "Run an instant LogQL query")
	instantQuery    = newQuery(true, instantQueryCmd)

	labelsCmd = app.Command("labels", "Find values for a given label.")
	labelName = labelsCmd.Arg("label", "The name of the label.").HintAction(hintActionLabelNames).String()
)

func main() {
	log.SetOutput(os.Stderr)

	cmd := kingpin.MustParse(app.Parse(os.Args[1:]))

	switch cmd {
	case queryCmd.FullCommand():
		location, err := time.LoadLocation(*timezone)
		if err != nil {
			log.Fatalf("Unable to load timezone '%s': %s", *timezone, err)
		}

		outputOptions := &output.LogOutputOptions{
			Timezone: location,
			NoLabels: rangeQuery.NoLabels,
		}

		out, err := output.NewLogOutput(*outputMode, outputOptions)
		if err != nil {
			log.Fatalf("Unable to create log output: %s", err)
		}

		if *tail {
			rangeQuery.TailQuery(*delayFor, queryClient, out)
		} else {
			rangeQuery.DoQuery(queryClient, out, *statistics)
		}
	case instantQueryCmd.FullCommand():
		location, err := time.LoadLocation(*timezone)
		if err != nil {
			log.Fatalf("Unable to load timezone '%s': %s", *timezone, err)
		}

		outputOptions := &output.LogOutputOptions{
			Timezone: location,
			NoLabels: instantQuery.NoLabels,
		}

		out, err := output.NewLogOutput(*outputMode, outputOptions)
		if err != nil {
			log.Fatalf("Unable to create log output: %s", err)
		}

		instantQuery.DoQuery(queryClient, out, *statistics)
	case labelsCmd.FullCommand():
		q := newLabelQuery(*labelName, *quiet)

		q.DoLabels(queryClient)
	}
}

func hintActionLabelNames() []string {
	q := newLabelQuery("", *quiet)

	return q.ListLabels(queryClient)
}

func newQueryClient(app *kingpin.Application) *client.Client {
	client := &client.Client{
		TLSConfig: config.TLSConfig{},
	}

	// extract host
	addressAction := func(c *kingpin.ParseContext) error {
		u, err := url.Parse(client.Address)
		if err != nil {
			return err
		}
		client.TLSConfig.ServerName = u.Host
		return nil
	}

	app.Flag("addr", "Server address. Can also be set using LOKI_ADDR env var.").Default("http://localhost:3100").Envar("LOKI_ADDR").Action(addressAction).StringVar(&client.Address)
	app.Flag("username", "Username for HTTP basic auth. Can also be set using LOKI_USERNAME env var.").Default("").Envar("LOKI_USERNAME").StringVar(&client.Username)
	app.Flag("password", "Password for HTTP basic auth. Can also be set using LOKI_PASSWORD env var.").Default("").Envar("LOKI_PASSWORD").StringVar(&client.Password)
	app.Flag("ca-cert", "Path to the server Certificate Authority. Can also be set using LOKI_CA_CERT_PATH env var.").Default("").Envar("LOKI_CA_CERT_PATH").StringVar(&client.TLSConfig.CAFile)
	app.Flag("tls-skip-verify", "Server certificate TLS skip verify.").Default("false").BoolVar(&client.TLSConfig.InsecureSkipVerify)
	app.Flag("cert", "Path to the client certificate. Can also be set using LOKI_CLIENT_CERT_PATH env var.").Default("").Envar("LOKI_CLIENT_CERT_PATH").StringVar(&client.TLSConfig.CertFile)
	app.Flag("key", "Path to the client certificate key. Can also be set using LOKI_CLIENT_KEY_PATH env var.").Default("").Envar("LOKI_CLIENT_KEY_PATH").StringVar(&client.TLSConfig.KeyFile)
	app.Flag("org-id", "org ID header to be substituted for auth").StringVar(&client.OrgID)

	return client
}

func newLabelQuery(labelName string, quiet bool) *labelquery.LabelQuery {
	return &labelquery.LabelQuery{
		LabelName: labelName,
		Quiet:     quiet,
	}
}

func newQuery(instant bool, cmd *kingpin.CmdClause) *query.Query {
	// calculcate query range from cli params
	var now, from, to string
	var since time.Duration

	query := &query.Query{}

	// executed after all command flags are parsed
	cmd.Action(func(c *kingpin.ParseContext) error {

		if instant {
			query.SetInstant(mustParse(now, time.Now()))
		} else {
			defaultEnd := time.Now()
			defaultStart := defaultEnd.Add(-since)

			query.Start = mustParse(from, defaultStart)
			query.End = mustParse(to, defaultEnd)
		}

		return nil
	})

	cmd.Arg("query", "eg '{foo=\"bar\",baz=~\".*blip\"} |~ \".*error.*\"'").Required().StringVar(&query.QueryString)
	cmd.Flag("limit", "Limit on number of entries to print.").Default("30").IntVar(&query.Limit)
	if instant {
		cmd.Flag("now", "Time at which to execute the instant query.").StringVar(&now)
	} else {
		cmd.Flag("since", "Lookback window.").Default("1h").DurationVar(&since)
		cmd.Flag("from", "Start looking for logs at this absolute time (inclusive)").StringVar(&from)
		cmd.Flag("to", "Stop looking for logs at this absolute time (exclusive)").StringVar(&to)
		cmd.Flag("step", "Query resolution step width").DurationVar(&query.Step)
	}

	cmd.Flag("forward", "Scan forwards through logs.").Default("false").BoolVar(&query.Forward)
	cmd.Flag("no-labels", "Do not print any labels").Default("false").BoolVar(&query.NoLabels)
	cmd.Flag("exclude-label", "Exclude labels given the provided key during output.").StringsVar(&query.IgnoreLabelsKey)
	cmd.Flag("include-label", "Include labels given the provided key during output.").StringsVar(&query.ShowLabelsKey)
	cmd.Flag("labels-length", "Set a fixed padding to labels").Default("0").IntVar(&query.FixedLabelsLen)

	return query
}

func mustParse(t string, defaultTime time.Time) time.Time {
	if t == "" {
		return defaultTime
	}

	ret, err := time.Parse(time.RFC3339Nano, t)

	if err != nil {
		log.Fatalf("Unable to parse time %v", err)
	}

	return ret
}
<commit_msg>Fix logcli --quiet parameter parsing issue (#1682)<commit_after>package main

import (
	"log"
	"net/url"
	"os"
	"time"

	_ "github.com/grafana/loki/pkg/build"
	"github.com/grafana/loki/pkg/logcli/client"
	"github.com/grafana/loki/pkg/logcli/labelquery"
	"github.com/grafana/loki/pkg/logcli/output"
	"github.com/grafana/loki/pkg/logcli/query"
	"github.com/prometheus/common/config"
	"github.com/prometheus/common/version"

	"gopkg.in/alecthomas/kingpin.v2"
)

var (
	app        = kingpin.New("logcli", "A command-line for loki.").Version(version.Print("logcli"))
	quiet      = app.Flag("quiet", "suppress everything but log lines").Default("false").Short('q').Bool()
	statistics = app.Flag("stats", "show query statistics").Default("false").Bool()
	outputMode = app.Flag("output", "specify output mode [default, raw, jsonl]").Default("default").Short('o').Enum("default", "raw", "jsonl")
	timezone   = app.Flag("timezone", "Specify the timezone to use when formatting output timestamps [Local, UTC]").Default("Local").Short('z').Enum("Local", "UTC")

	queryClient = newQueryClient(app)

	queryCmd   = app.Command("query", "Run a LogQL query.")
	rangeQuery = newQuery(false, queryCmd)
	tail       = queryCmd.Flag("tail", "Tail the logs").Short('t').Default("false").Bool()
	delayFor   = queryCmd.Flag("delay-for", "Delay in tailing by number of seconds to accumulate logs for re-ordering").Default("0").Int()

	instantQueryCmd = app.Command("instant-query", "Run an instant LogQL query")
	instantQuery    = newQuery(true, instantQueryCmd)

	labelsCmd = app.Command("labels", "Find values for a given label.")
	labelName = labelsCmd.Arg("label", "The name of the label.").HintAction(hintActionLabelNames).String()
)

func main() {
	log.SetOutput(os.Stderr)

	cmd := kingpin.MustParse(app.Parse(os.Args[1:]))

	switch cmd {
	case queryCmd.FullCommand():
		location, err := time.LoadLocation(*timezone)
		if err != nil {
			log.Fatalf("Unable to load timezone '%s': %s", *timezone, err)
		}

		outputOptions := &output.LogOutputOptions{
			Timezone: location,
			NoLabels: rangeQuery.NoLabels,
		}

		out, err := output.NewLogOutput(*outputMode, outputOptions)
		if err != nil {
			log.Fatalf("Unable to create log output: %s", err)
		}

		if *tail {
			rangeQuery.TailQuery(*delayFor, queryClient, out)
		} else {
			rangeQuery.DoQuery(queryClient, out, *statistics)
		}
	case instantQueryCmd.FullCommand():
		location, err := time.LoadLocation(*timezone)
		if err != nil {
			log.Fatalf("Unable to load timezone '%s': %s", *timezone, err)
		}

		outputOptions := &output.LogOutputOptions{
			Timezone: location,
			NoLabels: instantQuery.NoLabels,
		}

		out, err := output.NewLogOutput(*outputMode, outputOptions)
		if err != nil {
			log.Fatalf("Unable to create log output: %s", err)
		}

		instantQuery.DoQuery(queryClient, out, *statistics)
	case labelsCmd.FullCommand():
		q := newLabelQuery(*labelName, *quiet)

		q.DoLabels(queryClient)
	}
}

func hintActionLabelNames() []string {
	q := newLabelQuery("", *quiet)

	return q.ListLabels(queryClient)
}

func newQueryClient(app *kingpin.Application) *client.Client {
	client := &client.Client{
		TLSConfig: config.TLSConfig{},
	}

	// extract host
	addressAction := func(c *kingpin.ParseContext) error {
		u, err := url.Parse(client.Address)
		if err != nil {
			return err
		}
		client.TLSConfig.ServerName = u.Host
		return nil
	}

	app.Flag("addr", "Server address. Can also be set using LOKI_ADDR env var.").Default("http://localhost:3100").Envar("LOKI_ADDR").Action(addressAction).StringVar(&client.Address)
	app.Flag("username", "Username for HTTP basic auth. Can also be set using LOKI_USERNAME env var.").Default("").Envar("LOKI_USERNAME").StringVar(&client.Username)
	app.Flag("password", "Password for HTTP basic auth. Can also be set using LOKI_PASSWORD env var.").Default("").Envar("LOKI_PASSWORD").StringVar(&client.Password)
	app.Flag("ca-cert", "Path to the server Certificate Authority. Can also be set using LOKI_CA_CERT_PATH env var.").Default("").Envar("LOKI_CA_CERT_PATH").StringVar(&client.TLSConfig.CAFile)
	app.Flag("tls-skip-verify", "Server certificate TLS skip verify.").Default("false").BoolVar(&client.TLSConfig.InsecureSkipVerify)
	app.Flag("cert", "Path to the client certificate. Can also be set using LOKI_CLIENT_CERT_PATH env var.").Default("").Envar("LOKI_CLIENT_CERT_PATH").StringVar(&client.TLSConfig.CertFile)
	app.Flag("key", "Path to the client certificate key. Can also be set using LOKI_CLIENT_KEY_PATH env var.").Default("").Envar("LOKI_CLIENT_KEY_PATH").StringVar(&client.TLSConfig.KeyFile)
	app.Flag("org-id", "org ID header to be substituted for auth").StringVar(&client.OrgID)

	return client
}

func newLabelQuery(labelName string, quiet bool) *labelquery.LabelQuery {
	return &labelquery.LabelQuery{
		LabelName: labelName,
		Quiet:     quiet,
	}
}

func newQuery(instant bool, cmd *kingpin.CmdClause) *query.Query {
	// calculcate query range from cli params
	var now, from, to string
	var since time.Duration

	query := &query.Query{}

	// executed after all command flags are parsed
	cmd.Action(func(c *kingpin.ParseContext) error {

		if instant {
			query.SetInstant(mustParse(now, time.Now()))
		} else {
			defaultEnd := time.Now()
			defaultStart := defaultEnd.Add(-since)

			query.Start = mustParse(from, defaultStart)
			query.End = mustParse(to, defaultEnd)
		}
		query.Quiet = *quiet
		return nil
	})

	cmd.Arg("query", "eg '{foo=\"bar\",baz=~\".*blip\"} |~ \".*error.*\"'").Required().StringVar(&query.QueryString)
	cmd.Flag("limit", "Limit on number of entries to print.").Default("30").IntVar(&query.Limit)
	if instant {
		cmd.Flag("now", "Time at which to execute the instant query.").StringVar(&now)
	} else {
		cmd.Flag("since", "Lookback window.").Default("1h").DurationVar(&since)
		cmd.Flag("from", "Start looking for logs at this absolute time (inclusive)").StringVar(&from)
		cmd.Flag("to", "Stop looking for logs at this absolute time (exclusive)").StringVar(&to)
		cmd.Flag("step", "Query resolution step width").DurationVar(&query.Step)
	}

	cmd.Flag("forward", "Scan forwards through logs.").Default("false").BoolVar(&query.Forward)
	cmd.Flag("no-labels", "Do not print any labels").Default("false").BoolVar(&query.NoLabels)
	cmd.Flag("exclude-label", "Exclude labels given the provided key during output.").StringsVar(&query.IgnoreLabelsKey)
	cmd.Flag("include-label", "Include labels given the provided key during output.").StringsVar(&query.ShowLabelsKey)
	cmd.Flag("labels-length", "Set a fixed padding to labels").Default("0").IntVar(&query.FixedLabelsLen)

	return query
}

func mustParse(t string, defaultTime time.Time) time.Time {
	if t == "" {
		return defaultTime
	}

	ret, err := time.Parse(time.RFC3339Nano, t)

	if err != nil {
		log.Fatalf("Unable to parse time %v", err)
	}

	return ret
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/x509"
	"fmt"
	"math"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/docker/notary/keystoremanager"
	"github.com/docker/notary/trustmanager"

	"github.com/spf13/cobra"
)

func init() {
	cmdKeys.AddCommand(cmdKeysRemoveRootKey)
	cmdKeys.AddCommand(cmdKeysGenerateRootKey)
}

var cmdKeys = &cobra.Command{
	Use:   "keys",
	Short: "Operates on root keys.",
	Long:  "operations on private root keys.",
	Run:   keysList,
}

var cmdKeysRemoveRootKey = &cobra.Command{
	Use:   "remove [ keyID ]",
	Short: "Removes the root key with the given keyID.",
	Long:  "remove the root key with the given keyID from the local host.",
	Run:   keysRemoveRootKey,
}

var cmdKeysGenerateRootKey = &cobra.Command{
	Use:   "generate [ algorithm ]",
	Short: "Generates a new root key with a given algorithm.",
	Long:  "generates a new root key with a given algorithm.",
	Run:   keysGenerateRootKey,
}

// keysRemoveRootKey deletes a root private key based on ID
func keysRemoveRootKey(cmd *cobra.Command, args []string) {
	if len(args) < 1 {
		cmd.Usage()
		fatalf("must specify the key ID of the root key to remove")
	}

	keyID := args[0]
	if len(keyID) != 64 {
		fatalf("please enter a valid root key ID")
	}
	parseConfig()

	keyStoreManager, err := keystoremanager.NewKeyStoreManager(trustDir, retriever)
	if err != nil {
		fatalf("failed to create a new truststore manager with directory: %s", trustDir)
	}

	// List all the keys about to be removed
	fmt.Printf("Are you sure you want to remove the following key?\n%s\n (yes/no)\n", keyID)

	// Ask for confirmation before removing keys
	confirmed := askConfirm()
	if !confirmed {
		fatalf("aborting action.")
	}

	// Remove all the keys under the Global Unique Name
	err = keyStoreManager.RootKeyStore().RemoveKey(keyID)
	if err != nil {
		fatalf("failed to remove root key with key ID: %s", keyID)
	}

	fmt.Printf("Root key %s removed\n", keyID)
}

func keysList(cmd *cobra.Command, args []string) {
	if len(args) > 0 {
		cmd.Usage()
		os.Exit(1)
	}

	parseConfig()

	keyStoreManager, err := keystoremanager.NewKeyStoreManager(trustDir, retriever)
	if err != nil {
		fatalf("failed to create a new truststore manager with directory: %s", trustDir)
	}

	fmt.Println("")
	fmt.Println("# Trusted Certificates:")
	trustedCerts := keyStoreManager.TrustedCertificateStore().GetCertificates()
	for _, c := range trustedCerts {
		printCert(c)
	}

	fmt.Println("")
	fmt.Println("# Root keys: ")
	for _, k := range keyStoreManager.RootKeyStore().ListKeys() {
		fmt.Println(k)
	}

	fmt.Println("")
	fmt.Println("# Signing keys: ")
	for _, k := range keyStoreManager.NonRootKeyStore().ListKeys() {
		printKey(k)
	}
}

func keysGenerateRootKey(cmd *cobra.Command, args []string) {
	if len(args) < 1 {
		cmd.Usage()
		fatalf("must specify an Algorithm (RSA, ECDSA)")
	}

	algorithm := args[0]
	allowedCiphers := map[string]bool{
		"rsa":   true,
		"ecdsa": true,
	}

	if !allowedCiphers[strings.ToLower(algorithm)] {
		fatalf("algorithm not allowed, possible values are: RSA, ECDSA")
	}

	parseConfig()

	keyStoreManager, err := keystoremanager.NewKeyStoreManager(trustDir, retriever)
	if err != nil {
		fatalf("failed to create a new truststore manager with directory: %s", trustDir)
	}

	keyID, err := keyStoreManager.GenRootKey(algorithm)
	if err != nil {
		fatalf("failed to create a new root key: %v", err)
	}

	fmt.Printf("Generated new %s key with keyID: %s\n", algorithm, keyID)
}

func printCert(cert *x509.Certificate) {
	timeDifference := cert.NotAfter.Sub(time.Now())
	certID, err := trustmanager.FingerprintCert(cert)
	if err != nil {
		fatalf("could not fingerprint certificate: %v", err)
	}

	fmt.Printf("%s %s (expires in: %v days)\n", cert.Subject.CommonName, certID, math.Floor(timeDifference.Hours()/24))
}

func printKey(keyPath string) {
	keyID := filepath.Base(keyPath)
	gun := filepath.Dir(keyPath)
	fmt.Printf("%s %s\n", gun, keyID)
}

func askConfirm() bool {
	var res string
	_, err := fmt.Scanln(&res)
	if err != nil {
		return false
	}
	if strings.EqualFold(res, "y") || strings.EqualFold(res, "yes") {
		return true
	}
	return false
}
<commit_msg>Add key import and export commands<commit_after>package main

import (
	"archive/zip"
	"crypto/x509"
	"fmt"
	"math"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/docker/notary/keystoremanager"
	"github.com/docker/notary/pkg/passphrase"
	"github.com/docker/notary/trustmanager"

	"github.com/spf13/cobra"
)

func init() {
	cmdKeys.AddCommand(cmdKeysRemoveRootKey)
	cmdKeys.AddCommand(cmdKeysGenerateRootKey)

	cmdKeysExport.Flags().StringVarP(&keysExportGUN, "gun", "g", "", "Globally unique name to export keys for. A new password will be set for all the keys. Output format is a zip archive.")
	cmdKeys.AddCommand(cmdKeysExport)
	cmdKeys.AddCommand(cmdKeysExportRoot)
	cmdKeys.AddCommand(cmdKeysImport)
	cmdKeys.AddCommand(cmdKeysImportRoot)
}

var cmdKeys = &cobra.Command{
	Use:   "keys",
	Short: "Operates on keys.",
	Long:  "operations on private keys.",
	Run:   keysList,
}

var cmdKeysRemoveRootKey = &cobra.Command{
	Use:   "remove [ keyID ]",
	Short: "Removes the root key with the given keyID.",
	Long:  "remove the root key with the given keyID from the local host.",
	Run:   keysRemoveRootKey,
}

var cmdKeysGenerateRootKey = &cobra.Command{
	Use:   "generate [ algorithm ]",
	Short: "Generates a new root key with a given algorithm.",
	Long:  "generates a new root key with a given algorithm.",
	Run:   keysGenerateRootKey,
}

var keysExportGUN string

var cmdKeysExport = &cobra.Command{
	Use:   "export [ filename ]",
	Short: "Exports keys to a ZIP file.",
	Long:  "exports a collection of keys. The keys are reencrypted with a new passphrase. The output is a ZIP file.",
	Run:   keysExport,
}

var cmdKeysExportRoot = &cobra.Command{
	Use:   "export-root [ keyID ] [ filename ]",
	Short: "Exports given root key to a file.",
	Long:  "exports a root key, without reencrypting. The output is a PEM file.",
	Run:   keysExportRoot,
}

var cmdKeysImport = &cobra.Command{
	Use:   "import [ filename ]",
	Short: "Imports keys from a ZIP file.",
	Long:  "imports one or more keys from a ZIP file.",
	Run:   keysImport,
}

var cmdKeysImportRoot = &cobra.Command{
	Use:   "import-root [ keyID ] [ filename ]",
	Short: "Imports root key.",
	Long:  "imports a root key from a PEM file.",
	Run:   keysImportRoot,
}

// keysRemoveRootKey deletes a root private key based on ID
func keysRemoveRootKey(cmd *cobra.Command, args []string) {
	if len(args) < 1 {
		cmd.Usage()
		fatalf("must specify the key ID of the root key to remove")
	}

	keyID := args[0]
	if len(keyID) != 64 {
		fatalf("please enter a valid root key ID")
	}
	parseConfig()

	keyStoreManager, err := keystoremanager.NewKeyStoreManager(trustDir, retriever)
	if err != nil {
		fatalf("failed to create a new truststore manager with directory: %s", trustDir)
	}

	// List all the keys about to be removed
	fmt.Printf("Are you sure you want to remove the following key?\n%s\n (yes/no)\n", keyID)

	// Ask for confirmation before removing keys
	confirmed := askConfirm()
	if !confirmed {
		fatalf("aborting action.")
	}

	// Remove all the keys under the Global Unique Name
	err = keyStoreManager.RootKeyStore().RemoveKey(keyID)
	if err != nil {
		fatalf("failed to remove root key with key ID: %s", keyID)
	}

	fmt.Printf("Root key %s removed\n", keyID)
}

func keysList(cmd *cobra.Command, args []string) {
	if len(args) > 0 {
		cmd.Usage()
		os.Exit(1)
	}

	parseConfig()

	keyStoreManager, err := keystoremanager.NewKeyStoreManager(trustDir, retriever)
	if err != nil {
		fatalf("failed to create a new truststore manager with directory: %s", trustDir)
	}

	fmt.Println("")
	fmt.Println("# Trusted Certificates:")
	trustedCerts := keyStoreManager.TrustedCertificateStore().GetCertificates()
	for _, c := range trustedCerts {
		printCert(c)
	}

	fmt.Println("")
	fmt.Println("# Root keys: ")
	for _, k := range keyStoreManager.RootKeyStore().ListKeys() {
		fmt.Println(k)
	}

	fmt.Println("")
	fmt.Println("# Signing keys: ")
	for _, k := range keyStoreManager.NonRootKeyStore().ListKeys() {
		printKey(k)
	}
}

func keysGenerateRootKey(cmd *cobra.Command, args []string) {
	if len(args) < 1 {
		cmd.Usage()
		fatalf("must specify an Algorithm (RSA, ECDSA)")
	}

	algorithm := args[0]
	allowedCiphers := map[string]bool{
		"rsa":   true,
		"ecdsa": true,
	}

	if !allowedCiphers[strings.ToLower(algorithm)] {
		fatalf("algorithm not allowed, possible values are: RSA, ECDSA")
	}

	parseConfig()

	keyStoreManager, err := keystoremanager.NewKeyStoreManager(trustDir, retriever)
	if err != nil {
		fatalf("failed to create a new truststore manager with directory: %s", trustDir)
	}

	keyID, err := keyStoreManager.GenRootKey(algorithm)
	if err != nil {
		fatalf("failed to create a new root key: %v", err)
	}

	fmt.Printf("Generated new %s key with keyID: %s\n", algorithm, keyID)
}

// keysExport exports a collection of keys to a ZIP file
func keysExport(cmd *cobra.Command, args []string) {
	if len(args) < 1 {
		cmd.Usage()
		fatalf("must specify output filename for export")
	}

	exportFilename := args[0]

	parseConfig()

	keyStoreManager, err := keystoremanager.NewKeyStoreManager(trustDir, retriever)
	if err != nil {
		fatalf("failed to create a new truststore manager with directory: %s", trustDir)
	}

	exportFile, err := os.Create(exportFilename)
	if err != nil {
		fatalf("error creating output file: %v", err)
	}

	// Must use a different passphrase retriever to avoid caching the
	// unlocking passphrase and reusing that.
	exportRetriever := passphrase.PromptRetriever()
	if keysExportGUN != "" {
		err = keyStoreManager.ExportKeysByGUN(exportFile, keysExportGUN, exportRetriever)
	} else {
		err = keyStoreManager.ExportAllKeys(exportFile, exportRetriever)
	}

	exportFile.Close()

	if err != nil {
		fatalf("error exporting keys: %v", err)
		os.Remove(exportFilename)
	}
}

// keysExportRoot exports a root key by ID to a PEM file
func keysExportRoot(cmd *cobra.Command, args []string) {
	if len(args) < 2 {
		cmd.Usage()
		fatalf("must specify key ID and output filename for export")
	}

	keyID := args[0]
	exportFilename := args[1]

	if len(keyID) != 64 {
		fatalf("please specify a valid root key ID")
	}

	parseConfig()

	keyStoreManager, err := keystoremanager.NewKeyStoreManager(trustDir, retriever)
	if err != nil {
		fatalf("failed to create a new truststore manager with directory: %s", trustDir)
	}

	exportFile, err := os.Create(exportFilename)
	if err != nil {
		fatalf("error creating output file: %v", err)
	}
	err = keyStoreManager.ExportRootKey(exportFile, keyID)
	exportFile.Close()
	if err != nil {
		fatalf("error exporting root key: %v", err)
		os.Remove(exportFilename)
	}
}

// keysImport imports keys from a ZIP file
func keysImport(cmd *cobra.Command, args []string) {
	if len(args) < 1 {
		cmd.Usage()
		fatalf("must specify input filename for import")
	}

	importFilename := args[0]

	parseConfig()

	keyStoreManager, err := keystoremanager.NewKeyStoreManager(trustDir, retriever)
	if err != nil {
		fatalf("failed to create a new truststore manager with directory: %s", trustDir)
	}

	zipReader, err := zip.OpenReader(importFilename)
	if err != nil {
		fatalf("opening file for import: %v", err)
	}
	defer zipReader.Close()

	err = keyStoreManager.ImportKeysZip(zipReader.Reader)

	if err != nil {
		fatalf("error importing keys: %v", err)
	}
}

// keysImportRoot imports a root key from a PEM file
func keysImportRoot(cmd *cobra.Command, args []string) {
	if len(args) < 2 {
		cmd.Usage()
		fatalf("must specify key ID and input filename for import")
	}

	keyID := args[0]
	importFilename := args[1]

	if len(keyID) != 64 {
		fatalf("please specify a valid root key ID")
	}

	parseConfig()

	keyStoreManager, err := keystoremanager.NewKeyStoreManager(trustDir, retriever)
	if err != nil {
		fatalf("failed to create a new truststore manager with directory: %s", trustDir)
	}

	importFile, err := os.Open(importFilename)
	if err != nil {
		fatalf("opening file for import: %v", err)
	}
	defer importFile.Close()

	err = keyStoreManager.ImportRootKey(importFile, keyID)

	if err != nil {
		fatalf("error importing root key: %v", err)
	}
}

func printCert(cert *x509.Certificate) {
	timeDifference := cert.NotAfter.Sub(time.Now())
	certID, err := trustmanager.FingerprintCert(cert)
	if err != nil {
		fatalf("could not fingerprint certificate: %v", err)
	}

	fmt.Printf("%s %s (expires in: %v days)\n", cert.Subject.CommonName, certID, math.Floor(timeDifference.Hours()/24))
}

func printKey(keyPath string) {
	keyID := filepath.Base(keyPath)
	gun := filepath.Dir(keyPath)
	fmt.Printf("%s %s\n", gun, keyID)
}

func askConfirm() bool {
	var res string
	_, err := fmt.Scanln(&res)
	if err != nil {
		return false
	}
	if strings.EqualFold(res, "y") || strings.EqualFold(res, "yes") {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>/*
 * Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"context"
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"syscall"

	"github.com/minio/cli"
	"github.com/minio/dsync"
	xhttp "github.com/minio/minio/cmd/http"
	"github.com/minio/minio/cmd/logger"
)

var serverFlags = []cli.Flag{
	cli.StringFlag{
		Name:  "address",
		Value: ":" + globalMinioPort,
		Usage: "Bind to a specific ADDRESS:PORT, ADDRESS can be an IP or hostname.",
	},
}

var serverCmd = cli.Command{
	Name:   "server",
	Usage:  "Start object storage server.",
	Flags:  append(serverFlags, globalFlags...),
	Action: serverMain,
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} {{if .VisibleFlags}}[FLAGS] {{end}}DIR1 [DIR2..]
  {{.HelpName}} {{if .VisibleFlags}}[FLAGS] {{end}}DIR{1...64}

DIR:
  DIR points to a directory on a filesystem. When you want to combine
  multiple drives into a single large system, pass one directory per
  filesystem separated by space. You may also use a '...' convention
  to abbreviate the directory arguments. Remote directories in a
  distributed setup are encoded as HTTP(s) URIs.
{{if .VisibleFlags}}
FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}{{end}}
ENVIRONMENT VARIABLES:
  ACCESS:
     MINIO_ACCESS_KEY: Custom username or access key of minimum 3 characters in length.
     MINIO_SECRET_KEY: Custom password or secret key of minimum 8 characters in length.

  BROWSER:
     MINIO_BROWSER: To disable web browser access, set this value to "off".

  CACHE:
     MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
     MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
     MINIO_CACHE_EXPIRY: Cache expiry duration in days.

  REGION:
     MINIO_REGION: To set custom region. By default all regions are accepted.

  UPDATE:
     MINIO_UPDATE: To turn off in-place upgrades, set this value to "off".

  DOMAIN:
     MINIO_DOMAIN: To enable virtual-host-style requests, set this value to Minio host domain name.

  WORM:
     MINIO_WORM: To turn on Write-Once-Read-Many in server, set this value to "on".

EXAMPLES:
  1. Start minio server on "/home/shared" directory.
      $ {{.HelpName}} /home/shared

  2. Start minio server bound to a specific ADDRESS:PORT.
      $ {{.HelpName}} --address 192.168.1.101:9000 /home/shared

  3. Start minio server and enable virtual-host-style requests.
      $ export MINIO_DOMAIN=mydomain.com
      $ {{.HelpName}} --address mydomain.com:9000 /mnt/export

  4. Start minio server on 64 disks server.
      $ {{.HelpName}} /mnt/export{1...64}

  5. Start distributed minio server on an 8 node setup with 8 drives each. Run following command on all the 8 nodes.
      $ export MINIO_ACCESS_KEY=minio
      $ export MINIO_SECRET_KEY=miniostorage
      $ {{.HelpName}} http://node{1...8}.example.com/mnt/export/{1...8}

  6. Start minio server with edge caching enabled.
     $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
     $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
     $ export MINIO_CACHE_EXPIRY=40
     $ {{.HelpName}} /home/shared
`,
}

func serverHandleCmdArgs(ctx *cli.Context) {
	// Handle common command args.
	handleCommonCmdArgs(ctx)

	// Server address.
	serverAddr := ctx.String("address")
	logger.FatalIf(CheckLocalServerAddr(serverAddr), "Unable to validate passed arguments")

	var setupType SetupType
	var err error

	if len(ctx.Args()) > serverCommandLineArgsMax {
		uErr := uiErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("Invalid total number of endpoints (%d) passed, supported upto 32 unique arguments", len(ctx.Args())))
		logger.FatalIf(uErr, "Unable to validate passed endpoints")
	}

	globalMinioAddr, globalEndpoints, setupType, globalXLSetCount, globalXLSetDriveCount, err = createServerEndpoints(serverAddr, ctx.Args()...)
	logger.FatalIf(err, "Invalid command line arguments")

	globalMinioHost, globalMinioPort = mustSplitHostPort(globalMinioAddr)

	// On macOS, if a process already listens on LOCALIPADDR:PORT, net.Listen() falls back
	// to IPv6 address ie minio will start listening on IPv6 address whereas another
	// (non-)minio process is listening on IPv4 of given port.
	// To avoid this error sutiation we check for port availability.
	logger.FatalIf(checkPortAvailability(globalMinioPort), "Unable to start the server")

	globalIsXL = (setupType == XLSetupType)
	globalIsDistXL = (setupType == DistXLSetupType)
	if globalIsDistXL {
		globalIsXL = true
	}
}

func serverHandleEnvVars() {
	// Handle common environment variables.
	handleCommonEnvVars()

	if serverRegion := os.Getenv("MINIO_REGION"); serverRegion != "" {
		// region Envs are set globally.
		globalIsEnvRegion = true
		globalServerRegion = serverRegion
	}

}

func init() {
	logger.Init(GOPATH)
}

// serverMain handler called for 'minio server' command.
func serverMain(ctx *cli.Context) {
	if (!ctx.IsSet("sets") && !ctx.Args().Present()) || ctx.Args().First() == "help" {
		cli.ShowCommandHelpAndExit(ctx, "server", 1)
	}

	// Disable logging until server initialization is complete, any
	// error during initialization will be shown as a fatal message
	logger.Disable = true

	// Get "json" flag from command line argument and
	// enable json and quite modes if jason flag is turned on.
	jsonFlag := ctx.IsSet("json") || ctx.GlobalIsSet("json")
	if jsonFlag {
		logger.EnableJSON()
	}

	// Get quiet flag from command line argument.
	quietFlag := ctx.IsSet("quiet") || ctx.GlobalIsSet("quiet")
	if quietFlag {
		logger.EnableQuiet()
	}

	logger.RegisterUIError(fmtError)

	// Handle all server command args.
	serverHandleCmdArgs(ctx)

	// Handle all server environment vars.
	serverHandleEnvVars()

	// Create certs path.
	logger.FatalIf(createConfigDir(), "Unable to initialize configuration files")

	// Initialize server config.
	initConfig()

	// Check and load SSL certificates.
	var err error
	globalPublicCerts, globalRootCAs, globalTLSCertificate, globalIsSSL, err = getSSLConfig()
	logger.FatalIf(err, "Unable to load the TLS configuration")

	// Is distributed setup, error out if no certificates are found for HTTPS endpoints.
	if globalIsDistXL {
		if globalEndpoints.IsHTTPS() && !globalIsSSL {
			logger.Fatal(uiErrNoCertsAndHTTPSEndpoints(nil), "Unable to start the server")
		}
		if !globalEndpoints.IsHTTPS() && globalIsSSL {
			logger.Fatal(uiErrCertsAndHTTPEndpoints(nil), "Unable to start the server")
		}
	}

	if !quietFlag {
		// Check for new updates from dl.minio.io.
		mode := globalMinioModeFS
		if globalIsDistXL {
			mode = globalMinioModeDistXL
		} else if globalIsXL {
			mode = globalMinioModeXL
		}
		checkUpdate(mode)
	}

	// Set system resources to maximum.
	logger.LogIf(context.Background(), setMaxResources())

	// Set nodes for dsync for distributed setup.
	if globalIsDistXL {
		globalDsync, err = dsync.New(newDsyncNodes(globalEndpoints))
		if err != nil {
			logger.Fatal(err, "Unable to initialize distributed locking on %s", globalEndpoints)
		}
	}

	// Initialize name space lock.
	initNSLock(globalIsDistXL)

	// Init global heal state
	initAllHealState(globalIsXL)

	// Configure server.
	var handler http.Handler
	handler, err = configureServerHandler(globalEndpoints)
	if err != nil {
		logger.Fatal(uiErrUnexpectedError(err), "Unable to configure one of server's RPC services")
	}

	// Create new notification system.
	globalNotificationSys, err = NewNotificationSys(globalServerConfig, globalEndpoints)
	if err != nil {
		logger.Fatal(err, "Unable to initialize the notification system")
	}

	// Create new policy system.
	globalPolicySys = NewPolicySys()

	// Initialize Admin Peers inter-node communication only in distributed setup.
	initGlobalAdminPeers(globalEndpoints)

	globalHTTPServer = xhttp.NewServer([]string{globalMinioAddr}, handler, globalTLSCertificate)
	globalHTTPServer.ReadTimeout = globalConnReadTimeout
	globalHTTPServer.WriteTimeout = globalConnWriteTimeout
	globalHTTPServer.UpdateBytesReadFunc = globalConnStats.incInputBytes
	globalHTTPServer.UpdateBytesWrittenFunc = globalConnStats.incOutputBytes
	go func() {
		globalHTTPServerErrorCh <- globalHTTPServer.Start()
	}()

	signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM)

	newObject, err := newObjectLayer(globalEndpoints)
	if err != nil {
		globalHTTPServer.Shutdown()
		logger.FatalIf(err, "Unable to initialize backend")
	}

	globalObjLayerMutex.Lock()
	globalObjectAPI = newObject
	globalObjLayerMutex.Unlock()

	// Prints the formatted startup message once object layer is initialized.
	apiEndpoints := getAPIEndpoints(globalMinioAddr)
	printStartupMessage(apiEndpoints)

	// Set uptime time after object layer has initialized.
	globalBootTime = UTCNow()

	// Re-enable logging
	logger.Disable = false

	handleSignals()
}

// Initialize object layer with the supplied disks, objectLayer is nil upon any error.
func newObjectLayer(endpoints EndpointList) (newObject ObjectLayer, err error) {
	// For FS only, directly use the disk.

	isFS := len(endpoints) == 1
	if isFS {
		// Initialize new FS object layer.
		return NewFSObjectLayer(endpoints[0].Path)
	}

	format, err := waitForFormatXL(context.Background(), endpoints[0].IsLocal, endpoints, globalXLSetCount, globalXLSetDriveCount)
	if err != nil {
		return nil, err
	}

	return newXLSets(endpoints, format, len(format.XL.Sets), len(format.XL.Sets[0]))
}
<commit_msg>Support endpoints through env (#5952)<commit_after>/*
 * Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"context"
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"

	"github.com/minio/cli"
	"github.com/minio/dsync"
	xhttp "github.com/minio/minio/cmd/http"
	"github.com/minio/minio/cmd/logger"
)

var serverFlags = []cli.Flag{
	cli.StringFlag{
		Name:  "address",
		Value: ":" + globalMinioPort,
		Usage: "Bind to a specific ADDRESS:PORT, ADDRESS can be an IP or hostname.",
	},
}

var serverCmd = cli.Command{
	Name:   "server",
	Usage:  "Start object storage server.",
	Flags:  append(serverFlags, globalFlags...),
	Action: serverMain,
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} {{if .VisibleFlags}}[FLAGS] {{end}}DIR1 [DIR2..]
  {{.HelpName}} {{if .VisibleFlags}}[FLAGS] {{end}}DIR{1...64}

DIR:
  DIR points to a directory on a filesystem. When you want to combine
  multiple drives into a single large system, pass one directory per
  filesystem separated by space. You may also use a '...' convention
  to abbreviate the directory arguments. Remote directories in a
  distributed setup are encoded as HTTP(s) URIs.
{{if .VisibleFlags}}
FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}{{end}}
ENVIRONMENT VARIABLES:
  ACCESS:
     MINIO_ACCESS_KEY: Custom username or access key of minimum 3 characters in length.
     MINIO_SECRET_KEY: Custom password or secret key of minimum 8 characters in length.

  ENDPOINTS:
     MINIO_ENDPOINTS: List of all endpoints delimited by ' '.

  BROWSER:
     MINIO_BROWSER: To disable web browser access, set this value to "off".

  CACHE:
     MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
     MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
     MINIO_CACHE_EXPIRY: Cache expiry duration in days.

  DOMAIN:
     MINIO_DOMAIN: To enable virtual-host-style requests, set this value to Minio host domain name.

  WORM:
     MINIO_WORM: To turn on Write-Once-Read-Many in server, set this value to "on".

EXAMPLES:
  1. Start minio server on "/home/shared" directory.
      $ {{.HelpName}} /home/shared

  2. Start minio server bound to a specific ADDRESS:PORT.
      $ {{.HelpName}} --address 192.168.1.101:9000 /home/shared

  3. Start minio server and enable virtual-host-style requests.
      $ export MINIO_DOMAIN=mydomain.com
      $ {{.HelpName}} --address mydomain.com:9000 /mnt/export

  4. Start minio server on 64 disks server with endpoints through environment variable.
      $ export MINIO_ENDPOINTS=/mnt/export{1...64}
      $ {{.HelpName}}

  5. Start distributed minio server on an 8 node setup with 8 drives each. Run following command on all the 8 nodes.
      $ export MINIO_ACCESS_KEY=minio
      $ export MINIO_SECRET_KEY=miniostorage
      $ {{.HelpName}} http://node{1...8}.example.com/mnt/export/{1...8}

  6. Start minio server with edge caching enabled.
     $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
     $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
     $ export MINIO_CACHE_EXPIRY=40
     $ {{.HelpName}} /home/shared
`,
}

// Checks if endpoints are either available through environment
// or command line, returns false if both fails.
func endpointsPresent(ctx *cli.Context) bool {
	_, ok := os.LookupEnv("MINIO_ENDPOINTS")
	if !ok {
		ok = ctx.Args().Present()
	}
	return ok
}

func serverHandleCmdArgs(ctx *cli.Context) {
	// Handle common command args.
	handleCommonCmdArgs(ctx)

	// Server address.
	serverAddr := ctx.String("address")
	logger.FatalIf(CheckLocalServerAddr(serverAddr), "Unable to validate passed arguments")

	var setupType SetupType
	var err error

	if len(ctx.Args()) > serverCommandLineArgsMax {
		uErr := uiErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("Invalid total number of endpoints (%d) passed, supported upto 32 unique arguments",
			len(ctx.Args())))
		logger.FatalIf(uErr, "Unable to validate passed endpoints")
	}

	endpoints := strings.Fields(os.Getenv("MINIO_ENDPOINTS"))
	if len(endpoints) > 0 {
		globalMinioAddr, globalEndpoints, setupType, globalXLSetCount, globalXLSetDriveCount, err = createServerEndpoints(serverAddr, endpoints...)
	} else {
		globalMinioAddr, globalEndpoints, setupType, globalXLSetCount, globalXLSetDriveCount, err = createServerEndpoints(serverAddr, ctx.Args()...)
	}
	logger.FatalIf(err, "Invalid command line arguments")

	globalMinioHost, globalMinioPort = mustSplitHostPort(globalMinioAddr)

	// On macOS, if a process already listens on LOCALIPADDR:PORT, net.Listen() falls back
	// to IPv6 address ie minio will start listening on IPv6 address whereas another
	// (non-)minio process is listening on IPv4 of given port.
	// To avoid this error sutiation we check for port availability.
	logger.FatalIf(checkPortAvailability(globalMinioPort), "Unable to start the server")

	globalIsXL = (setupType == XLSetupType)
	globalIsDistXL = (setupType == DistXLSetupType)
	if globalIsDistXL {
		globalIsXL = true
	}
}

func serverHandleEnvVars() {
	// Handle common environment variables.
	handleCommonEnvVars()

	if serverRegion := os.Getenv("MINIO_REGION"); serverRegion != "" {
		// region Envs are set globally.
		globalIsEnvRegion = true
		globalServerRegion = serverRegion
	}

}

func init() {
	logger.Init(GOPATH)
}

// serverMain handler called for 'minio server' command.
func serverMain(ctx *cli.Context) {
	if ctx.Args().First() == "help" || !endpointsPresent(ctx) {
		cli.ShowCommandHelpAndExit(ctx, "server", 1)
	}

	// Disable logging until server initialization is complete, any
	// error during initialization will be shown as a fatal message
	logger.Disable = true

	// Get "json" flag from command line argument and
	// enable json and quite modes if jason flag is turned on.
	jsonFlag := ctx.IsSet("json") || ctx.GlobalIsSet("json")
	if jsonFlag {
		logger.EnableJSON()
	}

	// Get quiet flag from command line argument.
	quietFlag := ctx.IsSet("quiet") || ctx.GlobalIsSet("quiet")
	if quietFlag {
		logger.EnableQuiet()
	}

	logger.RegisterUIError(fmtError)

	// Handle all server command args.
	serverHandleCmdArgs(ctx)

	// Handle all server environment vars.
	serverHandleEnvVars()

	// Create certs path.
	logger.FatalIf(createConfigDir(), "Unable to initialize configuration files")

	// Initialize server config.
	initConfig()

	// Check and load SSL certificates.
	var err error
	globalPublicCerts, globalRootCAs, globalTLSCertificate, globalIsSSL, err = getSSLConfig()
	logger.FatalIf(err, "Unable to load the TLS configuration")

	// Is distributed setup, error out if no certificates are found for HTTPS endpoints.
	if globalIsDistXL {
		if globalEndpoints.IsHTTPS() && !globalIsSSL {
			logger.Fatal(uiErrNoCertsAndHTTPSEndpoints(nil), "Unable to start the server")
		}
		if !globalEndpoints.IsHTTPS() && globalIsSSL {
			logger.Fatal(uiErrCertsAndHTTPEndpoints(nil), "Unable to start the server")
		}
	}

	if !quietFlag {
		// Check for new updates from dl.minio.io.
		mode := globalMinioModeFS
		if globalIsDistXL {
			mode = globalMinioModeDistXL
		} else if globalIsXL {
			mode = globalMinioModeXL
		}
		checkUpdate(mode)
	}

	// Set system resources to maximum.
	logger.LogIf(context.Background(), setMaxResources())

	// Set nodes for dsync for distributed setup.
	if globalIsDistXL {
		globalDsync, err = dsync.New(newDsyncNodes(globalEndpoints))
		if err != nil {
			logger.Fatal(err, "Unable to initialize distributed locking on %s", globalEndpoints)
		}
	}

	// Initialize name space lock.
	initNSLock(globalIsDistXL)

	// Init global heal state
	initAllHealState(globalIsXL)

	// Configure server.
	var handler http.Handler
	handler, err = configureServerHandler(globalEndpoints)
	if err != nil {
		logger.Fatal(uiErrUnexpectedError(err), "Unable to configure one of server's RPC services")
	}

	// Create new notification system.
	globalNotificationSys, err = NewNotificationSys(globalServerConfig, globalEndpoints)
	if err != nil {
		logger.Fatal(err, "Unable to initialize the notification system")
	}

	// Create new policy system.
	globalPolicySys = NewPolicySys()

	// Initialize Admin Peers inter-node communication only in distributed setup.
	initGlobalAdminPeers(globalEndpoints)

	globalHTTPServer = xhttp.NewServer([]string{globalMinioAddr}, handler, globalTLSCertificate)
	globalHTTPServer.ReadTimeout = globalConnReadTimeout
	globalHTTPServer.WriteTimeout = globalConnWriteTimeout
	globalHTTPServer.UpdateBytesReadFunc = globalConnStats.incInputBytes
	globalHTTPServer.UpdateBytesWrittenFunc = globalConnStats.incOutputBytes
	go func() {
		globalHTTPServerErrorCh <- globalHTTPServer.Start()
	}()

	signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM)

	newObject, err := newObjectLayer(globalEndpoints)
	if err != nil {
		globalHTTPServer.Shutdown()
		logger.FatalIf(err, "Unable to initialize backend")
	}

	globalObjLayerMutex.Lock()
	globalObjectAPI = newObject
	globalObjLayerMutex.Unlock()

	// Prints the formatted startup message once object layer is initialized.
	apiEndpoints := getAPIEndpoints(globalMinioAddr)
	printStartupMessage(apiEndpoints)

	// Set uptime time after object layer has initialized.
	globalBootTime = UTCNow()

	// Re-enable logging
	logger.Disable = false

	handleSignals()
}

// Initialize object layer with the supplied disks, objectLayer is nil upon any error.
func newObjectLayer(endpoints EndpointList) (newObject ObjectLayer, err error) {
	// For FS only, directly use the disk.

	isFS := len(endpoints) == 1
	if isFS {
		// Initialize new FS object layer.
		return NewFSObjectLayer(endpoints[0].Path)
	}

	format, err := waitForFormatXL(context.Background(), endpoints[0].IsLocal, endpoints, globalXLSetCount, globalXLSetDriveCount)
	if err != nil {
		return nil, err
	}

	return newXLSets(endpoints, format, len(format.XL.Sets), len(format.XL.Sets[0]))
}
<|endoftext|>


<commit_before>package main

import (
    "flag"
    "fmt"
    "github.com/powell0/projecteuler/problems"
    "sort"
    "time"
)

var displayHelp bool
var displayProblems bool
var displayProblemsFull bool
var problemsToSolve []int

func init() {

    flag.BoolVar(&displayHelp, "h", false, "Display usage information ")
    flag.BoolVar(&displayProblems, "d", false, "Display problem numbers")
    flag.BoolVar(&displayProblemsFull, "D", false, "Display problem numbers and descriptions")

    flag.Parse()
}

func main() {
    // Create an index of the problems in ascending order
    problemList := make([]int, 0, len(problems.Registry))

    for key, _ := range problems.Registry {
        problemList = append(problemList, key)
    }

    sort.Ints(problemList)

    if displayHelp {
        flag.PrintDefaults()
    } else if displayProblems || displayProblemsFull {
        // Iterate through the problems in sorted order
        for _, problemNumber := range problemList {
            problem := problems.Registry[problemNumber]
            fmt.Printf("Problem %d", problem.ID())

            if displayProblemsFull {
                fmt.Printf(": %s", problem.Description())
            }

            fmt.Printf("\n")
        }
    } else {
        start := time.Now()

        // Iterate through the problems in sorted order
        for _, problemNumber := range problemList {
            problem := problems.Registry[problemNumber]
            results, ellapsedTime := solveProblem(problem)

            fmt.Printf("Problem %d solved in %s: %s\n", problem.ID(), ellapsedTime, results)
        }

        fmt.Printf("\n%d problems solved in %s\n", len(problemList), time.Since(start))
    }
}

func solveProblem (problem problems.Problem) (string, time.Duration) {
     start := time.Now()
     results := problem.Solve()

     return results, time.Since(start)
}

<commit_msg>Added command line argument to specify which tests to run<commit_after>package main

import (
    "flag"
    "fmt"
    "github.com/powell0/projecteuler/problems"
    "sort"
    "strconv"
    "strings"
    "time"
)

var displayHelp bool
var displayProblems bool
var displayProblemsFull bool
var problemsToSolve problemSetFlag

func init() {

    flag.BoolVar(&displayHelp, "h", false, "Display usage information ")
    flag.BoolVar(&displayProblems, "d", false, "Display problem numbers")
    flag.BoolVar(&displayProblemsFull, "D", false, "Display problem numbers and descriptions")
    flag.Var(&problemsToSolve, "p", "Only run the given problem numbers (e.g. \"1, 2, 5-9, 10\")")

    flag.Parse()
}

func main() {
    // Create an index of the problems in ascending order
    problemList := make([]int, 0, len(problems.Registry))

    if len(problemsToSolve.problems) > 0 {
        for i := 0; i < len(problemsToSolve.problems); i++ {
            problemList = append(problemList, problemsToSolve.problems[i])
        }
    } else {
        for key, _ := range problems.Registry {
            problemList = append(problemList, key)
        }
    }

    sort.Ints(problemList)

    if displayHelp {
        flag.PrintDefaults()
    } else if displayProblems || displayProblemsFull {
        // Iterate through the problems in sorted order
        for _, problemNumber := range problemList {
            problem := problems.Registry[problemNumber]
            fmt.Printf("Problem %d", problem.ID())

            if displayProblemsFull {
                fmt.Printf(": %s", problem.Description())
            }

            fmt.Printf("\n")
        }
    } else {
        start := time.Now()

        // Iterate through the problems in sorted order
        for _, problemNumber := range problemList {
            problem := problems.Registry[problemNumber]
            results, ellapsedTime := solveProblem(problem)

            fmt.Printf("Problem %d solved in %s: %s\n", problem.ID(), ellapsedTime, results)
        }

        fmt.Printf("\n%d problems solved in %s\n", len(problemList), time.Since(start))
    }
}

func solveProblem (problem problems.Problem) (string, time.Duration) {
     start := time.Now()
     results := problem.Solve()

     return results, time.Since(start)
}

type problemSetFlag struct {
    problems []int
}

func (p *problemSetFlag) String() string {
    var text string

    if len(p.problems) > 0 {

        for i := 0; i < len(p.problems); i++ {
            count := 0

            for j := i + 1; j < len(p.problems); j++ {
                if p.problems[j-1] + 1 != p.problems[j] {
                    break
                }

                count++
            }

            text += strconv.Itoa(p.problems[i])

            if count > 1 {
                i += count
                text += "-" + strconv.Itoa(p.problems[i])
            }

            text += ", "
        }
    }

    if len(text) > 0 {
        text = text[:len(text)-2]
    }

    return text
}

func (p *problemSetFlag) Set(value string) error {
    if len(p.problems) > 0 {
    	return fmt.Errorf("The problems flag is already set")
    }

    problems := strings.Split(value, ",")
    for _, problem := range problems {
    	problemRange := strings.Split(problem, "-")

        if len(problemRange) == 1 {
            problemNumber, err := strconv.Atoi(strings.TrimSpace(problemRange[0]))

            if err == nil {
                p.problems = append(p.problems, problemNumber)
            } else {
                return err
            }

        } else if len(problemRange) == 2 {
            problemNumberStart, err := strconv.Atoi(strings.TrimSpace(problemRange[0]))

            if err != nil {
                return err
            }

            problemNumberEnd, err := strconv.Atoi(strings.TrimSpace(problemRange[1]))

            if err != nil {
                return err
            }

            for i := problemNumberStart; i <= problemNumberEnd; i++ {
                p.problems = append(p.problems, i)
            }
        } else {
            return fmt.Errorf("Invalid problem set ", problem)
        }
    }
    return nil
}
<|endoftext|>


<commit_before>package main

import (
	"compress/gzip"
	"errors"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"strconv"
	"strings"
	"sync"

	"github.com/thomersch/grandine/lib/mvt"
	"github.com/thomersch/grandine/lib/progressbar"
	"github.com/thomersch/grandine/lib/spaten"
	"github.com/thomersch/grandine/lib/spatial"
	"github.com/thomersch/grandine/lib/tile"
)

const indexThreshold = 100000000

type zmLvl []int

func (zm *zmLvl) String() string {
	return fmt.Sprintf("%d", *zm)
}

func (zm *zmLvl) Set(value string) error {
	for _, s := range strings.Split(value, ",") {
		v, err := strconv.Atoi(strings.TrimSpace(s))
		if err != nil {
			return fmt.Errorf("%s (only integer values are allowed)", value)
		}
		*zm = append(*zm, v)
	}
	return nil
}

type bbox spatial.BBox

func (b *bbox) String() string {
	return fmt.Sprintf("%v", *b)
}

func (b *bbox) Set(value string) error {
	var (
		fl    [4]float64
		parts = strings.Split(value, ",")
		err   error
	)
	if len(parts) != 4 {
		return errors.New("bbox takes exactly 4 parameters: SW Lon, SW Lat, NE Lon, NE Lat")
	}
	for i, s := range parts {
		fl[i], err = strconv.ParseFloat(s, 64)
		if err != nil {
			return fmt.Errorf("could not parse bbox expression: %v", err)
		}
	}
	b.SW = spatial.Point{fl[0], fl[1]}
	b.NE = spatial.Point{fl[2], fl[3]}
	return nil
}

var (
	zoomlevels zmLvl
	quiet      *bool
)

func main() {
	source := flag.String("in", "geo.geojson", "file to read from, supported format: spaten")
	target := flag.String("out", "tiles", "path where the tiles will be written")
	defaultLayer := flag.Bool("default-layer", true, "if no layer name is specified in the feature, whether it will be put into a default layer")
	workersNumber := flag.Int("workers", runtime.GOMAXPROCS(0), "number of workers")
	cpuProfile := flag.String("cpuprof", "", "writes CPU profiling data into a file")
	compressTiles := flag.Bool("compress", false, "compress tiles with gzip")
	quiet = flag.Bool("q", false, "argument to use if program should be run in quiet mode with reduced logging")

	flag.Var(&zoomlevels, "zoom", "one or more zoom level of which the tiles will be rendered")
	flag.Parse()

	if len(zoomlevels) == 0 {
		log.Fatal("no zoom levels specified")
	}

	if len(*cpuProfile) != 0 {
		f, err := os.Create(*cpuProfile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	f, err := os.Open(*source)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	err = os.MkdirAll(*target, 0777)
	if err != nil {
		log.Fatal(err)
	}

	log.Println("parsing input...")
	fc := spatial.FeatureCollection{}
	codec := spaten.Codec{}
	codec.Decode(f, &fc)

	if len(fc.Features) == 0 {
		log.Fatal("no features in input file")
	}

	log.Printf("read %d features", len(fc.Features))

	var bboxPts []spatial.Point
	for _, feat := range fc.Features {
		bb := feat.Geometry.BBox()
		bboxPts = append(bboxPts, bb.SW, bb.NE)
	}

	log.Println("determining which tiles need to be generated")
	var tc []tile.ID
	for _, zoomlevel := range zoomlevels {
		tc = append(tc, tile.Coverage(spatial.Line(bboxPts).BBox(), zoomlevel)...)
	}

	var fts spatial.Filterable
	if len(fc.Features)*len(tc) > indexThreshold {
		log.Println("building index...")
		fts = spatial.NewRTreeCollection(fc.Features...)
		log.Println("index complete")
	} else {
		fts = &fc
	}

	log.Printf("starting to generate %d tiles...", len(tc)-1)
	dtw := diskTileWriter{basedir: *target, compressTiles: *compressTiles}
	dlm := defaultLayerMapper{defaultLayer: *defaultLayer}

	var (
		wg       sync.WaitGroup
		ws       = workerSlices(tc, *workersNumber)
		pb, done = progressbar.NewBar(len(tc)-1, len(ws))
	)
	for wrk := 0; wrk < len(ws); wrk++ {
		wg.Add(1)
		go func(i int) {
			generateTiles(ws[i], fts, &dtw, &dlm, pb)
			wg.Done()
		}(wrk)
	}
	wg.Wait()
	done()
}

func workerSlices(tiles []tile.ID, wrkNum int) [][]tile.ID {
	var r [][]tile.ID
	if len(tiles) <= wrkNum {
		for t := 0; t < len(tiles); t++ {
			r = append(r, []tile.ID{tiles[t]})
		}
		return r
	}
	for wrkr := 0; wrkr < wrkNum; wrkr++ {
		start := (len(tiles) / wrkNum) * wrkr
		end := (len(tiles) / wrkNum) * (wrkr + 1)
		if wrkr == wrkNum-1 {
			end = len(tiles)
		}
		r = append(r, tiles[start:end])
	}
	return r
}

type diskTileWriter struct {
	basedir       string
	compressTiles bool
}

func (tw *diskTileWriter) WriteTile(tID tile.ID, buf []byte) error {
	err := os.MkdirAll(filepath.Join(tw.basedir, strconv.Itoa(tID.Z), strconv.Itoa(tID.X)), 0777)
	if err != nil {
		return err
	}
	tf, err := os.Create(filepath.Join(tw.basedir, strconv.Itoa(tID.Z), strconv.Itoa(tID.X), strconv.Itoa(tID.Y)+".mvt"))
	if err != nil {
		return err
	}
	defer tf.Close()

	if tw.compressTiles {
		_, err = gzip.NewWriter(tf).Write(buf)
	} else {
		_, err = tf.Write(buf)
	}
	if err != nil {
		return err
	}
	return nil
}

type defaultLayerMapper struct {
	defaultLayer bool
}

func (dlm *defaultLayerMapper) LayerName(props map[string]interface{}) string {
	if layerName, ok := props["@layer"]; ok {
		return layerName.(string)
	}
	if dlm.defaultLayer {
		return "default"
	}
	return ""
}

type layerMapper interface {
	LayerName(map[string]interface{}) string
}

type tileWriter interface {
	WriteTile(tile.ID, []byte) error
}

func generateTiles(tIDs []tile.ID, features spatial.Filterable, tw tileWriter, lm layerMapper, pb chan<- struct{}) {
	for _, tID := range tIDs {
		// if !*quiet {
		// 	log.Printf("Generating %s", tID)
		// }
		var (
			layers = map[string][]spatial.Feature{}
			ln     string
		)
		tileClipBBox := tID.BBox()

		for _, feat := range features.Filter(tileClipBBox) {
			sf := tile.Resolution(tID.Z, 4096) * 20
			gm := feat.Geometry.Simplify(sf)
			for _, geom := range gm.ClipToBBox(tileClipBBox) {
				feat.Geometry = geom
				ln = lm.LayerName(feat.Props)
				if len(ln) != 0 {
					if _, ok := layers[ln]; !ok {
						layers[ln] = []spatial.Feature{feat}
					} else {
						layers[ln] = append(layers[ln], feat)
					}
				}
			}
		}
		if !anyFeatures(layers) {
			pb <- struct{}{}
			continue
		}
		buf, err := mvt.EncodeTile(layers, tID)
		if err != nil {
			log.Fatal(err)
		}

		err = tw.WriteTile(tID, buf)
		if err != nil {
			log.Fatal(err)
		}
		pb <- struct{}{}
	}
}

func anyFeatures(layers map[string][]spatial.Feature) bool {
	for _, ly := range layers {
		if len(ly) > 0 {
			return true
		}
	}
	return false
}
<commit_msg>cmd/tiler: allow parsing from stdin<commit_after>package main

import (
	"compress/gzip"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"strconv"
	"strings"
	"sync"

	"github.com/thomersch/grandine/lib/mvt"
	"github.com/thomersch/grandine/lib/progressbar"
	"github.com/thomersch/grandine/lib/spaten"
	"github.com/thomersch/grandine/lib/spatial"
	"github.com/thomersch/grandine/lib/tile"
)

const indexThreshold = 100000000

type zmLvl []int

func (zm *zmLvl) String() string {
	return fmt.Sprintf("%d", *zm)
}

func (zm *zmLvl) Set(value string) error {
	for _, s := range strings.Split(value, ",") {
		v, err := strconv.Atoi(strings.TrimSpace(s))
		if err != nil {
			return fmt.Errorf("%s (only integer values are allowed)", value)
		}
		*zm = append(*zm, v)
	}
	return nil
}

type bbox spatial.BBox

func (b *bbox) String() string {
	return fmt.Sprintf("%v", *b)
}

func (b *bbox) Set(value string) error {
	var (
		fl    [4]float64
		parts = strings.Split(value, ",")
		err   error
	)
	if len(parts) != 4 {
		return errors.New("bbox takes exactly 4 parameters: SW Lon, SW Lat, NE Lon, NE Lat")
	}
	for i, s := range parts {
		fl[i], err = strconv.ParseFloat(s, 64)
		if err != nil {
			return fmt.Errorf("could not parse bbox expression: %v", err)
		}
	}
	b.SW = spatial.Point{fl[0], fl[1]}
	b.NE = spatial.Point{fl[2], fl[3]}
	return nil
}

var (
	zoomlevels zmLvl
	quiet      *bool
)

func main() {
	source := flag.String("in", "", "file to read from, supported format: spaten")
	sourceStdIn := flag.Bool("std-in", false, "will read the incoming file from stdin")
	target := flag.String("out", "tiles", "path where the tiles will be written")
	defaultLayer := flag.Bool("default-layer", true, "if no layer name is specified in the feature, whether it will be put into a default layer")
	workersNumber := flag.Int("workers", runtime.GOMAXPROCS(0), "number of workers")
	cpuProfile := flag.String("cpuprof", "", "writes CPU profiling data into a file")
	compressTiles := flag.Bool("compress", false, "compress tiles with gzip")
	quiet = flag.Bool("q", false, "argument to use if program should be run in quiet mode with reduced logging")

	flag.Var(&zoomlevels, "zoom", "one or more zoom level of which the tiles will be rendered")
	flag.Parse()

	if len(*source) != 0 && *sourceStdIn {
		log.Fatal("please specify only one input: either by filename or from stdin")
	}
	if len(*source) == 0 && !*sourceStdIn {
		log.Fatal("no input specified")
	}

	if len(zoomlevels) == 0 {
		log.Fatal("no zoom levels specified")
	}

	if len(*cpuProfile) != 0 {
		f, err := os.Create(*cpuProfile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	var (
		f   io.Reader
		err error
	)

	if len(*source) != 0 {
		f, err = os.Open(*source)
		if err != nil {
			log.Fatal(err)
		}
		defer f.(io.Closer).Close()
	} else {
		f = os.Stdin
	}

	err = os.MkdirAll(*target, 0777)
	if err != nil {
		log.Fatal(err)
	}

	log.Println("parsing input...")
	fc := spatial.FeatureCollection{}
	codec := spaten.Codec{}
	codec.Decode(f, &fc)

	if len(fc.Features) == 0 {
		log.Fatal("no features in input file")
	}

	log.Printf("read %d features", len(fc.Features))

	var bboxPts []spatial.Point
	for _, feat := range fc.Features {
		bb := feat.Geometry.BBox()
		bboxPts = append(bboxPts, bb.SW, bb.NE)
	}

	log.Println("determining which tiles need to be generated")
	var tc []tile.ID
	for _, zoomlevel := range zoomlevels {
		tc = append(tc, tile.Coverage(spatial.Line(bboxPts).BBox(), zoomlevel)...)
	}

	var fts spatial.Filterable
	if len(fc.Features)*len(tc) > indexThreshold {
		log.Println("building index...")
		fts = spatial.NewRTreeCollection(fc.Features...)
		log.Println("index complete")
	} else {
		fts = &fc
	}

	log.Printf("starting to generate %d tiles...", len(tc)-1)
	dtw := diskTileWriter{basedir: *target, compressTiles: *compressTiles}
	dlm := defaultLayerMapper{defaultLayer: *defaultLayer}

	var (
		wg       sync.WaitGroup
		ws       = workerSlices(tc, *workersNumber)
		pb, done = progressbar.NewBar(len(tc)-1, len(ws))
	)
	for wrk := 0; wrk < len(ws); wrk++ {
		wg.Add(1)
		go func(i int) {
			generateTiles(ws[i], fts, &dtw, &dlm, pb)
			wg.Done()
		}(wrk)
	}
	wg.Wait()
	done()
}

func workerSlices(tiles []tile.ID, wrkNum int) [][]tile.ID {
	var r [][]tile.ID
	if len(tiles) <= wrkNum {
		for t := 0; t < len(tiles); t++ {
			r = append(r, []tile.ID{tiles[t]})
		}
		return r
	}
	for wrkr := 0; wrkr < wrkNum; wrkr++ {
		start := (len(tiles) / wrkNum) * wrkr
		end := (len(tiles) / wrkNum) * (wrkr + 1)
		if wrkr == wrkNum-1 {
			end = len(tiles)
		}
		r = append(r, tiles[start:end])
	}
	return r
}

type diskTileWriter struct {
	basedir       string
	compressTiles bool
}

func (tw *diskTileWriter) WriteTile(tID tile.ID, buf []byte) error {
	err := os.MkdirAll(filepath.Join(tw.basedir, strconv.Itoa(tID.Z), strconv.Itoa(tID.X)), 0777)
	if err != nil {
		return err
	}
	tf, err := os.Create(filepath.Join(tw.basedir, strconv.Itoa(tID.Z), strconv.Itoa(tID.X), strconv.Itoa(tID.Y)+".mvt"))
	if err != nil {
		return err
	}
	defer tf.Close()

	if tw.compressTiles {
		_, err = gzip.NewWriter(tf).Write(buf)
	} else {
		_, err = tf.Write(buf)
	}
	if err != nil {
		return err
	}
	return nil
}

type defaultLayerMapper struct {
	defaultLayer bool
}

func (dlm *defaultLayerMapper) LayerName(props map[string]interface{}) string {
	if layerName, ok := props["@layer"]; ok {
		return layerName.(string)
	}
	if dlm.defaultLayer {
		return "default"
	}
	return ""
}

type layerMapper interface {
	LayerName(map[string]interface{}) string
}

type tileWriter interface {
	WriteTile(tile.ID, []byte) error
}

func generateTiles(tIDs []tile.ID, features spatial.Filterable, tw tileWriter, lm layerMapper, pb chan<- struct{}) {
	for _, tID := range tIDs {
		// if !*quiet {
		// 	log.Printf("Generating %s", tID)
		// }
		var (
			layers = map[string][]spatial.Feature{}
			ln     string
		)
		tileClipBBox := tID.BBox()

		for _, feat := range features.Filter(tileClipBBox) {
			sf := tile.Resolution(tID.Z, 4096) * 20
			gm := feat.Geometry.Simplify(sf)
			for _, geom := range gm.ClipToBBox(tileClipBBox) {
				feat.Geometry = geom
				ln = lm.LayerName(feat.Props)
				if len(ln) != 0 {
					if _, ok := layers[ln]; !ok {
						layers[ln] = []spatial.Feature{feat}
					} else {
						layers[ln] = append(layers[ln], feat)
					}
				}
			}
		}
		if !anyFeatures(layers) {
			pb <- struct{}{}
			continue
		}
		buf, err := mvt.EncodeTile(layers, tID)
		if err != nil {
			log.Fatal(err)
		}

		err = tw.WriteTile(tID, buf)
		if err != nil {
			log.Fatal(err)
		}
		pb <- struct{}{}
	}
}

func anyFeatures(layers map[string][]spatial.Feature) bool {
	for _, ly := range layers {
		if len(ly) > 0 {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"

	"hawx.me/code/hadfield"
	"hawx.me/code/xesende"
)

var (
	accountReference = flag.String("account-reference", "", "")
	username         = flag.String("username", "", "")
	password         = flag.String("password", "", "")
)

var templates = hadfield.Templates{
	Help: `usage: example [command] [arguments]

  This is an example.

  Commands: {{range .}}
    {{.Name | printf "%-15s"}} # {{.Short}}{{end}}
`,
	Command: `usage: example {{.Usage}}
{{.Long}}
`,
}

func main() {
	flag.Parse()

	if *username == "" || *password == "" {
		log.Fatal("Require --username and --password options")
	}

	client := xesende.New(*username, *password)

	commands := hadfield.Commands{
		ReceivedCmd(client),
	}

	hadfield.Run(commands, templates)
}

func ReceivedCmd(client *xesende.Client) *hadfield.Command {
	var page int

	cmd := &hadfield.Command{
		Usage: "received [options]",
		Short: "lists received messages",
		Long: `
  Received displays a list of received messages.

    --page <num>    # Display given page
`,
		Run: func(cmd *hadfield.Command, args []string) {
			resp, err := client.Received()
			if err != nil {
				log.Fatal(err)
			}

			for _, message := range resp.Messages {
				fmt.Printf("At: %s \r\nFrom: %s \r\nBody: %s\r\n", message.ReceivedAt, message.From, message.BodyURI)
			}
		},
	}

	cmd.Flag.IntVar(&page, "page", 0, "")

	return cmd
}
<commit_msg>Add sent messages subcommand<commit_after>package main

import (
	"flag"
	"fmt"
	"log"

	"hawx.me/code/hadfield"
	"hawx.me/code/xesende"
)

var (
	accountReference = flag.String("account-reference", "", "")
	username         = flag.String("username", "", "")
	password         = flag.String("password", "", "")
)

const pageSize = 20

func pageOpts(page int) xesende.Option {
	startIndex := (page - 1) * pageSize

	return xesende.Page(startIndex, pageSize)
}

var templates = hadfield.Templates{
	Help: `usage: example [command] [arguments]

  This is an example.

  Commands: {{range .}}
    {{.Name | printf "%-15s"}} # {{.Short}}{{end}}
`,
	Command: `usage: example {{.Usage}}
{{.Long}}
`,
}

func main() {
	flag.Parse()

	if *username == "" || *password == "" {
		log.Fatal("Require --username and --password options")
	}

	client := xesende.New(*username, *password)

	commands := hadfield.Commands{
		ReceivedCmd(client),
		SentCmd(client),
	}

	hadfield.Run(commands, templates)
}

func ReceivedCmd(client *xesende.Client) *hadfield.Command {
	var page int

	cmd := &hadfield.Command{
		Usage: "received [options]",
		Short: "lists received messages",
		Long: `
  Received displays a list of received messages.

    --page <num>    # Display given page
`,
		Run: func(cmd *hadfield.Command, args []string) {
			resp, err := client.Received()
			if err != nil {
				log.Fatal(err)
			}

			for _, message := range resp.Messages {
				fmt.Printf("At: %s \r\nFrom: %s \r\nBody: %s\r\n", message.ReceivedAt, message.From, message.BodyURI)
			}
		},
	}

	cmd.Flag.IntVar(&page, "page", 0, "")

	return cmd
}

func SentCmd(client *xesende.Client) *hadfield.Command {
	var page int

	cmd := &hadfield.Command{
		Usage: "sent [options]",
		Short: "lists sent messages",
		Long: `
  Sent displays a list of sent messages.

    --page <num>    # Display given page
`,
		Run: func(cmd *hadfield.Command, args []string) {
			resp, err := client.Sent(pageOpts(page))
			if err != nil {
				log.Fatal(err)
			}

			for _, message := range resp.Messages {
				fmt.Printf("At: %s \r\nTo: %s \r\nBody: %s\r\n\r\n", message.SubmittedAt, message.To, message.BodyURI)
			}
		},
	}

	cmd.Flag.IntVar(&page, "page", 1, "")

	return cmd
}
<|endoftext|>


<commit_before>package gosupplychain

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"time"

	"github.com/client9/go-license"
	"github.com/google/go-github/github"
	"golang.org/x/oauth2"
)

// Repo describes a repo basic
// NOTE: likely to be replaced with a larger structure
type Repo struct {
	Name        string
	Description string
	Updated     time.Time
}

// User is the top level GitHub user (maybe be a company or user)
// NOTE: like to be replaced with a larger structure
type User struct {
	Name  string
	Repos []Repo
}

// GitHubFile is contains everything needed to represent a file at a point in time
//  Likely to be generalized later
type GitHubFile struct {
	Owner string
	Repo  string
	Path  string
	Tree  string
	SHA   string
}

// RawURL returns a URL to the raw content, without formatting
func (file GitHubFile) RawURL() string {
	return fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", file.Owner, file.Repo, file.Tree, file.Path)
}

// WebURL returns a human-friend URL to github
func (file GitHubFile) WebURL() string {
	return fmt.Sprintf("https://github.com/%s/%s/blob/%s/%s", file.Owner, file.Repo, file.Tree, file.Path)
}

// GitHub is a VCS
type GitHub struct {
	Client *github.Client
}

// NewGitHub creates a github client using oauth token
func NewGitHub(oauthToken string) GitHub {
	ts := oauth2.StaticTokenSource(
		&oauth2.Token{
			AccessToken: oauthToken,
		})
	tc := oauth2.NewClient(oauth2.NoContext, ts)
	return GitHub{
		Client: github.NewClient(tc),
	}
}

// GetFileContentsURL generates a download URL
func (gh GitHub) GetFileContentsURL(owner, repo, sha, filepath string) string {
	return fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", owner, repo, sha, filepath)
}

// GetFileContents down loads a file
func (gh GitHub) GetFileContents(owner, repo, tree, filepath string) (string, error) {
	url := gh.GetFileContentsURL(owner, repo, tree, filepath)
	resp, err := http.Get(url)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	return string(body), err
}

// GetTreeFiles returns the list of files given a tree.
//
// sha must be a valid git sha value or "master"
func (gh GitHub) GetTreeFiles(owner string, repo string, sha string) ([]GitHubFile, error) {
	tree, _, err := gh.Client.Git.GetTree(owner, repo, sha, false)
	if err != nil {
		return nil, err
	}
	//log.Printf("TREE: %+v", *tree)
	out := make([]GitHubFile, 0, len(tree.Entries))
	for _, t := range tree.Entries {
		out = append(out, GitHubFile{
			Owner: owner,
			Repo:  repo,
			Tree:  sha,
			Path:  *t.Path,
			SHA:   *t.SHA,
		})

		//log.Printf("TREE: %s", t)
	}
	return out, nil
}

// SearchByUsers performs a search on multiple users
func (gh GitHub) SearchByUsers(oauthToken string, searchQuery string, users []string) ([]User, error) {
	opts := &github.SearchOptions{
		Sort:  "updated",
		Order: "desc",
		ListOptions: github.ListOptions{
			PerPage: 100,
		},
	}

	out := make([]User, 0, len(users))

	// assume each query takes 1 second round trip
	//  and we get 20/minute
	//  wait 4 seconds between calls
	for pos, co := range users {
		if pos > 0 {
			time.Sleep(time.Second * 4)
		}
		q := fmt.Sprintf("user:%s %s", co, searchQuery)
		log.Printf("Running query %q", q)
		repos, _, err := gh.Client.Search.Repositories(q, opts)
		if err != nil {
			return nil, err
		}
		if repos == nil || *repos.Total == 0 || repos.Repositories == nil {
			continue
		}
		user := User{
			Name: co,
		}
		for _, val := range repos.Repositories {

			r := Repo{}
			if val.FullName != nil {
				r.Name = *val.FullName
			}
			if val.Description != nil {
				r.Description = *val.Description
			}
			if val.UpdatedAt != nil {
				// UpdateAt is a odd github.Time that embeds a time.Time
				tmp := *val.UpdatedAt
				r.Updated = tmp.Time
			}
			user.Repos = append(user.Repos, r)
		}
		out = append(out, user)
	}
	return out, nil
}

// GuessLicenseFromRepo attempts to determine a license
func (gh GitHub) GuessLicenseFromRepo(owner string, repo string, sha string) (license.License, error) {

	files, err := gh.GetTreeFiles(owner, repo, sha)
	if err != nil {
		return license.License{}, err
	}
	out := []string{}
	for _, filename := range files {
		if IsLicenseFile(filename.Path) {
			out = append(out, filename.Path)
			body, err := gh.GetFileContents(owner, repo, sha, filename.Path)
			if err != nil {
				return license.License{}, fmt.Errorf("unable to download %s: %s", filename, err)
			}
			lic := license.License{
				Text: body,
				File: filename.WebURL(),
			}
			err = lic.GuessType()
			if err == nil {
				return lic, nil
			}
		}
	}
	// empty
	return license.License{}, nil
}
<commit_msg>switch to correct repo<commit_after>package gosupplychain

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"time"

	"github.com/google/go-github/github"
	"github.com/ryanuber/go-license"
	"golang.org/x/oauth2"
)

// Repo describes a repo basic
// NOTE: likely to be replaced with a larger structure
type Repo struct {
	Name        string
	Description string
	Updated     time.Time
}

// User is the top level GitHub user (maybe be a company or user)
// NOTE: like to be replaced with a larger structure
type User struct {
	Name  string
	Repos []Repo
}

// GitHubFile is contains everything needed to represent a file at a point in time
//  Likely to be generalized later
type GitHubFile struct {
	Owner string
	Repo  string
	Path  string
	Tree  string
	SHA   string
}

// RawURL returns a URL to the raw content, without formatting
func (file GitHubFile) RawURL() string {
	return fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", file.Owner, file.Repo, file.Tree, file.Path)
}

// WebURL returns a human-friend URL to github
func (file GitHubFile) WebURL() string {
	return fmt.Sprintf("https://github.com/%s/%s/blob/%s/%s", file.Owner, file.Repo, file.Tree, file.Path)
}

// GitHub is a VCS
type GitHub struct {
	Client *github.Client
}

// NewGitHub creates a github client using oauth token
func NewGitHub(oauthToken string) GitHub {
	ts := oauth2.StaticTokenSource(
		&oauth2.Token{
			AccessToken: oauthToken,
		})
	tc := oauth2.NewClient(oauth2.NoContext, ts)
	return GitHub{
		Client: github.NewClient(tc),
	}
}

// GetFileContentsURL generates a download URL
func (gh GitHub) GetFileContentsURL(owner, repo, sha, filepath string) string {
	return fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", owner, repo, sha, filepath)
}

// GetFileContents down loads a file
func (gh GitHub) GetFileContents(owner, repo, tree, filepath string) (string, error) {
	url := gh.GetFileContentsURL(owner, repo, tree, filepath)
	resp, err := http.Get(url)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	return string(body), err
}

// GetTreeFiles returns the list of files given a tree.
//
// sha must be a valid git sha value or "master"
func (gh GitHub) GetTreeFiles(owner string, repo string, sha string) ([]GitHubFile, error) {
	tree, _, err := gh.Client.Git.GetTree(owner, repo, sha, false)
	if err != nil {
		return nil, err
	}
	//log.Printf("TREE: %+v", *tree)
	out := make([]GitHubFile, 0, len(tree.Entries))
	for _, t := range tree.Entries {
		out = append(out, GitHubFile{
			Owner: owner,
			Repo:  repo,
			Tree:  sha,
			Path:  *t.Path,
			SHA:   *t.SHA,
		})

		//log.Printf("TREE: %s", t)
	}
	return out, nil
}

// SearchByUsers performs a search on multiple users
func (gh GitHub) SearchByUsers(oauthToken string, searchQuery string, users []string) ([]User, error) {
	opts := &github.SearchOptions{
		Sort:  "updated",
		Order: "desc",
		ListOptions: github.ListOptions{
			PerPage: 100,
		},
	}

	out := make([]User, 0, len(users))

	// assume each query takes 1 second round trip
	//  and we get 20/minute
	//  wait 4 seconds between calls
	for pos, co := range users {
		if pos > 0 {
			time.Sleep(time.Second * 4)
		}
		q := fmt.Sprintf("user:%s %s", co, searchQuery)
		log.Printf("Running query %q", q)
		repos, _, err := gh.Client.Search.Repositories(q, opts)
		if err != nil {
			return nil, err
		}
		if repos == nil || *repos.Total == 0 || repos.Repositories == nil {
			continue
		}
		user := User{
			Name: co,
		}
		for _, val := range repos.Repositories {

			r := Repo{}
			if val.FullName != nil {
				r.Name = *val.FullName
			}
			if val.Description != nil {
				r.Description = *val.Description
			}
			if val.UpdatedAt != nil {
				// UpdateAt is a odd github.Time that embeds a time.Time
				tmp := *val.UpdatedAt
				r.Updated = tmp.Time
			}
			user.Repos = append(user.Repos, r)
		}
		out = append(out, user)
	}
	return out, nil
}

// GuessLicenseFromRepo attempts to determine a license
func (gh GitHub) GuessLicenseFromRepo(owner string, repo string, sha string) (license.License, error) {

	files, err := gh.GetTreeFiles(owner, repo, sha)
	if err != nil {
		return license.License{}, err
	}
	out := []string{}
	for _, filename := range files {
		if IsLicenseFile(filename.Path) {
			out = append(out, filename.Path)
			body, err := gh.GetFileContents(owner, repo, sha, filename.Path)
			if err != nil {
				return license.License{}, fmt.Errorf("unable to download %s: %s", filename, err)
			}
			lic := license.License{
				Text: body,
				File: filename.WebURL(),
			}
			err = lic.GuessType()
			if err == nil {
				return lic, nil
			}
		}
	}
	// empty
	return license.License{}, nil
}
<|endoftext|>


<commit_before>package testutil

import (
	"bytes"
	"fmt"
	"io"
	"path/filepath"
	"reflect"
	"regexp"
	"sort"
	"strings"
	"time"

	"v.io/tools/lib/collect"
	"v.io/tools/lib/util"
)

var (
	signatureRE = regexp.MustCompile(`^func (.*)\(.*\) \(.*\)$`)
)

// methods parses the given signature, which is expected to be
// generated by the "vrpc describe ..." command, extracting the list
// of methods contained in the signature of a vanadium RPC server the
// input describes.
func methods(signature string) ([]string, error) {
	signature = strings.TrimSpace(signature)
	result := []string{}
	lines := strings.Split(signature, "\n")
	for _, line := range lines {
		if !signatureRE.MatchString(line) {
			return nil, fmt.Errorf("unexpected format: %v", line)
		}
		matches := signatureRE.FindStringSubmatch(line)
		if len(matches) != 2 {
			return nil, fmt.Errorf("unexpected format: %v", line)
		}
		result = append(result, matches[1])
	}
	sort.Strings(result)
	return result, nil
}

// generateTestSuite generates an xUnit test suite that encapsulates
// the given input.
func generateTestSuite(ctx *util.Context, success bool, pkg string, duration time.Duration, output string) *testSuite {
	// Generate an xUnit test suite describing the result.
	s := testSuite{Name: pkg}
	c := testCase{
		Classname: pkg,
		Name:      "Test",
		Time:      fmt.Sprintf("%.2f", duration.Seconds()),
	}
	if !success {
		fmt.Fprintf(ctx.Stdout(), "%s ... failed\n%v\n", pkg, output)
		f := testFailure{
			Message: "vrpc",
			Data:    output,
		}
		c.Failures = append(c.Failures, f)
		s.Failures++
	} else {
		fmt.Fprintf(ctx.Stdout(), "%s ... ok\n", pkg)
	}
	s.Tests++
	s.Cases = append(s.Cases, c)
	return &s
}

// testProdService test the given production service.
func testProdService(ctx *util.Context, service prodService) (*testSuite, error) {
	root, err := util.VanadiumRoot()
	if err != nil {
		return nil, err
	}
	bin := filepath.Join(root, "release", "go", "bin", "vrpc")
	var out bytes.Buffer
	opts := ctx.Run().Opts()
	opts.Stdout = &out
	opts.Stderr = &out
	start := time.Now()
	if err := ctx.Run().TimedCommandWithOpts(DefaultTestTimeout, opts, bin, "describe", service.objectName); err != nil {
		return generateTestSuite(ctx, false, service.name, time.Now().Sub(start), out.String()), nil
	}
	got, err := methods(out.String())
	if err != nil {
		return nil, err
	}
	if want := service.signature; !reflect.DeepEqual(got, want) {
		fmt.Fprintf(ctx.Stderr(), "mismatching methods: got %v, want %v\n", got, want)
		return generateTestSuite(ctx, false, service.name, time.Now().Sub(start), "mismatching signature"), nil
	}
	return generateTestSuite(ctx, true, service.name, time.Now().Sub(start), ""), nil
}

type prodService struct {
	name       string
	objectName string
	signature  []string
}

// vanadiumProdServicesTest runs a test of vanadium production services.
func vanadiumProdServicesTest(ctx *util.Context, testName string) (_ *TestResult, e error) {
	// Initialize the test.
	cleanup, err := initTest(ctx, testName, nil)
	if err != nil {
		return nil, err
	}
	defer collect.Error(func() error { return cleanup() }, &e)

	// Install the vrpc tool.
	var out bytes.Buffer
	opts := ctx.Run().Opts()
	opts.Stderr = io.MultiWriter(&out, opts.Stderr)
	if err := ctx.Run().CommandWithOpts(opts, "v23", "go", "install", "v.io/core/veyron/tools/vrpc"); err != nil {
		// TODO(jingjin): create a utility function for this logic. See more in javascript.go.
		s := createTestSuiteWithFailure(testName, "BuildTools", "build failure", out.String(), 0)
		if err := createXUnitReport(ctx, testName, []testSuite{*s}); err != nil {
			return nil, err
		}
		return &TestResult{Status: TestFailed}, nil
	}

	// Describe the test cases.
	namespaceRoot := "/ns.dev.v.io:8101"
	allPassed, suites := true, []testSuite{}
	services := []prodService{
		prodService{
			name:       "mounttable",
			objectName: namespaceRoot,
			signature:  []string{"Delete", "GetACL", "Mount", "ResolveStep", "ResolveStepX", "SetACL", "Unmount"},
		},
		prodService{
			name:       "application repository",
			objectName: namespaceRoot + "/applicationd",
			signature:  []string{"Match", "Put", "Remove"},
		},
		prodService{
			name:       "binary repository",
			objectName: namespaceRoot + "/binaryd",
			signature:  []string{"Create", "Delete", "Download", "DownloadURL", "Stat", "Upload"},
		},
		prodService{
			name:       "macaroon service",
			objectName: namespaceRoot + "/identity/dev.v.io/macaroon",
			signature:  []string{"Bless"},
		},
		prodService{
			name:       "google identity service",
			objectName: namespaceRoot + "/identity/dev.v.io/google",
			signature:  []string{"BlessUsingAccessToken"},
		},
		prodService{
			name:       "binary discharger",
			objectName: namespaceRoot + "/identity/dev.v.io/discharger",
			signature:  []string{"Discharge"},
		},
	}

	for _, service := range services {
		suite, err := testProdService(ctx, service)
		if err != nil {
			return nil, err
		}
		allPassed = allPassed && (suite.Failures == 0)
		suites = append(suites, *suite)
	}

	// Create the xUnit report.
	if err := createXUnitReport(ctx, testName, suites); err != nil {
		return nil, err
	}
	if !allPassed {
		return &TestResult{Status: TestFailed}, nil
	}
	return &TestResult{Status: TestPassed}, nil
}
<commit_msg>tools/testutil: make sure prod services failures are reported in a test-suite<commit_after>package testutil

import (
	"bytes"
	"fmt"
	"io"
	"path/filepath"
	"reflect"
	"regexp"
	"sort"
	"strings"
	"time"

	"v.io/tools/lib/collect"
	"v.io/tools/lib/util"
)

var (
	signatureRE = regexp.MustCompile(`^func (.*)\(.*\) \(.*\)$`)
)

// methods parses the given signature, which is expected to be
// generated by the "vrpc describe ..." command, extracting the list
// of methods contained in the signature of a vanadium RPC server the
// input describes.
func methods(signature string) ([]string, error) {
	signature = strings.TrimSpace(signature)
	result := []string{}
	lines := strings.Split(signature, "\n")
	for _, line := range lines {
		if !signatureRE.MatchString(line) {
			return nil, fmt.Errorf("unexpected line in service signature: %v", line)
		}
		matches := signatureRE.FindStringSubmatch(line)
		if len(matches) != 2 {
			return nil, fmt.Errorf("unexpected line in services signature: %v", line)
		}
		result = append(result, matches[1])
	}
	sort.Strings(result)
	return result, nil
}

// generateTestSuite generates an xUnit test suite that encapsulates
// the given input.
func generateTestSuite(ctx *util.Context, success bool, pkg string, duration time.Duration, output string) *testSuite {
	// Generate an xUnit test suite describing the result.
	s := testSuite{Name: pkg}
	c := testCase{
		Classname: pkg,
		Name:      "Test",
		Time:      fmt.Sprintf("%.2f", duration.Seconds()),
	}
	if !success {
		fmt.Fprintf(ctx.Stdout(), "%s ... failed\n%v\n", pkg, output)
		f := testFailure{
			Message: "vrpc",
			Data:    output,
		}
		c.Failures = append(c.Failures, f)
		s.Failures++
	} else {
		fmt.Fprintf(ctx.Stdout(), "%s ... ok\n", pkg)
	}
	s.Tests++
	s.Cases = append(s.Cases, c)
	return &s
}

// testProdService test the given production service.
func testProdService(ctx *util.Context, service prodService) (*testSuite, error) {
	root, err := util.VanadiumRoot()
	if err != nil {
		return nil, err
	}
	bin := filepath.Join(root, "release", "go", "bin", "vrpc")
	var out bytes.Buffer
	opts := ctx.Run().Opts()
	opts.Stdout = &out
	opts.Stderr = &out
	start := time.Now()
	if err := ctx.Run().TimedCommandWithOpts(DefaultTestTimeout, opts, bin, "describe", service.objectName); err != nil {
		return generateTestSuite(ctx, false, service.name, time.Now().Sub(start), out.String()), nil
	}
	output := out.String()
	got, err := methods(output)
	if err != nil {
		return generateTestSuite(ctx, false, service.name, time.Now().Sub(start), err.Error()), nil
	}
	if want := service.signature; !reflect.DeepEqual(got, want) {
		fmt.Fprintf(ctx.Stderr(), "mismatching methods: got %v, want %v\n", got, want)
		return generateTestSuite(ctx, false, service.name, time.Now().Sub(start), "mismatching signature"), nil
	}
	return generateTestSuite(ctx, true, service.name, time.Now().Sub(start), ""), nil
}

type prodService struct {
	name       string
	objectName string
	signature  []string
}

// vanadiumProdServicesTest runs a test of vanadium production services.
func vanadiumProdServicesTest(ctx *util.Context, testName string) (_ *TestResult, e error) {
	// Initialize the test.
	cleanup, err := initTest(ctx, testName, nil)
	if err != nil {
		return nil, err
	}
	defer collect.Error(func() error { return cleanup() }, &e)

	// Install the vrpc tool.
	var out bytes.Buffer
	opts := ctx.Run().Opts()
	opts.Stderr = io.MultiWriter(&out, opts.Stderr)
	if err := ctx.Run().CommandWithOpts(opts, "v23", "go", "install", "v.io/core/veyron/tools/vrpc"); err != nil {
		// TODO(jingjin): create a utility function for this logic. See more in javascript.go.
		s := createTestSuiteWithFailure(testName, "BuildTools", "build failure", out.String(), 0)
		if err := createXUnitReport(ctx, testName, []testSuite{*s}); err != nil {
			return nil, err
		}
		return &TestResult{Status: TestFailed}, nil
	}

	// Describe the test cases.
	namespaceRoot := "/ns.dev.v.io:8101"
	allPassed, suites := true, []testSuite{}
	services := []prodService{
		prodService{
			name:       "mounttable",
			objectName: namespaceRoot,
			signature:  []string{"Delete", "GetACL", "Mount", "ResolveStep", "ResolveStepX", "SetACL", "Unmount"},
		},
		prodService{
			name:       "application repository",
			objectName: namespaceRoot + "/applicationd",
			signature:  []string{"Match", "Put", "Remove"},
		},
		prodService{
			name:       "binary repository",
			objectName: namespaceRoot + "/binaryd",
			signature:  []string{"Create", "Delete", "Download", "DownloadURL", "Stat", "Upload"},
		},
		prodService{
			name:       "macaroon service",
			objectName: namespaceRoot + "/identity/dev.v.io/macaroon",
			signature:  []string{"Bless"},
		},
		prodService{
			name:       "google identity service",
			objectName: namespaceRoot + "/identity/dev.v.io/google",
			signature:  []string{"BlessUsingAccessToken"},
		},
		prodService{
			name:       "binary discharger",
			objectName: namespaceRoot + "/identity/dev.v.io/discharger",
			signature:  []string{"Discharge"},
		},
	}

	for _, service := range services {
		suite, err := testProdService(ctx, service)
		if err != nil {
			return nil, err
		}
		allPassed = allPassed && (suite.Failures == 0)
		suites = append(suites, *suite)
	}

	// Create the xUnit report.
	if err := createXUnitReport(ctx, testName, suites); err != nil {
		return nil, err
	}
	if !allPassed {
		return &TestResult{Status: TestFailed}, nil
	}
	return &TestResult{Status: TestPassed}, nil
}
<|endoftext|>