<commit_before>package cmd

import (
	"bufio"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"strconv"
	"strings"
	"text/tabwriter"
	"time"

	"github.com/cozy/cozy-stack/client"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
	humanize "github.com/dustin/go-humanize"
	"github.com/spf13/cobra"
)

var flagLocale string
var flagTimezone string
var flagEmail string
var flagPublicName string
var flagSettings string
var flagDiskQuota string
var flagApps []string
var flagDev bool
var flagPassphrase string
var flagForce bool
var flagExpire time.Duration
var flagDry bool

// instanceCmdGroup represents the instances command
var instanceCmdGroup = &cobra.Command{
	Use:   "instances [command]",
	Short: "Manage instances of a stack",
	Long: `
cozy-stack instances allows to manage the instances of this stack

An instance is a logical space owned by one user and identified by a domain.
For example, bob.cozycloud.cc is the instance of Bob. A single cozy-stack
process can manage several instances.

Each instance has a separate space for storing files and a prefix used to
create its CouchDB databases.
`,
	RunE: func(cmd *cobra.Command, args []string) error {
		return cmd.Help()
	},
}

var cleanInstanceCmd = &cobra.Command{
	Use:   "clean [domain]",
	Short: "Clean badly removed instances",
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) == 0 {
			return cmd.Help()
		}

		domain := args[0]
		i := couchdb.SimpleDatabasePrefix(domain)
		return couchdb.DeleteAllDBs(i)
	},
}

var showInstanceCmd = &cobra.Command{
	Use:   "show [domain]",
	Short: "Show the instance of the specified domain",
	Long: `
cozy-stack instances show allows to show the instance on the cozy for a
given domain.
`,
	Example: "$ cozy-stack instances show cozy.tools:8080",
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) == 0 {
			return cmd.Help()
		}
		domain := args[0]
		c := newAdminClient()
		in, err := c.GetInstance(domain)
		if err != nil {
			return err
		}
		json, err := json.MarshalIndent(in, "", "  ")
		if err != nil {
			return err
		}
		fmt.Println(string(json))
		return nil
	},
}

var addInstanceCmd = &cobra.Command{
	Use:   "add [domain]",
	Short: "Manage instances of a stack",
	Long: `
cozy-stack instances add allows to create an instance on the cozy for a
given domain.
`,
	Example: "$ cozy-stack instances add --dev --passphrase cozy --apps drive,photos,settings cozy.tools:8080",
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) == 0 {
			return cmd.Help()
		}

		var diskQuota uint64
		if flagDiskQuota != "" {
			var err error
			diskQuota, err = humanize.ParseBytes(flagDiskQuota)
			if err != nil {
				return err
			}
		}

		domain := args[0]
		c := newAdminClient()
		in, err := c.CreateInstance(&client.InstanceOptions{
			Domain:     domain,
			Apps:       flagApps,
			Locale:     flagLocale,
			Timezone:   flagTimezone,
			Email:      flagEmail,
			PublicName: flagPublicName,
			Settings:   flagSettings,
			DiskQuota:  int64(diskQuota),
			Dev:        flagDev,
			Passphrase: flagPassphrase,
		})
		if err != nil {
			errPrintfln(
				"Failed to create instance for domain %s", domain)
			return err
		}

		fmt.Printf("Instance created with success for domain %s\n", in.Attrs.Domain)
		if in.Attrs.RegisterToken != nil {
			fmt.Printf("Registration token: \"%s\"\n", hex.EncodeToString(in.Attrs.RegisterToken))
		}
		return nil
	},
}

var quotaInstanceCmd = &cobra.Command{
	Use:   "set-disk-quota [domain] [disk-quota]",
	Short: "Change the disk-quota of the instance",
	Long: `
cozy-stack instances set-disk-quota allows to change the disk-quota of the
instance of the given domain. Set the quota to 0 to remove the quota.
`,
	Example: "$ cozy-stack instances set-disk-quota cozy.tools:8080 3GB",
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) != 2 {
			return cmd.Help()
		}
		diskQuota, err := humanize.ParseBytes(args[1])
		if err != nil {
			return fmt.Errorf("Could not parse disk-quota: %s", err)
		}
		domain := args[0]
		c := newAdminClient()
		_, err = c.ModifyInstance(domain, &client.InstanceOptions{
			DiskQuota: int64(diskQuota),
		})
		return err
	},
}

var debugInstanceCmd = &cobra.Command{
	Use:   "debug [domain] [true/false]",
	Short: "Activate or deactivate debugging of the instance",
	Long: `
cozy-stack instances debug allows to activate or deactivate the debugging of a
specific domain.
`,
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) != 2 {
			return cmd.Help()
		}
		domain := args[0]
		debug, err := strconv.ParseBool(args[1])
		if err != nil {
			return err
		}
		c := newAdminClient()
		_, err = c.ModifyInstance(domain, &client.InstanceOptions{
			Debug: &debug,
		})
		return err
	},
}

var lsInstanceCmd = &cobra.Command{
	Use:   "ls",
	Short: "List instances",
	Long: `
cozy-stack instances ls allows to list all the instances that can be served
by this server.
`,
	RunE: func(cmd *cobra.Command, args []string) error {
		c := newAdminClient()
		list, err := c.ListInstances()
		if err != nil {
			return err
		}
		w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
		for _, i := range list {
			fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\tv%d\n",
				i.Attrs.Domain,
				i.Attrs.Locale,
				formatSize(i.Attrs.BytesDiskQuota),
				formatDev(i.Attrs.Dev),
				formatOnboarded(i),
				i.Attrs.IndexViewsVersion,
			)
		}
		return w.Flush()
	},
}

func formatSize(size int64) string {
	if size == 0 {
		return "unlimited"
	}
	return humanize.Bytes(uint64(size))
}

func formatDev(dev bool) string {
	if dev {
		return "dev"
	}
	return "prod"
}

func formatOnboarded(i *client.Instance) string {
	if i.Attrs.OnboardingFinished {
		return "onboarded"
	}
	if len(i.Attrs.RegisterToken) > 0 {
		return "onboarding"
	}
	return "pending"
}

var destroyInstanceCmd = &cobra.Command{
	Use:   "destroy [domain]",
	Short: "Remove instance",
	Long: `
cozy-stack instances destroy allows to remove an instance
and all its data.
`,
	Aliases: []string{"rm"},
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) == 0 {
			return cmd.Help()
		}

		domain := args[0]

		if !flagForce {
			reader := bufio.NewReader(os.Stdin)
			fmt.Printf(`Are you sure you want to remove instance for domain %s ?
All data associated with this domain will be permanently lost.
[yes/NO]: `, domain)

			str, err := reader.ReadString('\n')
			if err != nil {
				return err
			}

			str = strings.ToLower(strings.TrimSpace(str))
			if str != "yes" && str != "y" {
				return nil
			}

			fmt.Println()
		}

		c := newAdminClient()
		err := c.DestroyInstance(domain)
		if err != nil {
			errPrintfln(
				"An error occured while destroying instance for domain %s", domain)
			return err
		}

		fmt.Printf("Instance for domain %s has been destroyed with success\n", domain)
		return nil
	},
}

var fsckInstanceCmd = &cobra.Command{
	Use:   "fsck [domain]",
	Short: "Check and repair a vfs",
	Long: `
The cozy-stack fsck command checks that the files in the VFS are not
desynchronized, ie a file present in CouchDB but not swift/localfs, or present
in swift/localfs but not couchdb.
`,
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) == 0 {
			return cmd.Help()
		}

		domain := args[0]

		if !flagDry {
			fmt.Printf("Sorry, only cozy-stack fsck --dry is implemented currently.")
			return errors.New("Not implemented yet")
		}

		c := newAdminClient()
		list, err := c.FsckInstance(domain)
		if err != nil {
			return err
		}

		if len(list) == 0 {
			fmt.Printf("Instance for domain %s is clean\n", domain)
		} else {
			for _, entry := range list {
				fmt.Printf("- %s: %s\n", entry["filename"], entry["message"])
			}
		}
		return nil
	},
}

var appTokenInstanceCmd = &cobra.Command{
	Use:   "token-app [domain] [slug]",
	Short: "Generate a new application token",
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) < 2 {
			return cmd.Help()
		}
		c := newAdminClient()
		token, err := c.GetToken(&client.TokenOptions{
			Domain:   args[0],
			Subject:  args[1],
			Audience: "app",
			Expire:   flagExpire,
		})
		if err != nil {
			return err
		}
		_, err = fmt.Println(token)
		return err
	},
}

var cliTokenInstanceCmd = &cobra.Command{
	Use:   "token-cli [domain] [scopes]",
	Short: "Generate a new CLI access token (global access)",
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) < 2 {
			return cmd.Help()
		}
		c := newAdminClient()
		token, err := c.GetToken(&client.TokenOptions{
			Domain:   args[0],
			Scope:    args[1:],
			Audience: "cli",
			Expire:   flagExpire,
		})
		if err != nil {
			return err
		}
		_, err = fmt.Println(token)
		return err
	},
}

var oauthTokenInstanceCmd = &cobra.Command{
	Use:   "token-oauth [domain] [clientid] [scopes]",
	Short: "Generate a new OAuth access token",
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) < 3 {
			return cmd.Help()
		}
		c := newAdminClient()
		token, err := c.GetToken(&client.TokenOptions{
			Domain:   args[0],
			Subject:  args[1],
			Audience: "access-token",
			Scope:    args[2:],
			Expire:   flagExpire,
		})
		if err != nil {
			return err
		}
		_, err = fmt.Println(token)
		return err
	},
}

var oauthClientInstanceCmd = &cobra.Command{
	Use:   "client-oauth [domain] [redirect_uri] [client_name] [software_id]",
	Short: "Register a new OAuth client",
	Long:  `It registers a new OAuth client and returns its client_id`,
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) < 4 {
			return cmd.Help()
		}
		c := newAdminClient()
		clientID, err := c.RegisterOAuthClient(&client.OAuthClientOptions{
			Domain:      args[0],
			RedirectURI: args[1],
			ClientName:  args[2],
			SoftwareID:  args[3],
		})
		if err != nil {
			return err
		}
		_, err = fmt.Println(clientID)
		return err
	},
}

func init() {
	instanceCmdGroup.AddCommand(showInstanceCmd)
	instanceCmdGroup.AddCommand(addInstanceCmd)
	instanceCmdGroup.AddCommand(cleanInstanceCmd)
	instanceCmdGroup.AddCommand(lsInstanceCmd)
	instanceCmdGroup.AddCommand(quotaInstanceCmd)
	instanceCmdGroup.AddCommand(debugInstanceCmd)
	instanceCmdGroup.AddCommand(destroyInstanceCmd)
	instanceCmdGroup.AddCommand(fsckInstanceCmd)
	instanceCmdGroup.AddCommand(appTokenInstanceCmd)
	instanceCmdGroup.AddCommand(cliTokenInstanceCmd)
	instanceCmdGroup.AddCommand(oauthTokenInstanceCmd)
	instanceCmdGroup.AddCommand(oauthClientInstanceCmd)
	addInstanceCmd.Flags().StringVar(&flagLocale, "locale", instance.DefaultLocale, "Locale of the new cozy instance")
	addInstanceCmd.Flags().StringVar(&flagTimezone, "tz", "", "The timezone for the user")
	addInstanceCmd.Flags().StringVar(&flagEmail, "email", "", "The email of the owner")
	addInstanceCmd.Flags().StringVar(&flagPublicName, "public-name", "", "The public name of the owner")
	addInstanceCmd.Flags().StringVar(&flagSettings, "settings", "", "A list of settings (eg context:foo,offer:premium)")
	addInstanceCmd.Flags().StringVar(&flagDiskQuota, "disk-quota", "", "The quota allowed to the instance's VFS")
	addInstanceCmd.Flags().StringSliceVar(&flagApps, "apps", nil, "Apps to be preinstalled")
	addInstanceCmd.Flags().BoolVar(&flagDev, "dev", false, "To create a development instance")
	addInstanceCmd.Flags().StringVar(&flagPassphrase, "passphrase", "", "Register the instance with this passphrase (useful for tests)")
	destroyInstanceCmd.Flags().BoolVar(&flagForce, "force", false, "Force the deletion without asking for confirmation")
	fsckInstanceCmd.Flags().BoolVar(&flagDry, "dry", false, "Don't modify the VFS, only show the inconsistencies")
	appTokenInstanceCmd.Flags().DurationVar(&flagExpire, "expire", 0, "Make the token expires in this amount of time")
	oauthTokenInstanceCmd.Flags().DurationVar(&flagExpire, "expire", 0, "Make the token expires in this amount of time")
	RootCmd.AddCommand(instanceCmdGroup)
}
<commit_msg>Warn when the install of an app failed on creating a new instance<commit_after>package cmd

import (
	"bufio"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"strconv"
	"strings"
	"text/tabwriter"
	"time"

	"github.com/cozy/cozy-stack/client"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
	humanize "github.com/dustin/go-humanize"
	"github.com/spf13/cobra"
)

var flagLocale string
var flagTimezone string
var flagEmail string
var flagPublicName string
var flagSettings string
var flagDiskQuota string
var flagApps []string
var flagDev bool
var flagPassphrase string
var flagForce bool
var flagExpire time.Duration
var flagDry bool

// instanceCmdGroup represents the instances command
var instanceCmdGroup = &cobra.Command{
	Use:   "instances [command]",
	Short: "Manage instances of a stack",
	Long: `
cozy-stack instances allows to manage the instances of this stack

An instance is a logical space owned by one user and identified by a domain.
For example, bob.cozycloud.cc is the instance of Bob. A single cozy-stack
process can manage several instances.

Each instance has a separate space for storing files and a prefix used to
create its CouchDB databases.
`,
	RunE: func(cmd *cobra.Command, args []string) error {
		return cmd.Help()
	},
}

var cleanInstanceCmd = &cobra.Command{
	Use:   "clean [domain]",
	Short: "Clean badly removed instances",
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) == 0 {
			return cmd.Help()
		}

		domain := args[0]
		i := couchdb.SimpleDatabasePrefix(domain)
		return couchdb.DeleteAllDBs(i)
	},
}

var showInstanceCmd = &cobra.Command{
	Use:   "show [domain]",
	Short: "Show the instance of the specified domain",
	Long: `
cozy-stack instances show allows to show the instance on the cozy for a
given domain.
`,
	Example: "$ cozy-stack instances show cozy.tools:8080",
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) == 0 {
			return cmd.Help()
		}
		domain := args[0]
		c := newAdminClient()
		in, err := c.GetInstance(domain)
		if err != nil {
			return err
		}
		json, err := json.MarshalIndent(in, "", "  ")
		if err != nil {
			return err
		}
		fmt.Println(string(json))
		return nil
	},
}

var addInstanceCmd = &cobra.Command{
	Use:   "add [domain]",
	Short: "Manage instances of a stack",
	Long: `
cozy-stack instances add allows to create an instance on the cozy for a
given domain.
`,
	Example: "$ cozy-stack instances add --dev --passphrase cozy --apps drive,photos,settings cozy.tools:8080",
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) == 0 {
			return cmd.Help()
		}

		var diskQuota uint64
		if flagDiskQuota != "" {
			var err error
			diskQuota, err = humanize.ParseBytes(flagDiskQuota)
			if err != nil {
				return err
			}
		}

		domain := args[0]
		c := newAdminClient()
		in, err := c.CreateInstance(&client.InstanceOptions{
			Domain:     domain,
			Apps:       flagApps,
			Locale:     flagLocale,
			Timezone:   flagTimezone,
			Email:      flagEmail,
			PublicName: flagPublicName,
			Settings:   flagSettings,
			DiskQuota:  int64(diskQuota),
			Dev:        flagDev,
			Passphrase: flagPassphrase,
		})
		if err != nil {
			errPrintfln(
				"Failed to create instance for domain %s", domain)
			return err
		}

		fmt.Printf("Instance created with success for domain %s\n", in.Attrs.Domain)
		if in.Attrs.RegisterToken != nil {
			fmt.Printf("Registration token: \"%s\"\n", hex.EncodeToString(in.Attrs.RegisterToken))
		}
		if len(flagApps) == 0 {
			return nil
		}
		apps, err := newClient(domain, consts.Apps).ListApps(consts.Apps)
		if err == nil && len(flagApps) != len(apps) {
			for _, slug := range flagApps {
				found := false
				for _, app := range apps {
					if app.Attrs.Slug == slug {
						found = true
						break
					}
				}
				if !found {
					fmt.Printf("/!\\ Application %s has not been installed\n", slug)
				}
			}
		}
		return nil
	},
}

var quotaInstanceCmd = &cobra.Command{
	Use:   "set-disk-quota [domain] [disk-quota]",
	Short: "Change the disk-quota of the instance",
	Long: `
cozy-stack instances set-disk-quota allows to change the disk-quota of the
instance of the given domain. Set the quota to 0 to remove the quota.
`,
	Example: "$ cozy-stack instances set-disk-quota cozy.tools:8080 3GB",
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) != 2 {
			return cmd.Help()
		}
		diskQuota, err := humanize.ParseBytes(args[1])
		if err != nil {
			return fmt.Errorf("Could not parse disk-quota: %s", err)
		}
		domain := args[0]
		c := newAdminClient()
		_, err = c.ModifyInstance(domain, &client.InstanceOptions{
			DiskQuota: int64(diskQuota),
		})
		return err
	},
}

var debugInstanceCmd = &cobra.Command{
	Use:   "debug [domain] [true/false]",
	Short: "Activate or deactivate debugging of the instance",
	Long: `
cozy-stack instances debug allows to activate or deactivate the debugging of a
specific domain.
`,
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) != 2 {
			return cmd.Help()
		}
		domain := args[0]
		debug, err := strconv.ParseBool(args[1])
		if err != nil {
			return err
		}
		c := newAdminClient()
		_, err = c.ModifyInstance(domain, &client.InstanceOptions{
			Debug: &debug,
		})
		return err
	},
}

var lsInstanceCmd = &cobra.Command{
	Use:   "ls",
	Short: "List instances",
	Long: `
cozy-stack instances ls allows to list all the instances that can be served
by this server.
`,
	RunE: func(cmd *cobra.Command, args []string) error {
		c := newAdminClient()
		list, err := c.ListInstances()
		if err != nil {
			return err
		}
		w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
		for _, i := range list {
			fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\tv%d\n",
				i.Attrs.Domain,
				i.Attrs.Locale,
				formatSize(i.Attrs.BytesDiskQuota),
				formatDev(i.Attrs.Dev),
				formatOnboarded(i),
				i.Attrs.IndexViewsVersion,
			)
		}
		return w.Flush()
	},
}

func formatSize(size int64) string {
	if size == 0 {
		return "unlimited"
	}
	return humanize.Bytes(uint64(size))
}

func formatDev(dev bool) string {
	if dev {
		return "dev"
	}
	return "prod"
}

func formatOnboarded(i *client.Instance) string {
	if i.Attrs.OnboardingFinished {
		return "onboarded"
	}
	if len(i.Attrs.RegisterToken) > 0 {
		return "onboarding"
	}
	return "pending"
}

var destroyInstanceCmd = &cobra.Command{
	Use:   "destroy [domain]",
	Short: "Remove instance",
	Long: `
cozy-stack instances destroy allows to remove an instance
and all its data.
`,
	Aliases: []string{"rm"},
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) == 0 {
			return cmd.Help()
		}

		domain := args[0]

		if !flagForce {
			reader := bufio.NewReader(os.Stdin)
			fmt.Printf(`Are you sure you want to remove instance for domain %s ?
All data associated with this domain will be permanently lost.
[yes/NO]: `, domain)

			str, err := reader.ReadString('\n')
			if err != nil {
				return err
			}

			str = strings.ToLower(strings.TrimSpace(str))
			if str != "yes" && str != "y" {
				return nil
			}

			fmt.Println()
		}

		c := newAdminClient()
		err := c.DestroyInstance(domain)
		if err != nil {
			errPrintfln(
				"An error occured while destroying instance for domain %s", domain)
			return err
		}

		fmt.Printf("Instance for domain %s has been destroyed with success\n", domain)
		return nil
	},
}

var fsckInstanceCmd = &cobra.Command{
	Use:   "fsck [domain]",
	Short: "Check and repair a vfs",
	Long: `
The cozy-stack fsck command checks that the files in the VFS are not
desynchronized, ie a file present in CouchDB but not swift/localfs, or present
in swift/localfs but not couchdb.
`,
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) == 0 {
			return cmd.Help()
		}

		domain := args[0]

		if !flagDry {
			fmt.Printf("Sorry, only cozy-stack fsck --dry is implemented currently.")
			return errors.New("Not implemented yet")
		}

		c := newAdminClient()
		list, err := c.FsckInstance(domain)
		if err != nil {
			return err
		}

		if len(list) == 0 {
			fmt.Printf("Instance for domain %s is clean\n", domain)
		} else {
			for _, entry := range list {
				fmt.Printf("- %s: %s\n", entry["filename"], entry["message"])
			}
		}
		return nil
	},
}

var appTokenInstanceCmd = &cobra.Command{
	Use:   "token-app [domain] [slug]",
	Short: "Generate a new application token",
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) < 2 {
			return cmd.Help()
		}
		c := newAdminClient()
		token, err := c.GetToken(&client.TokenOptions{
			Domain:   args[0],
			Subject:  args[1],
			Audience: "app",
			Expire:   flagExpire,
		})
		if err != nil {
			return err
		}
		_, err = fmt.Println(token)
		return err
	},
}

var cliTokenInstanceCmd = &cobra.Command{
	Use:   "token-cli [domain] [scopes]",
	Short: "Generate a new CLI access token (global access)",
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) < 2 {
			return cmd.Help()
		}
		c := newAdminClient()
		token, err := c.GetToken(&client.TokenOptions{
			Domain:   args[0],
			Scope:    args[1:],
			Audience: "cli",
			Expire:   flagExpire,
		})
		if err != nil {
			return err
		}
		_, err = fmt.Println(token)
		return err
	},
}

var oauthTokenInstanceCmd = &cobra.Command{
	Use:   "token-oauth [domain] [clientid] [scopes]",
	Short: "Generate a new OAuth access token",
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) < 3 {
			return cmd.Help()
		}
		c := newAdminClient()
		token, err := c.GetToken(&client.TokenOptions{
			Domain:   args[0],
			Subject:  args[1],
			Audience: "access-token",
			Scope:    args[2:],
			Expire:   flagExpire,
		})
		if err != nil {
			return err
		}
		_, err = fmt.Println(token)
		return err
	},
}

var oauthClientInstanceCmd = &cobra.Command{
	Use:   "client-oauth [domain] [redirect_uri] [client_name] [software_id]",
	Short: "Register a new OAuth client",
	Long:  `It registers a new OAuth client and returns its client_id`,
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) < 4 {
			return cmd.Help()
		}
		c := newAdminClient()
		clientID, err := c.RegisterOAuthClient(&client.OAuthClientOptions{
			Domain:      args[0],
			RedirectURI: args[1],
			ClientName:  args[2],
			SoftwareID:  args[3],
		})
		if err != nil {
			return err
		}
		_, err = fmt.Println(clientID)
		return err
	},
}

func init() {
	instanceCmdGroup.AddCommand(showInstanceCmd)
	instanceCmdGroup.AddCommand(addInstanceCmd)
	instanceCmdGroup.AddCommand(cleanInstanceCmd)
	instanceCmdGroup.AddCommand(lsInstanceCmd)
	instanceCmdGroup.AddCommand(quotaInstanceCmd)
	instanceCmdGroup.AddCommand(debugInstanceCmd)
	instanceCmdGroup.AddCommand(destroyInstanceCmd)
	instanceCmdGroup.AddCommand(fsckInstanceCmd)
	instanceCmdGroup.AddCommand(appTokenInstanceCmd)
	instanceCmdGroup.AddCommand(cliTokenInstanceCmd)
	instanceCmdGroup.AddCommand(oauthTokenInstanceCmd)
	instanceCmdGroup.AddCommand(oauthClientInstanceCmd)
	addInstanceCmd.Flags().StringVar(&flagLocale, "locale", instance.DefaultLocale, "Locale of the new cozy instance")
	addInstanceCmd.Flags().StringVar(&flagTimezone, "tz", "", "The timezone for the user")
	addInstanceCmd.Flags().StringVar(&flagEmail, "email", "", "The email of the owner")
	addInstanceCmd.Flags().StringVar(&flagPublicName, "public-name", "", "The public name of the owner")
	addInstanceCmd.Flags().StringVar(&flagSettings, "settings", "", "A list of settings (eg context:foo,offer:premium)")
	addInstanceCmd.Flags().StringVar(&flagDiskQuota, "disk-quota", "", "The quota allowed to the instance's VFS")
	addInstanceCmd.Flags().StringSliceVar(&flagApps, "apps", nil, "Apps to be preinstalled")
	addInstanceCmd.Flags().BoolVar(&flagDev, "dev", false, "To create a development instance")
	addInstanceCmd.Flags().StringVar(&flagPassphrase, "passphrase", "", "Register the instance with this passphrase (useful for tests)")
	destroyInstanceCmd.Flags().BoolVar(&flagForce, "force", false, "Force the deletion without asking for confirmation")
	fsckInstanceCmd.Flags().BoolVar(&flagDry, "dry", false, "Don't modify the VFS, only show the inconsistencies")
	appTokenInstanceCmd.Flags().DurationVar(&flagExpire, "expire", 0, "Make the token expires in this amount of time")
	oauthTokenInstanceCmd.Flags().DurationVar(&flagExpire, "expire", 0, "Make the token expires in this amount of time")
	RootCmd.AddCommand(instanceCmdGroup)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"

	"github.com/constabulary/kang"
)

func check(err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "fatal: %+v\n", err)
		os.Exit(1)
	}
}

func main() {
	kangfile := flag.String("f", ".kangfile", "path to .kangfile")
	flag.Parse()

	workdir, err := ioutil.TempDir("", "kang")
	check(err)

	*kangfile, err = filepath.Abs(*kangfile)
	check(err)

	rootdir := filepath.Dir(*kangfile)
	pkgdir := filepath.Join(rootdir, ".kang", "pkg")

	ctx := &kang.Context{
		GOOS:    runtime.GOOS,
		GOARCH:  runtime.GOARCH,
		Workdir: workdir,
		Pkgdir:  pkgdir,
	}

	pkg := &kang.Package{
		Context:    ctx,
		ImportPath: "github.com/constabulary/cmd/kang",
		Main:       true,
		Dir:        filepath.Join(rootdir, "cmd", "kang"),
		GoFiles:    []string{"main.go"},
		Imports: []*kang.Package{{
			Context:    ctx,
			ImportPath: "github.com/constabulary/kang",
			Dir:        rootdir,
			GoFiles:    []string{"kang.go"},
		}}}

	build(pkg)
}

func build(pkgs ...*kang.Package) {
	targets := make(map[string]func() error)

	fn, err := buildPackages(targets, pkgs)
	check(err)
	check(fn())
}

func buildPackages(targets map[string]func() error, pkgs []*kang.Package) (func() error, error) {
	var deps []func() error
	for _, pkg := range pkgs {
		fn, err := buildPackage(targets, pkg)
		check(err)
		deps = append(deps, fn)
	}
	return func() error {
		for _, fn := range deps {
			if err := fn(); err != nil {
				return err
			}
		}
		return nil
	}, nil
}

func buildPackage(targets map[string]func() error, pkg *kang.Package) (func() error, error) {

	// if this action is already present in the map, return it
	// rather than creating a new action.
	if a, ok := targets[pkg.ImportPath]; ok {
		return a, nil
	}

	// step 0. are we stale ?
	// if this package is not stale, then by definition none of its
	// dependencies are stale, so ignore this whole tree.
	if !pkg.IsStale() {
		return func() error {
			fmt.Println(pkg.ImportPath, "is up to date")
			return nil
		}, nil
	}

	return func() error {
		fmt.Println("building", pkg.ImportPath)
		return nil
	}, nil
}
<commit_msg>dependent builds, stale computation is a bit busted<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"

	"github.com/constabulary/kang"
)

func check(err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "fatal: %+v\n", err)
		os.Exit(1)
	}
}

func main() {
	kangfile := flag.String("f", ".kangfile", "path to .kangfile")
	flag.Parse()

	workdir, err := ioutil.TempDir("", "kang")
	check(err)

	*kangfile, err = filepath.Abs(*kangfile)
	check(err)

	rootdir := filepath.Dir(*kangfile)
	pkgdir := filepath.Join(rootdir, ".kang", "pkg")

	ctx := &kang.Context{
		GOOS:    runtime.GOOS,
		GOARCH:  runtime.GOARCH,
		Workdir: workdir,
		Pkgdir:  pkgdir,
	}

	pkg := &kang.Package{
		Context:    ctx,
		ImportPath: "github.com/constabulary/cmd/kang",
		Main:       true,
		Dir:        filepath.Join(rootdir, "cmd", "kang"),
		GoFiles:    []string{"main.go"},
		Imports: []*kang.Package{{
			Context:    ctx,
			ImportPath: "github.com/constabulary/kang",
			Dir:        rootdir,
			GoFiles:    []string{"kang.go"},
		}}}

	build(pkg)
}

func build(pkgs ...*kang.Package) {
	targets := make(map[string]func() error)

	fn, err := buildPackages(targets, pkgs)
	check(err)
	check(fn())
}

func buildPackages(targets map[string]func() error, pkgs []*kang.Package) (func() error, error) {
	var deps []func() error
	for _, pkg := range pkgs {
		fn, err := buildPackage(targets, pkg)
		check(err)
		deps = append(deps, fn)
	}
	return func() error {
		for _, fn := range deps {
			if err := fn(); err != nil {
				return err
			}
		}
		return nil
	}, nil
}

func buildPackage(targets map[string]func() error, pkg *kang.Package) (func() error, error) {

	// if this action is already present in the map, return it
	// rather than creating a new action.
	if fn, ok := targets[pkg.ImportPath]; ok {
		return fn, nil
	}

	var deps []func() error
	for _, pkg := range pkg.Imports {
		fn, err := buildPackage(targets, pkg)
		check(err)
		deps = append(deps, fn)
	}

	// if this package is not stale, then by definition none of its
	// dependencies are stale, so ignore this whole tree.
	if !pkg.IsStale() {
		return func() error {
			fmt.Println(pkg.ImportPath, "is up to date")
			return nil
		}, nil
	}

	return func() error {
		for _, dep := range deps {
			if err := dep(); err != nil {
				return err
			}
		}
		fmt.Println("building", pkg.ImportPath)
		return nil
	}, nil
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"
	"sync"
	"time"

	"github.com/funkygao/go-metrics"
	log "github.com/funkygao/log4go"
)

const (
	MsInNano = 1000 * 1000
	HttpLoop = 1000
)

// MonitorConsumers monitors num of online consumer groups over the time.
type MonitorF5 struct {
	stop chan struct{}
	tick time.Duration
	wg   *sync.WaitGroup

	latencyWithF5    metrics.Histogram
	latencyWithoutF5 metrics.Histogram
}

func (this *MonitorF5) Run() {
	defer this.wg.Done()

	ticker := time.NewTicker(this.tick)
	defer ticker.Stop()

	this.latencyWithF5 = metrics.NewRegisteredHistogram("api.with.f5", nil, metrics.NewExpDecaySample(1028, 0.015))
	this.latencyWithoutF5 = metrics.NewRegisteredHistogram("api.without.f5", nil, metrics.NewExpDecaySample(1028, 0.015))
	for {
		select {
		case <-this.stop:
			return

		case <-ticker.C:
			this.callWithF5()
			this.callWithoutF5()
		}
	}
}

func (this *MonitorF5) callWithF5() {
	url := "http://api.ffan.com/pubsub/v1/pub/alive"
	this.callHttp(url, this.latencyWithF5)
}

func (this *MonitorF5) callWithoutF5() {
	url := "http://sub.sit.ffan.com:9191/alive"
	this.callHttp(url, this.latencyWithoutF5)
}

func (this *MonitorF5) callHttp(url string, h metrics.Histogram) {
	var t time.Time
	for i := 0; i < HttpLoop; i++ {
		t = time.Now()

		resp, err := http.Get(url)
		if err != nil {
			log.Error("%s %v", url, err)
			continue
		}

		if resp.StatusCode != http.StatusOK {
			log.Error("%s %s", url, resp.Status)
			continue
		}

		h.Update(time.Since(t).Nanoseconds() / MsInNano)
	}
}
<commit_msg>more metrics lines to compare<commit_after>package main

import (
	"net/http"
	"sync"
	"time"

	"github.com/funkygao/gafka/mpool"
	"github.com/funkygao/go-metrics"
	log "github.com/funkygao/log4go"
)

const (
	MsInNano = 1000 * 1000
	HttpLoop = 1000
)

// MonitorConsumers monitors num of online consumer groups over the time.
type MonitorF5 struct {
	stop chan struct{}
	tick time.Duration
	wg   *sync.WaitGroup

	latencyWithF5WithGateway       metrics.Histogram
	latencyWithoutF5WithoutGateway metrics.Histogram
	latencyWithoutF5WithGateway    metrics.Histogram

	httpConn *http.Client
}

func (this *MonitorF5) Run() {
	defer this.wg.Done()

	ticker := time.NewTicker(this.tick)
	defer ticker.Stop()

	this.httpConn = &http.Client{
		Transport: &http.Transport{
			MaxIdleConnsPerHost: 1,
			DisableKeepAlives:   true, // enable http conn reuse
		},
	}

	this.latencyWithF5WithGateway = metrics.NewRegisteredHistogram("latency.api.f5yes.gwyes", nil, metrics.NewExpDecaySample(1028, 0.015))
	this.latencyWithoutF5WithoutGateway = metrics.NewRegisteredHistogram("latency.api.f5no.gwno", nil, metrics.NewExpDecaySample(1028, 0.015))
	this.latencyWithoutF5WithGateway = metrics.NewRegisteredHistogram("latency.api.f5no.gwyes", nil, metrics.NewExpDecaySample(1028, 0.015))
	for {
		select {
		case <-this.stop:
			return

		case <-ticker.C:
			this.callWithF5WithGateway()
			this.callWithoutF5WithoutGateway()
			this.callWithoutF5WithGateway()
		}
	}
}

func (this *MonitorF5) callWithF5WithGateway() {
	url := "http://api.ffan.com/pubsub/v1/pub/alive"
	this.callHttp(url, this.latencyWithF5WithGateway)
}

func (this *MonitorF5) callWithoutF5WithoutGateway() {
	url := "http://sub.sit.ffan.com:9191/alive"
	this.callHttp(url, this.latencyWithoutF5WithoutGateway)
}

func (this *MonitorF5) callWithoutF5WithGateway() {
	url := "10.209.36.67/pubsub/v1/pub/alive"
	host := "api.ffan.com"

	buf := mpool.BytesBufferGet()
	defer mpool.BytesBufferPut(buf)

	var t time.Time
	for i := 0; i < HttpLoop; i++ {
		t = time.Now()

		buf.Reset()
		req, err := http.NewRequest("GET", url, buf)
		if err != nil {
			log.Error("%s %v", url, err)
			continue
		}

		req.Header.Set("Host", host)

		resp, err := this.httpConn.Do(req)
		if err != nil {
			log.Error("%s %v", url, err)
			continue
		}

		resp.Body.Close()
		this.latencyWithoutF5WithGateway.Update(time.Since(t).Nanoseconds() / MsInNano)
	}

}

func (this *MonitorF5) callHttp(url string, h metrics.Histogram) {
	var t time.Time
	for i := 0; i < HttpLoop; i++ {
		t = time.Now()

		resp, err := http.Get(url)
		if err != nil {
			log.Error("%s %v", url, err)
			continue
		}

		if resp.StatusCode != http.StatusOK {
			log.Error("%s %s", url, resp.Status)
			continue
		}

		resp.Body.Close()

		h.Update(time.Since(t).Nanoseconds() / MsInNano)
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"github.com/spf13/cobra"
	"os"

	"github.com/mmcloughlin/globe"
	"io"
	"image/png"
)

// locationsCmd represents the locations command
var locationsCmd = &cobra.Command{
	Use:   "locations",
	Short: "TODO",
	Long: `TODO
.`,
	RunE: func(cmd *cobra.Command, args []string) error {
		out, err := cmd.Flags().GetString("output")
		if err != nil {
			return fmt.Errorf("cannot determine output: %s", err)
		}
		g := globe.New()
		g.DrawGraticule(10.0)
		g.DrawLandBoundaries()
		g.DrawCountryBoundaries()
		g.CenterOn(51.453349, -2.588323)
		writer, err := openWriter(out)
		if err != nil {
			return fmt.Errorf("cannot open output '%s': %s", out, err)
		}
		defer writer.Close()
		image := g.Image(400)
		return png.Encode(writer, image)
	},

}

func openWriter(out string) (io.WriteCloser, error) {
	if out == "" {
		return os.Stdout, nil
	}
	return os.Create(out)
}

func init() {
	RootCmd.AddCommand(locationsCmd)

	// Here you will define your flags and configuration settings.

	// Cobra supports Persistent Flags which will work for this command
	// and all subcommands, e.g.:
	// locationsCmd.PersistentFlags().String("foo", "", "A help for foo")

	// Cobra supports local flags which will only run when this command
	// is called directly, e.g.:
	locationsCmd.Flags().StringP("output", "o", "", "Output")
}
<commit_msg>add lat/lng country/land and grat parameters<commit_after>package cmd

import (
	"fmt"
	"github.com/spf13/cobra"
	"os"

	"github.com/mmcloughlin/globe"
	"io"
	"image/png"
	"github.com/spf13/pflag"
)

type Point struct {
	Latitude  float64 `json:"latitude"`
	Longitude float64 `json:"longitude"`
}

// locationsCmd represents the locations command
var locationsCmd = &cobra.Command{
	Use:   "locations",
	Short: "TODO",
	Long: `TODO
.`,
	RunE: func(cmd *cobra.Command, args []string) error {
		g, err := createGlobe(cmd.Flags())
		if err != nil {
			return fmt.Errorf("cannot create globe: %s", err)
		}
		err = appendPointsFromFiles(g, args)
		if err != nil {
			return fmt.Errorf("cannot draw points: %s", err)
		}
		out, err := cmd.Flags().GetString("output")
		if err != nil {
			return fmt.Errorf("cannot determine output: %s", err)
		}
		writer, err := openWriter(out)
		if err != nil {
			return fmt.Errorf("cannot open output '%s': %s", out, err)
		}
		defer writer.Close()
		image := g.Image(400)
		return png.Encode(writer, image)
	},

}

func appendPointsFromFiles(g *globe.Globe, paths []string) error {
	for _, path := range paths {
		err := appendPointsFromFile(g, path)
		if err != nil {
			return fmt.Errorf("error appending points from '%s': %s", path, err)
		}
	}
	return nil
}

func appendPointsFromFile(g *globe.Globe, path string) error {
	file, err := os.Open(path)
	if err != nil {
		return err
	}
	defer file.Close()
	return nil
}

func createGlobe(flags *pflag.FlagSet) (*globe.Globe, error) {
	g := globe.New()
	graticule, _ := flags.GetFloat64("graticule")
	if graticule <= 0 {
		return nil, fmt.Errorf("invalid value: graticule must be greater than zero")
	}
	g.DrawGraticule(graticule)
	if drawLand, _ := flags.GetBool("land-boundaries"); drawLand {
		g.DrawLandBoundaries()
	}
	if drawCountry, _ := flags.GetBool("country-boundaries"); drawCountry {
		g.DrawCountryBoundaries()
	}
	lat, _ := flags.GetFloat64("center-latitude")
	lon, _ := flags.GetFloat64("center-longitude")
	g.CenterOn(lat, lon)
	return g, nil
}

func openWriter(out string) (io.WriteCloser, error) {
	if out == "" {
		return os.Stdout, nil
	}
	return os.Create(out)
}

func init() {

	// Here you will define your flags and configuration settings.

	// Cobra supports Persistent Flags which will work for this command
	// and all subcommands, e.g.:
	// locationsCmd.PersistentFlags().String("foo", "", "A help for foo")

	// Cobra supports local flags which will only run when this command
	// is called directly, e.g.:
	locationsCmd.Flags().StringP("output", "o", "", "specify the output file, leave empty for stdout")
	locationsCmd.Flags().Float64P("graticule", "g", 10, "specify the graticule for the globe")
	locationsCmd.Flags().BoolP("land-boundaries", "l", true, "specify if land boundaries shall be drawn")
	locationsCmd.Flags().BoolP("country-boundaries", "c", true, "specify if country boundaries shall be drawn")
	locationsCmd.Flags().Float64P("center-latitude", "f", 51.509865, "specify the center latitude of the view")
	locationsCmd.Flags().Float64P("center-longitude", "t", -0.118092, "specify the center longitude of the view")

	RootCmd.AddCommand(locationsCmd)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"os"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/apcera/nats"
	"github.com/clsung/tailer"
)

var (
	logDir    = flag.String("nail_log_dir", "", "Directory to store logs.")
	logPrefix = flag.String("nail_log_prefix", "", "Log file prefix")
	regExp    = flag.String("nail_filter_pattern", "", "pattern to filter the regex")
	hasStdout = flag.Bool("nail_stdout", true, "send to stdout")
)

func main() {
	flag.Parse()
	opts := nats.DefaultOptions
	natsURL := os.Getenv("NATS_CLUSTER")
	if natsURL == "" {
		natsURL = "nats://localhost:4222"
	}
	opts.Servers = strings.Split(natsURL, ",")
	opts.MaxReconnect = 10
	opts.ReconnectWait = (1 * time.Second)
	nc, err := opts.Connect()
	log.SetFlags(0)

	if err != nil {
		log.Fatal(err)
	}
	defer nc.Close()

	// TODO: make it flag
	mq := tailer.NewMessageQueue(1 << 20)
	nc.Opts.DisconnectedCB = func(_ *nats.Conn) {
		log.Printf("Got disconnected! Queued %d messagse\n", mq.Len())
	}
	nc.Opts.ReconnectedCB = func(nc *nats.Conn) {
		log.Printf("Got reconnected to %v!\n", nc.ConnectedUrl())
	}
	nc.Opts.AsyncErrorCB = func(nc *nats.Conn, s *nats.Subscription, err error) {
		log.Printf("Got asyncerror %v, %v: %v!\n", nc.ConnectedUrl(), s, err)
	}

	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		for {
			err := nc.LastError()
			if err != nil {
				log.Printf("Error: %v", err)
				break
			}
			time.Sleep(1 * time.Second)
		}
	}()
	if len(flag.Args()) < 1 {
		log.Fatal("Need specify the topic")
	}
	topic := flag.Arg(0)
	nc.Subscribe(topic, func(m *nats.Msg) {
		mq.Push(m)
	})
	var matchLine *regexp.Regexp
	if *regExp != "" {
		log.Printf("Filter line by regex: %s", *regExp)

		matchLine, err = regexp.Compile(*regExp)
		if err != nil {
			log.Fatalf("regex %s error:%v", *regExp, err)
		}
	}
	emitters := make([]tailer.Emitter, 0)
	if *hasStdout {
		emitters = append(emitters, &tailer.StdoutEmitter{})
	}
	if *logDir != "" {
		fileEmitter, err := tailer.NewFileEmitter(*logDir, *logPrefix)
		if err != nil {
			log.Fatalf("create file emitter error %v", err)
		}
		fileEmitter.Start()
		emitters = append(emitters, fileEmitter)
		defer fileEmitter.Stop()
	}

	go func() {
		for {
			m, ok := mq.Pop()
			if ok {
				// TODO: filter logic put here
				if matchLine != nil && !matchLine.Match(m.Data) {
					// skip unmatched
					continue
				}
				for _, emitter := range emitters {
					emitter.Emit(m)
				}
			}
		}
	}()

	wg.Wait()
}
<commit_msg>move subscriber to tailer<commit_after>package main

import (
	"flag"
	"log"
	"os"
	"regexp"
	"sync"
	"time"

	"github.com/clsung/tailer"
)

var (
	logDir    = flag.String("nail_log_dir", "", "Directory to store logs.")
	logPrefix = flag.String("nail_log_prefix", "", "Log file prefix")
	regExp    = flag.String("nail_filter_pattern", "", "pattern to filter the regex")
	hasStdout = flag.Bool("nail_stdout", true, "send to stdout")
	workers   = flag.Int("nail_worker", 2, "queue workers")
)

func main() {
	flag.Parse()
	log.SetFlags(0)
	natsURL := os.Getenv("NATS_CLUSTER")
	if natsURL == "" {
		natsURL = "nats://localhost:4222"
	}
	if len(flag.Args()) < 1 {
		log.Fatal("Need specify the topic")
	}
	topic := flag.Arg(0)
	// TODO: make it flag
	mq := tailer.NewMessageQueue(1 << 20)
	var wg sync.WaitGroup
	var err error
	wg.Add(*workers)
	for i := 1; i <= *workers; i++ {
		subscriber, err := tailer.NewNatsSubscriber(natsURL)
		if err != nil {
			log.Fatal(err)
		}
		subscriber.Bind(mq)
		err = subscriber.Subscribe(topic)
		if err != nil {
			log.Fatal(err)
		}
		defer subscriber.Close()
		go func() {
			defer wg.Done()
			for {
				err := subscriber.LastError()
				if err != nil {
					log.Printf("Error: %v", err)
					break
				}
				time.Sleep(1 * time.Second)
			}
		}()
	}
	var matchLine *regexp.Regexp
	if *regExp != "" {
		log.Printf("Filter line by regex: %s", *regExp)

		matchLine, err = regexp.Compile(*regExp)
		if err != nil {
			log.Fatalf("regex %s error:%v", *regExp, err)
		}
	}
	emitters := make([]tailer.Emitter, 0)
	if *hasStdout {
		emitters = append(emitters, &tailer.StdoutEmitter{})
	}
	if *logDir != "" {
		fileEmitter, err := tailer.NewFileEmitter(*logDir, *logPrefix)
		if err != nil {
			log.Fatalf("create file emitter error %v", err)
		}
		fileEmitter.Start()
		emitters = append(emitters, fileEmitter)
		defer fileEmitter.Stop()
	}

	go func() {
		for {
			m, ok := mq.Pop()
			if ok {
				// TODO: filter logic put here
				if matchLine != nil && !matchLine.Match(m.Data) {
					// skip unmatched
					continue
				}
				for _, emitter := range emitters {
					emitter.Emit(m)
				}
			}
		}
	}()

	wg.Wait()
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"math"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	etcdraft "github.com/coreos/etcd/raft"
	hashic "github.com/hashicorp/raft"
	"github.com/hashicorp/raft-boltdb"
	"github.com/relab/raft"
	etcd "github.com/relab/raft/etcd"
	hraft "github.com/relab/raft/hashicorp"
	"github.com/relab/raft/raftgorums"
	"github.com/relab/rkv/rkvpb"

	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"
)

const (
	bgorums    = "gorums"
	betcd      = "etcd"
	bhashicorp = "hashicorp"
)

var (
	bench             = flag.Bool("quiet", false, "Silence log output")
	recover           = flag.Bool("recover", false, "Recover from stable storage")
	batch             = flag.Bool("batch", true, "enable batching")
	serverMetrics     = flag.Bool("servermetrics", true, "enable server-side metrics")
	electionTimeout   = flag.Duration("election", time.Second, "How long servers wait before starting an election")
	heartbeatTimeout  = flag.Duration("heartbeat", 20*time.Millisecond, "How often a heartbeat should be sent")
	entriesPerMsg     = flag.Uint64("entriespermsg", 64, "Entries per Appendentries message")
	catchupMultiplier = flag.Uint64("catchupmultiplier", 1024, "How many more times entries per message allowed during catch up")
)

func main() {
	var (
		id      = flag.Uint64("id", 0, "server ID")
		servers = flag.String("servers", ":9201,:9202,:9203,:9204,:9205,:9206,:9207", "comma separated list of server addresses")
		cluster = flag.String("cluster", "1,2,3", "comma separated list of server ids to form cluster with, [1 >= id <= len(servers)]")
		backend = flag.String("backend", "gorums", "Raft backend to use [gorums|etcd|hashicorp]")
	)

	flag.Parse()
	rand.Seed(time.Now().UnixNano())

	if *id == 0 {
		fmt.Print("-id argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	nodes := strings.Split(*servers, ",")

	if len(nodes) == 0 {
		fmt.Print("-server argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	selected := strings.Split(*cluster, ",")

	var ids []uint64

	for _, sid := range selected {
		id, err := strconv.ParseUint(sid, 10, 64)

		if err != nil {
			fmt.Print("could not parse -cluster argument\n\n")
			flag.Usage()
			os.Exit(1)
		}

		if id <= 0 || id > uint64(len(nodes)) {
			fmt.Print("invalid -cluster argument\n\n")
			flag.Usage()
			os.Exit(1)
		}

		ids = append(ids, id)
	}

	if len(ids) == 0 {
		fmt.Print("-cluster argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if len(ids) > len(nodes) {
		fmt.Print("-cluster specifies too many servers\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if *entriesPerMsg < 1 {
		fmt.Print("-entriespermsg must be atleast 1\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if *catchupMultiplier < 1 {
		fmt.Print("-catchupmultiplier must be atleast 1\n\n")
		flag.Usage()
		os.Exit(1)
	}

	logger := logrus.New()

	logFile, err := os.OpenFile(
		fmt.Sprintf("%s%sraft%.2d.log", os.TempDir(), string(filepath.Separator), *id),
		os.O_CREATE|os.O_TRUNC|os.O_APPEND|os.O_WRONLY, 0600,
	)

	if err != nil {
		logger.Fatal(err)
	}

	logger.Hooks.Add(NewLogToFileHook(logFile))

	if *bench {
		logger.Out = ioutil.Discard
		grpc.EnableTracing = false
	}

	grpclog.SetLogger(logger)

	lis, err := net.Listen("tcp", nodes[*id-1])

	if err != nil {
		logger.Fatal(err)
	}

	grpcServer := grpc.NewServer()

	switch *backend {
	case bgorums:
		rungorums(logger, lis, grpcServer, *id, ids, nodes)
	case betcd:
		runetcd(logger, lis, grpcServer, *id, ids, nodes)
	case bhashicorp:
		runhashicorp(logger, lis, grpcServer, *id, ids, nodes)
	}

}

func runhashicorp(
	logger logrus.FieldLogger,
	lis net.Listener, grpcServer *grpc.Server,
	id uint64, ids []uint64, nodes []string,
) {
	var selflis string
	selflis, ids, nodes = selfIDsNodes(logger, id, ids, nodes)

	servers := make([]hashic.Server, len(nodes)+1)
	servers[0] = hashic.Server{
		Suffrage: hashic.Voter,
		ID:       hashic.ServerID(selflis),
		Address:  hashic.ServerAddress(selflis),
	}
	for i, addr := range nodes {
		host, port, err := net.SplitHostPort(addr)
		if err != nil {
			logger.Fatal(err)
		}
		p, _ := strconv.Atoi(port)
		addr = host + ":" + strconv.Itoa(p-100)

		servers[i+1] = hashic.Server{
			Suffrage: hashic.Voter,
			ID:       hashic.ServerID(addr),
			Address:  hashic.ServerAddress(addr),
		}
	}

	addr, err := net.ResolveTCPAddr("tcp", selflis)
	if err != nil {
		logger.Fatal(err)
	}
	trans, err := hashic.NewTCPTransport(selflis, addr, len(ids), 10*time.Second, os.Stderr)
	if err != nil {
		logger.Fatal(err)
	}

	path := fmt.Sprintf("hashicorp%.2d.bolt", id)
	overwrite := !*recover
	// Check if file already exists.
	if _, err := os.Stat(path); err != nil {
		if os.IsNotExist(err) {
			// We don't need to overwrite a file that doesn't exist.
			overwrite = false
		} else {
			// If we are unable to verify the existence of the file,
			// there is probably a permission problem.
			logger.Fatal(err)
		}
	}
	if overwrite {
		if err := os.Remove(path); err != nil {
			logger.Fatal(err)
		}
	}
	logs, err := raftboltdb.NewBoltStore(path)
	if err != nil {
		logger.Fatal(err)
	}

	snaps := hashic.NewDiscardSnapshotStore()

	cfg := &hashic.Config{
		LocalID:            hashic.ServerID(selflis),
		ProtocolVersion:    hashic.ProtocolVersionMax,
		HeartbeatTimeout:   *electionTimeout,
		ElectionTimeout:    *electionTimeout,
		CommitTimeout:      *heartbeatTimeout,
		MaxAppendEntries:   int(*entriesPerMsg),
		ShutdownOnRemove:   true,
		TrailingLogs:       10240,
		SnapshotInterval:   120 * time.Hour,
		SnapshotThreshold:  math.MaxUint64,
		LeaderLeaseTimeout: *electionTimeout / 2,
	}

	node := hraft.NewRaft(logger, NewStore(), cfg, servers, trans, logs, logs, snaps)

	service := NewService(node)
	rkvpb.RegisterRKVServer(grpcServer, service)

	if *serverMetrics {
		_, port, err := net.SplitHostPort(selflis)

		if err != nil {
			logger.Fatal(err)
		}

		go func() {
			http.Handle("/metrics", promhttp.Handler())
			logger.Fatal(http.ListenAndServe(":5"+port, nil))
		}()
	}

	logger.Fatal(grpcServer.Serve(lis))
}

func runetcd(
	logger logrus.FieldLogger,
	lis net.Listener, grpcServer *grpc.Server,
	id uint64, ids []uint64, nodes []string,
) {
	var selflis string
	selflis, ids, nodes = selfIDsNodes(logger, id, ids, nodes)

	peers := make([]etcdraft.Peer, len(nodes))

	for i, addr := range nodes {
		nid := ids[i]

		host, port, err := net.SplitHostPort(addr)
		if err != nil {
			logger.Fatal(err)
		}
		p, _ := strconv.Atoi(port)

		ur, err := url.Parse("http://" + addr)
		ur.Host = host + ":" + strconv.Itoa(p-100)
		if err != nil {
			logger.Fatal(err)
		}

		peers[i] = etcdraft.Peer{
			ID:      nid,
			Context: []byte(ur.String()),
		}
	}

	storage := etcdraft.NewMemoryStorage()
	node := etcd.NewRaft(
		logger,
		NewStore(),
		storage,
		&etcdraft.Config{
			ID:            id,
			ElectionTick:  int(*electionTimeout / *heartbeatTimeout),
			HeartbeatTick: 1,
			Storage:       storage,
			MaxSizePerMsg: *entriesPerMsg,
			// etcdserver says: Never overflow the rafthttp buffer,
			// which is 4096. We keep the same constant.
			MaxInflightMsgs: 4096 / 8,
			Logger:          logger,
		},
		peers,
		*heartbeatTimeout,
	)

	service := NewService(node)
	rkvpb.RegisterRKVServer(grpcServer, service)

	go func() {
		logger.Fatal(grpcServer.Serve(lis))
	}()

	if *serverMetrics {
		_, port, err := net.SplitHostPort(selflis)

		if err != nil {
			logger.Fatal(err)
		}

		go func() {
			http.Handle("/metrics", promhttp.Handler())
			logger.Fatal(http.ListenAndServe(":5"+port, nil))
		}()
	}

	lishttp, err := net.Listen("tcp", selflis)

	if err != nil {
		logger.Fatal(err)
	}

	logger.Fatal(http.Serve(lishttp, node.Handler()))
}

func rungorums(
	logger logrus.FieldLogger,
	lis net.Listener, grpcServer *grpc.Server,
	id uint64, ids []uint64, nodes []string,
) {
	storage, err := raft.NewFileStorage(fmt.Sprintf("db%.2d.bolt", id), !*recover)

	if err != nil {
		logger.Fatal(err)
	}

	storageWithCache := raft.NewCacheStorage(storage, 20000)

	node := raftgorums.NewRaft(NewStore(), &raftgorums.Config{
		ID:                id,
		Servers:           nodes,
		InitialCluster:    ids,
		Batch:             *batch,
		Storage:           storageWithCache,
		ElectionTimeout:   *electionTimeout,
		HeartbeatTimeout:  *heartbeatTimeout,
		EntriesPerMsg:     *entriesPerMsg,
		CatchupMultiplier: *catchupMultiplier,
		Logger:            logger,
		MetricsEnabled:    true,
	})

	service := NewService(node)
	rkvpb.RegisterRKVServer(grpcServer, service)

	go func() {
		logger.Fatal(grpcServer.Serve(lis))
	}()

	if *serverMetrics {
		_, port, err := net.SplitHostPort(nodes[id-1])

		if err != nil {
			logger.Fatal(err)
		}

		go func() {
			http.Handle("/metrics", promhttp.Handler())
			logger.Fatal(http.ListenAndServe(":5"+port, nil))
		}()
	}

	logger.Fatal(node.Run(grpcServer))
}

func selfIDsNodes(logger logrus.FieldLogger, id uint64, ids []uint64, nodes []string) (string, []uint64, []string) {
	host, port, err := net.SplitHostPort(nodes[id-1])
	if err != nil {
		logger.Fatal(err)
	}
	p, _ := strconv.Atoi(port)
	selflis := host + ":" + strconv.Itoa(p-100)

	ids = append(ids[:id-1], ids[id:]...)
	nodes = append(nodes[:id-1], nodes[id:]...)

	return selflis, ids, nodes
}
<commit_msg>rkvd/main.go: Fix hashicorp/raft discarding log<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"math"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	etcdraft "github.com/coreos/etcd/raft"
	hashic "github.com/hashicorp/raft"
	"github.com/hashicorp/raft-boltdb"
	"github.com/relab/raft"
	etcd "github.com/relab/raft/etcd"
	hraft "github.com/relab/raft/hashicorp"
	"github.com/relab/raft/raftgorums"
	"github.com/relab/rkv/rkvpb"

	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"
)

const (
	bgorums    = "gorums"
	betcd      = "etcd"
	bhashicorp = "hashicorp"
)

var (
	bench             = flag.Bool("quiet", false, "Silence log output")
	recover           = flag.Bool("recover", false, "Recover from stable storage")
	batch             = flag.Bool("batch", true, "enable batching")
	serverMetrics     = flag.Bool("servermetrics", true, "enable server-side metrics")
	electionTimeout   = flag.Duration("election", time.Second, "How long servers wait before starting an election")
	heartbeatTimeout  = flag.Duration("heartbeat", 20*time.Millisecond, "How often a heartbeat should be sent")
	entriesPerMsg     = flag.Uint64("entriespermsg", 64, "Entries per Appendentries message")
	catchupMultiplier = flag.Uint64("catchupmultiplier", 1024, "How many more times entries per message allowed during catch up")
)

func main() {
	var (
		id      = flag.Uint64("id", 0, "server ID")
		servers = flag.String("servers", ":9201,:9202,:9203,:9204,:9205,:9206,:9207", "comma separated list of server addresses")
		cluster = flag.String("cluster", "1,2,3", "comma separated list of server ids to form cluster with, [1 >= id <= len(servers)]")
		backend = flag.String("backend", "gorums", "Raft backend to use [gorums|etcd|hashicorp]")
	)

	flag.Parse()
	rand.Seed(time.Now().UnixNano())

	if *id == 0 {
		fmt.Print("-id argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	nodes := strings.Split(*servers, ",")

	if len(nodes) == 0 {
		fmt.Print("-server argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	selected := strings.Split(*cluster, ",")

	var ids []uint64

	for _, sid := range selected {
		id, err := strconv.ParseUint(sid, 10, 64)

		if err != nil {
			fmt.Print("could not parse -cluster argument\n\n")
			flag.Usage()
			os.Exit(1)
		}

		if id <= 0 || id > uint64(len(nodes)) {
			fmt.Print("invalid -cluster argument\n\n")
			flag.Usage()
			os.Exit(1)
		}

		ids = append(ids, id)
	}

	if len(ids) == 0 {
		fmt.Print("-cluster argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if len(ids) > len(nodes) {
		fmt.Print("-cluster specifies too many servers\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if *entriesPerMsg < 1 {
		fmt.Print("-entriespermsg must be atleast 1\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if *catchupMultiplier < 1 {
		fmt.Print("-catchupmultiplier must be atleast 1\n\n")
		flag.Usage()
		os.Exit(1)
	}

	logger := logrus.New()

	logFile, err := os.OpenFile(
		fmt.Sprintf("%s%sraft%.2d.log", os.TempDir(), string(filepath.Separator), *id),
		os.O_CREATE|os.O_TRUNC|os.O_APPEND|os.O_WRONLY, 0600,
	)

	if err != nil {
		logger.Fatal(err)
	}

	logger.Hooks.Add(NewLogToFileHook(logFile))

	if *bench {
		logger.Out = ioutil.Discard
		grpc.EnableTracing = false
	}

	grpclog.SetLogger(logger)

	lis, err := net.Listen("tcp", nodes[*id-1])

	if err != nil {
		logger.Fatal(err)
	}

	grpcServer := grpc.NewServer()

	switch *backend {
	case bgorums:
		rungorums(logger, lis, grpcServer, *id, ids, nodes)
	case betcd:
		runetcd(logger, lis, grpcServer, *id, ids, nodes)
	case bhashicorp:
		runhashicorp(logger, lis, grpcServer, *id, ids, nodes)
	}

}

func runhashicorp(
	logger logrus.FieldLogger,
	lis net.Listener, grpcServer *grpc.Server,
	id uint64, ids []uint64, nodes []string,
) {
	var selflis string
	selflis, ids, nodes = selfIDsNodes(logger, id, ids, nodes)

	servers := make([]hashic.Server, len(nodes)+1)
	servers[0] = hashic.Server{
		Suffrage: hashic.Voter,
		ID:       hashic.ServerID(selflis),
		Address:  hashic.ServerAddress(selflis),
	}
	for i, addr := range nodes {
		host, port, err := net.SplitHostPort(addr)
		if err != nil {
			logger.Fatal(err)
		}
		p, _ := strconv.Atoi(port)
		addr = host + ":" + strconv.Itoa(p-100)

		servers[i+1] = hashic.Server{
			Suffrage: hashic.Voter,
			ID:       hashic.ServerID(addr),
			Address:  hashic.ServerAddress(addr),
		}
	}

	addr, err := net.ResolveTCPAddr("tcp", selflis)
	if err != nil {
		logger.Fatal(err)
	}
	trans, err := hashic.NewTCPTransport(selflis, addr, len(ids), 10*time.Second, os.Stderr)
	if err != nil {
		logger.Fatal(err)
	}

	path := fmt.Sprintf("hashicorp%.2d.bolt", id)
	overwrite := !*recover
	// Check if file already exists.
	if _, err := os.Stat(path); err != nil {
		if os.IsNotExist(err) {
			// We don't need to overwrite a file that doesn't exist.
			overwrite = false
		} else {
			// If we are unable to verify the existence of the file,
			// there is probably a permission problem.
			logger.Fatal(err)
		}
	}
	if overwrite {
		if err := os.Remove(path); err != nil {
			logger.Fatal(err)
		}
	}
	logs, err := raftboltdb.NewBoltStore(path)
	if err != nil {
		logger.Fatal(err)
	}

	snaps := hashic.NewInmemSnapshotStore()

	cfg := &hashic.Config{
		LocalID:            hashic.ServerID(selflis),
		ProtocolVersion:    hashic.ProtocolVersionMax,
		HeartbeatTimeout:   *electionTimeout,
		ElectionTimeout:    *electionTimeout,
		CommitTimeout:      *heartbeatTimeout,
		MaxAppendEntries:   int(*entriesPerMsg),
		ShutdownOnRemove:   true,
		TrailingLogs:       math.MaxUint64,
		SnapshotInterval:   120 * time.Hour,
		SnapshotThreshold:  math.MaxUint64,
		LeaderLeaseTimeout: *electionTimeout / 2,
	}

	node := hraft.NewRaft(logger, NewStore(), cfg, servers, trans, logs, logs, snaps)

	service := NewService(node)
	rkvpb.RegisterRKVServer(grpcServer, service)

	if *serverMetrics {
		_, port, err := net.SplitHostPort(selflis)

		if err != nil {
			logger.Fatal(err)
		}

		go func() {
			http.Handle("/metrics", promhttp.Handler())
			logger.Fatal(http.ListenAndServe(":5"+port, nil))
		}()
	}

	logger.Fatal(grpcServer.Serve(lis))
}

func runetcd(
	logger logrus.FieldLogger,
	lis net.Listener, grpcServer *grpc.Server,
	id uint64, ids []uint64, nodes []string,
) {
	var selflis string
	selflis, ids, nodes = selfIDsNodes(logger, id, ids, nodes)

	peers := make([]etcdraft.Peer, len(nodes))

	for i, addr := range nodes {
		nid := ids[i]

		host, port, err := net.SplitHostPort(addr)
		if err != nil {
			logger.Fatal(err)
		}
		p, _ := strconv.Atoi(port)

		ur, err := url.Parse("http://" + addr)
		ur.Host = host + ":" + strconv.Itoa(p-100)
		if err != nil {
			logger.Fatal(err)
		}

		peers[i] = etcdraft.Peer{
			ID:      nid,
			Context: []byte(ur.String()),
		}
	}

	storage := etcdraft.NewMemoryStorage()
	node := etcd.NewRaft(
		logger,
		NewStore(),
		storage,
		&etcdraft.Config{
			ID:            id,
			ElectionTick:  int(*electionTimeout / *heartbeatTimeout),
			HeartbeatTick: 1,
			Storage:       storage,
			MaxSizePerMsg: *entriesPerMsg,
			// etcdserver says: Never overflow the rafthttp buffer,
			// which is 4096. We keep the same constant.
			MaxInflightMsgs: 4096 / 8,
			Logger:          logger,
		},
		peers,
		*heartbeatTimeout,
	)

	service := NewService(node)
	rkvpb.RegisterRKVServer(grpcServer, service)

	go func() {
		logger.Fatal(grpcServer.Serve(lis))
	}()

	if *serverMetrics {
		_, port, err := net.SplitHostPort(selflis)

		if err != nil {
			logger.Fatal(err)
		}

		go func() {
			http.Handle("/metrics", promhttp.Handler())
			logger.Fatal(http.ListenAndServe(":5"+port, nil))
		}()
	}

	lishttp, err := net.Listen("tcp", selflis)

	if err != nil {
		logger.Fatal(err)
	}

	logger.Fatal(http.Serve(lishttp, node.Handler()))
}

func rungorums(
	logger logrus.FieldLogger,
	lis net.Listener, grpcServer *grpc.Server,
	id uint64, ids []uint64, nodes []string,
) {
	storage, err := raft.NewFileStorage(fmt.Sprintf("db%.2d.bolt", id), !*recover)

	if err != nil {
		logger.Fatal(err)
	}

	storageWithCache := raft.NewCacheStorage(storage, 20000)

	node := raftgorums.NewRaft(NewStore(), &raftgorums.Config{
		ID:                id,
		Servers:           nodes,
		InitialCluster:    ids,
		Batch:             *batch,
		Storage:           storageWithCache,
		ElectionTimeout:   *electionTimeout,
		HeartbeatTimeout:  *heartbeatTimeout,
		EntriesPerMsg:     *entriesPerMsg,
		CatchupMultiplier: *catchupMultiplier,
		Logger:            logger,
		MetricsEnabled:    true,
	})

	service := NewService(node)
	rkvpb.RegisterRKVServer(grpcServer, service)

	go func() {
		logger.Fatal(grpcServer.Serve(lis))
	}()

	if *serverMetrics {
		_, port, err := net.SplitHostPort(nodes[id-1])

		if err != nil {
			logger.Fatal(err)
		}

		go func() {
			http.Handle("/metrics", promhttp.Handler())
			logger.Fatal(http.ListenAndServe(":5"+port, nil))
		}()
	}

	logger.Fatal(node.Run(grpcServer))
}

func selfIDsNodes(logger logrus.FieldLogger, id uint64, ids []uint64, nodes []string) (string, []uint64, []string) {
	host, port, err := net.SplitHostPort(nodes[id-1])
	if err != nil {
		logger.Fatal(err)
	}
	p, _ := strconv.Atoi(port)
	selflis := host + ":" + strconv.Itoa(p-100)

	ids = append(ids[:id-1], ids[id:]...)
	nodes = append(nodes[:id-1], nodes[id:]...)

	return selflis, ids, nodes
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"strings"
	"syscall"

	"code.cloudfoundry.org/cflager"
	"code.cloudfoundry.org/debugserver"
	"code.cloudfoundry.org/diego-ssh/authenticators"
	"code.cloudfoundry.org/diego-ssh/daemon"
	"code.cloudfoundry.org/diego-ssh/keys"
	"code.cloudfoundry.org/lager"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/sigmon"
	"golang.org/x/crypto/ssh"
)

var address = flag.String(
	"address",
	"127.0.0.1:2222",
	"listen address for ssh daemon",
)

var hostKey = flag.String(
	"hostKey",
	"",
	"PEM encoded RSA host key",
)

var authorizedKey = flag.String(
	"authorizedKey",
	"",
	"Public key in the OpenSSH authorized_keys format",
)

var allowUnauthenticatedClients = flag.Bool(
	"allowUnauthenticatedClients",
	false,
	"Allow access to unauthenticated clients",
)

var inheritDaemonEnv = flag.Bool(
	"inheritDaemonEnv",
	false,
	"Inherit daemon's environment",
)

var allowedCiphers = flag.String(
	"allowedCiphers",
	"",
	"Limit cipher algorithms to those provided (comma separated)",
)

var allowedMACs = flag.String(
	"allowedMACs",
	"",
	"Limit MAC algorithms to those provided (comma separated)",
)

var allowedKeyExchanges = flag.String(
	"allowedKeyExchanges",
	"",
	"Limit key exchanges algorithms to those provided (comma separated)",
)

var hostKeyPEM string
var authorizedKeyValue string

func main() {
	debugserver.AddFlags(flag.CommandLine)
	cflager.AddFlags(flag.CommandLine)
	flag.Parse()
	exec := false

	logger, reconfigurableSink := cflager.New("sshd")

	hostKeyPEM = os.Getenv("SSHD_HOSTKEY")
	if hostKeyPEM != "" {
		authorizedKeyValue = os.Getenv("SSHD_AUTHKEY")

		// unset the variables so child processes don't inherit them
		os.Unsetenv("SSHD_HOSTKEY")
		os.Unsetenv("SSHD_AUTHKEY")
	} else {
		hostKeyPEM = *hostKey
		if hostKeyPEM == "" {
			var err error
			hostKeyPEM, err = generateNewHostKey()
			if err != nil {
				logger.Error("failed-to-generate-host-key", err)
				os.Exit(1)
			}
		}
		authorizedKeyValue = *authorizedKey
		exec = true
	}

	if exec {
		os.Setenv("SSHD_HOSTKEY", hostKeyPEM)
		os.Setenv("SSHD_AUTHKEY", authorizedKeyValue)

		err := syscall.Exec(os.Args[0], []string{
			os.Args[0],
			fmt.Sprintf("--allowedKeyExchanges=%s", *allowedKeyExchanges),
			fmt.Sprintf("--address=%s", *address),
			fmt.Sprintf("--allowUnauthenticatedClients=%t", *allowUnauthenticatedClients),
			fmt.Sprintf("--inheritDaemonEnv=%t", *inheritDaemonEnv),
			fmt.Sprintf("--allowedCiphers=%s", *allowedCiphers),
			fmt.Sprintf("--allowedMACs=%s", *allowedMACs),
		}, os.Environ())
		if err != nil {
			logger.Error("failed-exec", err)
			os.Exit(1)
		}
	}

	serverConfig, err := configure(logger)
	if err != nil {
		logger.Error("configure-failed", err)
		os.Exit(1)
	}

	sshDaemon := daemon.New(logger, serverConfig, nil, newChannelHandlers())
	server, err := createServer(logger, *address, sshDaemon)

	members := grouper.Members{
		{"sshd", server},
	}

	if dbgAddr := debugserver.DebugAddress(flag.CommandLine); dbgAddr != "" {
		members = append(grouper.Members{
			{"debug-server", debugserver.Runner(dbgAddr, reconfigurableSink)},
		}, members...)
	}

	group := grouper.NewOrdered(os.Interrupt, members)
	monitor := ifrit.Invoke(sigmon.New(group))

	logger.Info("started")

	err = <-monitor.Wait()
	if err != nil {
		logger.Error("exited-with-failure", err)
		os.Exit(1)
	}

	logger.Info("exited")
	os.Exit(0)
}

func getDaemonEnvironment() map[string]string {
	daemonEnv := map[string]string{}

	if *inheritDaemonEnv {
		envs := os.Environ()
		for _, env := range envs {
			nvp := strings.SplitN(env, "=", 2)
			if len(nvp) == 2 && nvp[0] != "PATH" {
				daemonEnv[nvp[0]] = nvp[1]
			}
		}
	}
	return daemonEnv
}

func configure(logger lager.Logger) (*ssh.ServerConfig, error) {
	errorStrings := []string{}
	sshConfig := &ssh.ServerConfig{}

	key, err := acquireHostKey(logger)
	if err != nil {
		logger.Error("failed-to-acquire-host-key", err)
		errorStrings = append(errorStrings, err.Error())
	}

	sshConfig.AddHostKey(key)
	sshConfig.NoClientAuth = *allowUnauthenticatedClients

	if authorizedKeyValue == "" && !*allowUnauthenticatedClients {
		logger.Error("authorized-key-required", nil)
		errorStrings = append(errorStrings, "Public user key is required")
	}

	if authorizedKeyValue != "" {
		decodedPublicKey, err := decodeAuthorizedKey(logger)
		if err == nil {
			authenticator := authenticators.NewPublicKeyAuthenticator(decodedPublicKey)
			sshConfig.PublicKeyCallback = authenticator.Authenticate
		} else {
			errorStrings = append(errorStrings, err.Error())
		}
	}

	if *allowedCiphers != "" {
		sshConfig.Config.Ciphers = strings.Split(*allowedCiphers, ",")
	}
	if *allowedMACs != "" {
		sshConfig.Config.MACs = strings.Split(*allowedMACs, ",")
	}
	if *allowedKeyExchanges != "" {
		sshConfig.Config.KeyExchanges = strings.Split(*allowedKeyExchanges, ",")
	}

	err = nil
	if len(errorStrings) > 0 {
		err = errors.New(strings.Join(errorStrings, ", "))
	}

	return sshConfig, err
}

func decodeAuthorizedKey(logger lager.Logger) (ssh.PublicKey, error) {
	publicKey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(authorizedKeyValue))
	return publicKey, err
}

func acquireHostKey(logger lager.Logger) (ssh.Signer, error) {
	var encoded []byte
	if hostKeyPEM == "" {
		return nil, errors.New("empty-host-key")
	} else {
		encoded = []byte(hostKeyPEM)
	}

	key, err := ssh.ParsePrivateKey(encoded)
	if err != nil {
		logger.Error("failed-to-parse-host-key", err)
		return nil, err
	}
	return key, nil
}

func generateNewHostKey() (string, error) {
	hostKeyPair, err := keys.RSAKeyPairFactory.NewKeyPair(1024)

	if err != nil {
		return "", err
	}
	return hostKeyPair.PEMEncodedPrivateKey(), nil
}
<commit_msg>propagate logLevel and debugAddr to diego-sshd process<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"strings"
	"syscall"

	"code.cloudfoundry.org/cflager"
	"code.cloudfoundry.org/debugserver"
	"code.cloudfoundry.org/diego-ssh/authenticators"
	"code.cloudfoundry.org/diego-ssh/daemon"
	"code.cloudfoundry.org/diego-ssh/keys"
	"code.cloudfoundry.org/lager"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/sigmon"
	"golang.org/x/crypto/ssh"
)

var address = flag.String(
	"address",
	"127.0.0.1:2222",
	"listen address for ssh daemon",
)

var hostKey = flag.String(
	"hostKey",
	"",
	"PEM encoded RSA host key",
)

var authorizedKey = flag.String(
	"authorizedKey",
	"",
	"Public key in the OpenSSH authorized_keys format",
)

var allowUnauthenticatedClients = flag.Bool(
	"allowUnauthenticatedClients",
	false,
	"Allow access to unauthenticated clients",
)

var inheritDaemonEnv = flag.Bool(
	"inheritDaemonEnv",
	false,
	"Inherit daemon's environment",
)

var allowedCiphers = flag.String(
	"allowedCiphers",
	"",
	"Limit cipher algorithms to those provided (comma separated)",
)

var allowedMACs = flag.String(
	"allowedMACs",
	"",
	"Limit MAC algorithms to those provided (comma separated)",
)

var allowedKeyExchanges = flag.String(
	"allowedKeyExchanges",
	"",
	"Limit key exchanges algorithms to those provided (comma separated)",
)

var hostKeyPEM string
var authorizedKeyValue string

func main() {
	debugserver.AddFlags(flag.CommandLine)
	cflager.AddFlags(flag.CommandLine)
	flag.Parse()
	exec := false

	logger, reconfigurableSink := cflager.New("sshd")

	hostKeyPEM = os.Getenv("SSHD_HOSTKEY")
	if hostKeyPEM != "" {
		authorizedKeyValue = os.Getenv("SSHD_AUTHKEY")

		// unset the variables so child processes don't inherit them
		os.Unsetenv("SSHD_HOSTKEY")
		os.Unsetenv("SSHD_AUTHKEY")
	} else {
		hostKeyPEM = *hostKey
		if hostKeyPEM == "" {
			var err error
			hostKeyPEM, err = generateNewHostKey()
			if err != nil {
				logger.Error("failed-to-generate-host-key", err)
				os.Exit(1)
			}
		}
		authorizedKeyValue = *authorizedKey
		exec = true
	}

	if exec {
		os.Setenv("SSHD_HOSTKEY", hostKeyPEM)
		os.Setenv("SSHD_AUTHKEY", authorizedKeyValue)

		logLevel := "info"
		flag.CommandLine.Lookup("logLevel")
		logLevelFlag := flag.CommandLine.Lookup("logLevel")
		if logLevelFlag != nil {
			logLevel = logLevelFlag.Value.String()
		}

		err := syscall.Exec(os.Args[0], []string{
			os.Args[0],
			fmt.Sprintf("--allowedKeyExchanges=%s", *allowedKeyExchanges),
			fmt.Sprintf("--address=%s", *address),
			fmt.Sprintf("--allowUnauthenticatedClients=%t", *allowUnauthenticatedClients),
			fmt.Sprintf("--inheritDaemonEnv=%t", *inheritDaemonEnv),
			fmt.Sprintf("--allowedCiphers=%s", *allowedCiphers),
			fmt.Sprintf("--allowedMACs=%s", *allowedMACs),
			fmt.Sprintf("--logLevel=%s", logLevel),
			fmt.Sprintf("--debugAddr=%s", debugserver.DebugAddress(flag.CommandLine)),
		}, os.Environ())
		if err != nil {
			logger.Error("failed-exec", err)
			os.Exit(1)
		}
	}

	serverConfig, err := configure(logger)
	if err != nil {
		logger.Error("configure-failed", err)
		os.Exit(1)
	}

	sshDaemon := daemon.New(logger, serverConfig, nil, newChannelHandlers())
	server, err := createServer(logger, *address, sshDaemon)

	members := grouper.Members{
		{"sshd", server},
	}

	if dbgAddr := debugserver.DebugAddress(flag.CommandLine); dbgAddr != "" {
		members = append(grouper.Members{
			{"debug-server", debugserver.Runner(dbgAddr, reconfigurableSink)},
		}, members...)
	}

	group := grouper.NewOrdered(os.Interrupt, members)
	monitor := ifrit.Invoke(sigmon.New(group))

	logger.Info("started")

	err = <-monitor.Wait()
	if err != nil {
		logger.Error("exited-with-failure", err)
		os.Exit(1)
	}

	logger.Info("exited")
	os.Exit(0)
}

func getDaemonEnvironment() map[string]string {
	daemonEnv := map[string]string{}

	if *inheritDaemonEnv {
		envs := os.Environ()
		for _, env := range envs {
			nvp := strings.SplitN(env, "=", 2)
			if len(nvp) == 2 && nvp[0] != "PATH" {
				daemonEnv[nvp[0]] = nvp[1]
			}
		}
	}
	return daemonEnv
}

func configure(logger lager.Logger) (*ssh.ServerConfig, error) {
	errorStrings := []string{}
	sshConfig := &ssh.ServerConfig{}

	key, err := acquireHostKey(logger)
	if err != nil {
		logger.Error("failed-to-acquire-host-key", err)
		errorStrings = append(errorStrings, err.Error())
	}

	sshConfig.AddHostKey(key)
	sshConfig.NoClientAuth = *allowUnauthenticatedClients

	if authorizedKeyValue == "" && !*allowUnauthenticatedClients {
		logger.Error("authorized-key-required", nil)
		errorStrings = append(errorStrings, "Public user key is required")
	}

	if authorizedKeyValue != "" {
		decodedPublicKey, err := decodeAuthorizedKey(logger)
		if err == nil {
			authenticator := authenticators.NewPublicKeyAuthenticator(decodedPublicKey)
			sshConfig.PublicKeyCallback = authenticator.Authenticate
		} else {
			errorStrings = append(errorStrings, err.Error())
		}
	}

	if *allowedCiphers != "" {
		sshConfig.Config.Ciphers = strings.Split(*allowedCiphers, ",")
	}
	if *allowedMACs != "" {
		sshConfig.Config.MACs = strings.Split(*allowedMACs, ",")
	}
	if *allowedKeyExchanges != "" {
		sshConfig.Config.KeyExchanges = strings.Split(*allowedKeyExchanges, ",")
	}

	err = nil
	if len(errorStrings) > 0 {
		err = errors.New(strings.Join(errorStrings, ", "))
	}

	return sshConfig, err
}

func decodeAuthorizedKey(logger lager.Logger) (ssh.PublicKey, error) {
	publicKey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(authorizedKeyValue))
	return publicKey, err
}

func acquireHostKey(logger lager.Logger) (ssh.Signer, error) {
	var encoded []byte
	if hostKeyPEM == "" {
		return nil, errors.New("empty-host-key")
	} else {
		encoded = []byte(hostKeyPEM)
	}

	key, err := ssh.ParsePrivateKey(encoded)
	if err != nil {
		logger.Error("failed-to-parse-host-key", err)
		return nil, err
	}
	return key, nil
}

func generateNewHostKey() (string, error) {
	hostKeyPair, err := keys.RSAKeyPairFactory.NewKeyPair(1024)

	if err != nil {
		return "", err
	}
	return hostKeyPair.PEMEncodedPrivateKey(), nil
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"fmt"
	"html/template"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"github.com/wader/ydls/internal/ydls"
)

var gitCommit = "dev"

var versionFlag = flag.Bool("version", false, "Print version ("+gitCommit+")")

var debugFlag = flag.Bool("debug", false, "Debug output")
var configFlag = flag.String("config", "ydls.json", "Config file")
var infoFlag = flag.Bool("info", false, "Info output")

var serverFlag = flag.Bool("server", false, "Start server")
var listenFlag = flag.String("listen", ":8080", "Listen address")
var indexFlag = flag.String("index", "", "Path to index template")

func fatalIfErrorf(err error, format string, a ...interface{}) {
	if err != nil {
		a = append(a, err)
		log.Fatalf(format+": %v", a...)
	}
}

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s [flags] URL [format] [options]...:\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()

	if *versionFlag {
		fmt.Println(gitCommit)
		os.Exit(0)
	}
	if os.Getenv("DEBUG") != "" {
		*debugFlag = true
	}
}

func server(y ydls.YDLS) {
	yh := &ydls.Handler{YDLS: y}

	if *infoFlag {
		yh.InfoLog = log.New(os.Stdout, "INFO: ", log.Ltime)
	}
	if *debugFlag {
		yh.DebugLog = log.New(os.Stdout, "DEBUG: ", log.Ltime)
	}
	if *indexFlag != "" {
		indexTmpl, err := template.ParseFiles(*indexFlag)
		fatalIfErrorf(err, "failed to parse index template")
		yh.IndexTmpl = indexTmpl
	}

	log.Printf("Listening on %s", *listenFlag)
	if err := http.ListenAndServe(*listenFlag, yh); err != nil {
		log.Fatal(err)
	}
}

type progressWriter struct {
	fn    func(bytes uint64)
	bytes uint64
}

func (pw *progressWriter) Write(p []byte) (n int, err error) {
	pw.bytes += uint64(len(p))
	pw.fn(pw.bytes)
	return len(p), nil
}

func absRootPath(root string, path string) (string, error) {
	abs, err := filepath.Abs(filepath.Join(root, path))
	if err != nil {
		return "", err
	}
	if !strings.HasPrefix(abs, filepath.Clean(root+string(filepath.Separator))) {
		return "", fmt.Errorf("%s is outside root path %s", abs, root)
	}

	return abs, nil
}

func download(y ydls.YDLS) {
	var debugLog ydls.Printer
	if *debugFlag {
		debugLog = log.New(os.Stdout, "DEBUG: ", log.Ltime)
	}

	rawURL := flag.Arg(0)
	if rawURL == "" {
		log.Fatalf("no URL specified")
	}

	downloadOptions, downloadOptionsErr := ydls.NewDownloadOptionsFromOpts(flag.Args()[1:], y.Config.Formats)
	downloadOptions.MediaRawURL = flag.Arg(0)
	fatalIfErrorf(downloadOptionsErr, "format and options")

	ctx, cancelFn := context.WithCancel(context.Background())
	defer cancelFn()

	dr, err := y.Download(ctx, downloadOptions, debugLog)
	fatalIfErrorf(err, "download failed")
	defer dr.Media.Close()
	defer dr.Wait()

	wd, err := os.Getwd()
	fatalIfErrorf(err, "getwd")

	path, err := absRootPath(wd, dr.Filename)
	fatalIfErrorf(err, "write path")

	var mediaWriter io.Writer
	if dr.Filename != "" {
		mediaFile, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
		fatalIfErrorf(err, "failed to open file")
		defer mediaFile.Close()
		pw := &progressWriter{fn: func(bytes uint64) {
			fmt.Printf("\r%s %.2fMB", dr.Filename, float64(bytes)/(1024*1024))
		}}
		mediaWriter = io.MultiWriter(mediaFile, pw)
	} else {
		mediaWriter = os.Stdout
	}

	io.Copy(mediaWriter, dr.Media)
	dr.Media.Close()
	dr.Wait()
	fmt.Print("\n")
}

func main() {
	y, err := ydls.NewFromFile(*configFlag)
	fatalIfErrorf(err, "failed to read config")

	if *serverFlag {
		server(y)
	} else {
		download(y)
	}
}
<commit_msg>Improve cmd help a bit<commit_after>package main

import (
	"context"
	"flag"
	"fmt"
	"html/template"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"github.com/wader/ydls/internal/ydls"
)

var gitCommit = "dev"

var versionFlag = flag.Bool("version", false, "Print version ("+gitCommit+")")

var debugFlag = flag.Bool("debug", false, "Debug output")
var configFlag = flag.String("config", "ydls.json", "Config file")
var infoFlag = flag.Bool("info", false, "Info output")

var serverFlag = flag.Bool("server", false, "Start server")
var listenFlag = flag.String("listen", ":8080", "Listen address")
var indexFlag = flag.String("index", "", "Path to index template")

func fatalIfErrorf(err error, format string, a ...interface{}) {
	if err != nil {
		a = append(a, err)
		log.Fatalf(format+": %v", a...)
	}
}

func init() {
	log.SetFlags(0)
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [flags] URL [options]...\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()

	if *versionFlag {
		fmt.Println(gitCommit)
		os.Exit(0)
	}
	if os.Getenv("DEBUG") != "" {
		*debugFlag = true
	}
}

func server(y ydls.YDLS) {
	yh := &ydls.Handler{YDLS: y}

	if *infoFlag {
		yh.InfoLog = log.New(os.Stdout, "INFO: ", log.Ltime)
	}
	if *debugFlag {
		yh.DebugLog = log.New(os.Stdout, "DEBUG: ", log.Ltime)
	}
	if *indexFlag != "" {
		indexTmpl, err := template.ParseFiles(*indexFlag)
		fatalIfErrorf(err, "failed to parse index template")
		yh.IndexTmpl = indexTmpl
	}

	log.Printf("Listening on %s", *listenFlag)
	if err := http.ListenAndServe(*listenFlag, yh); err != nil {
		log.Fatal(err)
	}
}

type progressWriter struct {
	fn    func(bytes uint64)
	bytes uint64
}

func (pw *progressWriter) Write(p []byte) (n int, err error) {
	pw.bytes += uint64(len(p))
	pw.fn(pw.bytes)
	return len(p), nil
}

func absRootPath(root string, path string) (string, error) {
	abs, err := filepath.Abs(filepath.Join(root, path))
	if err != nil {
		return "", err
	}
	if !strings.HasPrefix(abs, filepath.Clean(root+string(filepath.Separator))) {
		return "", fmt.Errorf("%s is outside root path %s", abs, root)
	}

	return abs, nil
}

func download(y ydls.YDLS) {
	var debugLog ydls.Printer
	if *debugFlag {
		debugLog = log.New(os.Stdout, "DEBUG: ", log.Ltime)
	}

	rawURL := flag.Arg(0)
	if rawURL == "" {
		flag.Usage()
		os.Exit(1)
	}

	downloadOptions, downloadOptionsErr := ydls.NewDownloadOptionsFromOpts(flag.Args()[1:], y.Config.Formats)
	downloadOptions.MediaRawURL = flag.Arg(0)
	fatalIfErrorf(downloadOptionsErr, "format and options")

	ctx, cancelFn := context.WithCancel(context.Background())
	defer cancelFn()

	dr, err := y.Download(ctx, downloadOptions, debugLog)
	fatalIfErrorf(err, "download failed")
	defer dr.Media.Close()
	defer dr.Wait()

	wd, err := os.Getwd()
	fatalIfErrorf(err, "getwd")

	path, err := absRootPath(wd, dr.Filename)
	fatalIfErrorf(err, "write path")

	var mediaWriter io.Writer
	if dr.Filename != "" {
		mediaFile, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
		fatalIfErrorf(err, "failed to open file")
		defer mediaFile.Close()
		pw := &progressWriter{fn: func(bytes uint64) {
			fmt.Printf("\r%s %.2fMB", dr.Filename, float64(bytes)/(1024*1024))
		}}
		mediaWriter = io.MultiWriter(mediaFile, pw)
	} else {
		mediaWriter = os.Stdout
	}

	io.Copy(mediaWriter, dr.Media)
	dr.Media.Close()
	dr.Wait()
	fmt.Print("\n")
}

func main() {
	y, err := ydls.NewFromFile(*configFlag)
	fatalIfErrorf(err, "failed to read config")

	if *serverFlag {
		server(y)
	} else {
		download(y)
	}
}
<|endoftext|>


<commit_before>package seahash

import (
	"fmt"
	"testing"
)

func TestHash(t *testing.T) {
	testCases := []struct {
		s string
		n uint64
	}{
		{"to be or not to be", 1988685042348123509},
		{"love is a wonderful terrible thing", 4784284276849692846},
	}
	for _, tc := range testCases {
		t.Run(tc.s, func(t *testing.T) {
			t.Parallel()
			if sum := SumString(tc.s); sum != tc.n {
				t.Errorf("got %v, want %v", sum, tc.n)
			}
		})
	}
}

func TestNotEqual(t *testing.T) {
	testCases := []struct {
		a, b string
	}{
		{"to be or not to be ", "to be or not to be"},
		{"jkjke", "jkjk"},
		{"ijkjke", "ijkjk"},
		{"iijkjke", "iijkjk"},
		{"iiijkjke", "iiijkjk"},
		{"iiiijkjke", "iiiijkjk"},
		{"iiiiijkjke", "iiiiijkjk"},
		{"iiiiiijkjke", "iiiiiijkjk"},
		{"iiiiiiijkjke", "iiiiiiijkjk"},
		{"iiiiiiiijkjke", "iiiiiiiijkjk"},
		{"ab", "bb"},
	}
	for _, tc := range testCases {
		t.Run(tc.b, func(t *testing.T) {
			t.Parallel()
			if SumString(tc.a) == SumString(tc.b) {
				t.Fail()
			}
		})
	}
}

func TestZeroSenitive(t *testing.T) {
	testCases := []struct {
		a, b []byte
	}{
		{[]byte{1, 2, 3, 4}, []byte{1, 0, 2, 3, 4}},
		{[]byte{1, 2, 3, 4}, []byte{1, 0, 0, 2, 3, 4}},
		{[]byte{1, 2, 3, 4}, []byte{1, 2, 3, 4, 0}},
		{[]byte{1, 2, 3, 4}, []byte{0, 1, 2, 3, 4}},
		{[]byte{0, 0, 0}, []byte{0, 0, 0, 0, 0}},
	}
	for _, tc := range testCases {
		t.Run(fmt.Sprintf("% x", tc.b), func(t *testing.T) {
			t.Parallel()
			if Sum(tc.a) == Sum(tc.b) {
				t.Fail()
			}
		})
	}
}

func BenchmarkHash(b *testing.B) {
	d := New()
	p := []byte("to be or not to be")
	for i := 0; i < b.N; i++ {
		d.Write(p)
	}
}
<commit_msg>Better test names<commit_after>package seahash

import (
	"fmt"
	"testing"
)

func TestHash(t *testing.T) {
	testCases := []struct {
		s string
		n uint64
	}{
		{"to be or not to be", 1988685042348123509},
		{"love is a wonderful terrible thing", 4784284276849692846},
	}
	for _, tc := range testCases {
		t.Run(tc.s, func(t *testing.T) {
			t.Parallel()
			if sum := SumString(tc.s); sum != tc.n {
				t.Errorf("got %v, want %v", sum, tc.n)
			}
		})
	}
}

func TestNotEqual(t *testing.T) {
	testCases := []struct {
		a, b string
	}{
		{"to be or not to be ", "to be or not to be"},
		{"jkjke", "jkjk"},
		{"ijkjke", "ijkjk"},
		{"iijkjke", "iijkjk"},
		{"iiijkjke", "iiijkjk"},
		{"iiiijkjke", "iiiijkjk"},
		{"iiiiijkjke", "iiiiijkjk"},
		{"iiiiiijkjke", "iiiiiijkjk"},
		{"iiiiiiijkjke", "iiiiiiijkjk"},
		{"iiiiiiiijkjke", "iiiiiiiijkjk"},
		{"ab", "bb"},
	}
	for _, tc := range testCases {
		t.Run(fmt.Sprintf("%v %v", tc.a, tc.b), func(t *testing.T) {
			t.Parallel()
			if SumString(tc.a) == SumString(tc.b) {
				t.Fail()
			}
		})
	}
}

func TestZeroSenitive(t *testing.T) {
	testCases := []struct {
		a, b []byte
	}{
		{[]byte{1, 2, 3, 4}, []byte{1, 0, 2, 3, 4}},
		{[]byte{1, 2, 3, 4}, []byte{1, 0, 0, 2, 3, 4}},
		{[]byte{1, 2, 3, 4}, []byte{1, 2, 3, 4, 0}},
		{[]byte{1, 2, 3, 4}, []byte{0, 1, 2, 3, 4}},
		{[]byte{0, 0, 0}, []byte{0, 0, 0, 0, 0}},
	}
	for _, tc := range testCases {
		t.Run(fmt.Sprintf("%x %x", tc.a, tc.b), func(t *testing.T) {
			t.Parallel()
			if Sum(tc.a) == Sum(tc.b) {
				t.Fail()
			}
		})
	}
}

func BenchmarkHash(b *testing.B) {
	d := New()
	p := []byte("to be or not to be")
	for i := 0; i < b.N; i++ {
		d.Write(p)
	}
}
<|endoftext|>


<commit_before>package secureconfig

import (
	"crypto"
	"crypto/rand"
	"crypto/rsa"
	_ "crypto/sha256"
	"encoding/json"
	"fmt"
)

type SecureConfig struct {
	Key     *rsa.PrivateKey `json:"-"`
	Secure  secureTuple     `json:"secure"`
	Private secureTuple     `json:"private"`
}

type secureTuple struct {
	Data      interface{} `json:"data"`
	Signature []byte      `json:"signature"`
}

type privateTuple struct {
	Data          interface{} `json:"-"`
	EncryptedData []byte      `json:"data"`
	Signature     []byte      `json:"signature"`
}

func SecureConfigWithPrivateKey(key *rsa.PrivateKey) *SecureConfig {
	return &SecureConfig{
		Key: key,
	}
}

func (sc *SecureConfig) SetSecureData(data interface{}) error {
	sc.Secure.Data = data
	// marshal it all into a byte format, hash it, sign it, store the signature

	b, err := json.Marshal(sc.Secure.Data)
	if err != nil {
		return err
	}

	fmt.Printf("marshalled data: %s\n", b)

	hash := crypto.Hash(crypto.SHA256).New().Sum(b)

	fmt.Printf("hash: %v\n", hash)

	sig, err := rsa.SignPSS(rand.Reader, sc.Key, crypto.Hash(crypto.SHA256), hash, nil)
	if err != nil {
		return err
	}

	fmt.Printf("sig: %v\n", sig)

	sc.Secure.Signature = []byte("foo")

	return nil
}

func (sc *SecureConfig) SetPrivateData(data interface{}) {
	sc.Private.Data = data
	// marshal it all into a byte format, encrypt it, hash it, sign it, store the signature
}
<commit_msg>Fix hashing<commit_after>package secureconfig

import (
	"crypto"
	"crypto/rand"
	"crypto/rsa"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
)

type SecureConfig struct {
	Key     *rsa.PrivateKey `json:"-"`
	Secure  secureTuple     `json:"secure"`
	Private secureTuple     `json:"private"`
}

type secureTuple struct {
	Data      interface{} `json:"data"`
	Signature []byte      `json:"signature"`
}

type privateTuple struct {
	Data          interface{} `json:"-"`
	EncryptedData []byte      `json:"data"`
	Signature     []byte      `json:"signature"`
}

func SecureConfigWithPrivateKey(key *rsa.PrivateKey) *SecureConfig {
	return &SecureConfig{
		Key: key,
	}
}

func (sc *SecureConfig) SetSecureData(data interface{}) error {
	sc.Secure.Data = data
	// marshal it all into a byte format, hash it, sign it, store the signature

	b, err := json.Marshal(sc.Secure.Data)
	if err != nil {
		return err
	}

	fmt.Printf("marshalled data: %s\n", b)

	hashed := sha256.Sum256(b)

	sig, err := rsa.SignPSS(rand.Reader, sc.Key, crypto.SHA256, hashed[:], nil)
	if err != nil {
		return err
	}

	sc.Secure.Signature = make([]byte, len(sig)*2)
	hex.Encode(sc.Secure.Signature, sig)

	return nil
}

func (sc *SecureConfig) SetPrivateData(data interface{}) {
	sc.Private.Data = data
	// marshal it all into a byte format, encrypt it, hash it, sign it, store the signature
}
<|endoftext|>


<commit_before>package sensu

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"os"
	"strings"

	"github.com/upfluence/sensu-client-go/Godeps/_workspace/src/github.com/upfluence/sensu-go/sensu/check"
	"github.com/upfluence/sensu-client-go/Godeps/_workspace/src/github.com/upfluence/sensu-go/sensu/client"
	"github.com/upfluence/sensu-client-go/Godeps/_workspace/src/github.com/upfluence/sensu-go/sensu/transport/rabbitmq"
)

const defaultRabbitMQURI string = "amqp://guest:guest@localhost:5672/%2F"

var errNoClientName = errors.New("No client name provided")

type configFlagSet struct {
	configFile string
	verbose    bool
}

type Config struct {
	flagSet *configFlagSet
	config  *configPayload
}

type configPayload struct {
	Client            *client.Client                      `json:"client,omitempty"`
	Checks            []*check.Check                      `json:"checks,omitempty"`
	RabbitMQURI       *string                             `json:"rabbitmq_uri,omitempty"`
	RabbitMQTransport []*rabbitmq.RabbitMQTransportConfig `json:"rabbitmq,omitempty"`
}

func fetchEnv(envs ...string) string {
	for _, env := range envs {
		if v := os.Getenv(env); v != "" {
			return v
		}
	}

	return ""
}

// split is a wrapper for strings.Split, which returns an empty array
// for empty string inputs instead of an array containing an empty string
func split(str string, token string) []string {
	if len(str) == 0 {
		return []string{}
	}

	return strings.Split(str, token)
}

func NewConfigFromFile(flagset *configFlagSet, configFile string) (*Config, error) {
	cfg := Config{flagset, &configPayload{}}

	if configFile != "" {
		buf, err := ioutil.ReadFile(configFile)

		if err != nil {
			return nil, err
		}

		if err := json.Unmarshal(buf, &cfg.config); err != nil {
			return nil, err
		}
	}

	if cfg.Client().Name == "" {
		return nil, errNoClientName
	}

	return &cfg, nil
}

func NewConfigFromFlagSet(flagset *configFlagSet) (*Config, error) {
	var file string

	if flagset != nil {
		file = flagset.configFile
	}

	return NewConfigFromFile(flagset, file)
}

func (c *Config) RabbitMQURI() string {
	if cfg := c.config; cfg != nil && cfg.RabbitMQURI != nil {
		return *cfg.RabbitMQURI
	} else if uri := fetchEnv("RABBITMQ_URI", "RABBITMQ_URL"); uri != "" {
		return uri
	}

	return defaultRabbitMQURI
}

// RabbitMQHAConfig superseeds RabbitMQURI() by first checking
// for a HA cluster configuration and then calling RabbitMQURI()
// if it can't find one
func (c *Config) RabbitMQHAConfig() []*rabbitmq.RabbitMQTransportConfig {
	if cfg := c.config; cfg != nil && cfg.RabbitMQTransport != nil {
		return cfg.RabbitMQTransport
	}

	config, _ := rabbitmq.NewRabbitMQTransportConfig(c.RabbitMQURI())

	return []*rabbitmq.RabbitMQTransportConfig{config}
}

func (c *Config) Client() *client.Client {
	if cfg := c.config; cfg != nil && cfg.Client != nil {
		return cfg.Client
	}

	return &client.Client{
		Name:          os.Getenv("SENSU_CLIENT_NAME"),
		Address:       fetchEnv("SENSU_CLIENT_ADDRESS", "SENSU_ADDRESS"),
		Subscriptions: split(os.Getenv("SENSU_CLIENT_SUBSCRIPTIONS"), ","),
	}
}

func (c *Config) Checks() []*check.Check {
	if cfg := c.config; cfg != nil {
		return cfg.Checks
	}

	return []*check.Check{}
}
<commit_msg>Apply RabbitMQTransportConfig -> TransportConfig rename<commit_after>package sensu

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"os"
	"strings"

	"github.com/upfluence/sensu-client-go/Godeps/_workspace/src/github.com/upfluence/sensu-go/sensu/check"
	"github.com/upfluence/sensu-client-go/Godeps/_workspace/src/github.com/upfluence/sensu-go/sensu/client"
	"github.com/upfluence/sensu-client-go/Godeps/_workspace/src/github.com/upfluence/sensu-go/sensu/transport/rabbitmq"
)

const defaultRabbitMQURI string = "amqp://guest:guest@localhost:5672/%2F"

var errNoClientName = errors.New("No client name provided")

type configFlagSet struct {
	configFile string
	verbose    bool
}

type Config struct {
	flagSet *configFlagSet
	config  *configPayload
}

type configPayload struct {
	Client            *client.Client              `json:"client,omitempty"`
	Checks            []*check.Check              `json:"checks,omitempty"`
	RabbitMQURI       *string                     `json:"rabbitmq_uri,omitempty"`
	RabbitMQTransport []*rabbitmq.TransportConfig `json:"rabbitmq,omitempty"`
}

func fetchEnv(envs ...string) string {
	for _, env := range envs {
		if v := os.Getenv(env); v != "" {
			return v
		}
	}

	return ""
}

// split is a wrapper for strings.Split, which returns an empty array
// for empty string inputs instead of an array containing an empty string
func split(str string, token string) []string {
	if len(str) == 0 {
		return []string{}
	}

	return strings.Split(str, token)
}

func NewConfigFromFile(flagset *configFlagSet, configFile string) (*Config, error) {
	cfg := Config{flagset, &configPayload{}}

	if configFile != "" {
		buf, err := ioutil.ReadFile(configFile)

		if err != nil {
			return nil, err
		}

		if err := json.Unmarshal(buf, &cfg.config); err != nil {
			return nil, err
		}
	}

	if cfg.Client().Name == "" {
		return nil, errNoClientName
	}

	return &cfg, nil
}

func NewConfigFromFlagSet(flagset *configFlagSet) (*Config, error) {
	var file string

	if flagset != nil {
		file = flagset.configFile
	}

	return NewConfigFromFile(flagset, file)
}

func (c *Config) RabbitMQURI() string {
	if cfg := c.config; cfg != nil && cfg.RabbitMQURI != nil {
		return *cfg.RabbitMQURI
	} else if uri := fetchEnv("RABBITMQ_URI", "RABBITMQ_URL"); uri != "" {
		return uri
	}

	return defaultRabbitMQURI
}

// RabbitMQHAConfig superseeds RabbitMQURI() by first checking
// for a HA cluster configuration and then calling RabbitMQURI()
// if it can't find one
func (c *Config) RabbitMQHAConfig() []*rabbitmq.TransportConfig {
	if cfg := c.config; cfg != nil && cfg.RabbitMQTransport != nil {
		return cfg.RabbitMQTransport
	}

	config, _ := rabbitmq.NewTransportConfig(c.RabbitMQURI())

	return []*rabbitmq.TransportConfig{config}
}

func (c *Config) Client() *client.Client {
	if cfg := c.config; cfg != nil && cfg.Client != nil {
		return cfg.Client
	}

	return &client.Client{
		Name:          os.Getenv("SENSU_CLIENT_NAME"),
		Address:       fetchEnv("SENSU_CLIENT_ADDRESS", "SENSU_ADDRESS"),
		Subscriptions: split(os.Getenv("SENSU_CLIENT_SUBSCRIPTIONS"), ","),
	}
}

func (c *Config) Checks() []*check.Check {
	if cfg := c.config; cfg != nil {
		return cfg.Checks
	}

	return []*check.Check{}
}
<|endoftext|>


<commit_before>// +build !windows

package goserial

// #include <termios.h>
// #include <unistd.h>
import "C"

// TODO: Maybe change to using syscall package + ioctl instead of cgo

import (
	"errors"
	"fmt"
	"io"
	"os"
	"syscall"
	//"unsafe"
)

func openPort(name string, c *Config) (rwc io.ReadWriteCloser, err error) {
	f, err := os.OpenFile(name, syscall.O_RDWR|syscall.O_NOCTTY|syscall.O_NONBLOCK, 0666)
	if err != nil {
		return
	}

	defer func() {
		if err != nil {
			f.Close()
		}
	}()

	fd := C.int(f.Fd())
	if C.isatty(fd) != 1 {
		return nil, errors.New("File is not a tty")
	}

	var st C.struct_termios
	_, err = C.tcgetattr(fd, &st)
	if err != nil {
		return nil, err
	}
	var speed C.speed_t
	switch c.Baud {
	case 115200:
		speed = C.B115200
	case 57600:
		speed = C.B57600
	case 38400:
		speed = C.B38400
	case 19200:
		speed = C.B19200
	case 9600:
		speed = C.B9600
	default:
		return nil, fmt.Errorf("Unknown baud rate %v", c.Baud)
	}

	_, err = C.cfsetispeed(&st, speed)
	if err != nil {
		return nil, err
	}
	_, err = C.cfsetospeed(&st, speed)
	if err != nil {
		return nil, err
	}

	// Select local mode
	st.c_cflag |= C.CLOCAL | C.CREAD

	// Select stop bits
	switch c.StopBits {
	case StopBits1:
		st.c_cflag &^= C.CSTOPB
	case StopBits2:
		st.c_cflag |= C.CSTOPB
	default:
		panic(c.StopBits)
	}

	// Select character size
	st.c_cflag &^= C.CSIZE
	switch c.Size {
	case 5:
		st.c_cflag |= C.CS5
	case 6:
		st.c_cflag |= C.CS6
	case 7:
		st.c_cflag |= C.CS7
	case 8:
		st.c_cflag |= C.CS8
	default:
		panic(c.Size)
	}

	// Select parity mode
	switch c.Parity {
	case ParityNone:
		st.c_cflag &^= C.PARENB
	case ParityEven:
		st.c_cflag |= C.PARENB
		st.c_cflag &^= C.PARODD
	case ParityOdd:
		st.c_cflag |= C.PARENB
		st.c_cflag |= C.PARODD
	default:
		panic(c.Parity)
	}

	// Select CRLF translation
	if c.CRLFTranslate {
		st.c_iflag |= C.ICRNL
	} else {
		st.c_iflag &^= C.ICRNL
	}

	// Select raw mode
	st.c_lflag &^= C.ICANON | C.ECHO | C.ECHOE | C.ISIG
	st.c_oflag &^= C.OPOST

	_, err = C.tcsetattr(fd, C.TCSANOW, &st)
	if err != nil {
		return nil, err
	}

	//fmt.Println("Tweaking", name)
	r1, _, e := syscall.Syscall(syscall.SYS_FCNTL,
		uintptr(f.Fd()),
		uintptr(syscall.F_SETFL),
		uintptr(0))
	if e != 0 || r1 != 0 {
		s := fmt.Sprint("Clearing NONBLOCK syscall error:", e, r1)
		return nil, errors.New(s)
	}

	/*
				r1, _, e = syscall.Syscall(syscall.SYS_IOCTL,
			                uintptr(f.Fd()),
			                uintptr(0x80045402), // IOSSIOSPEED
			                uintptr(unsafe.Pointer(&baud)));
			        if e != 0 || r1 != 0 {
			                s := fmt.Sprint("Baudrate syscall error:", e, r1)
		                        return nil, os.NewError(s)
				}
	*/

	return f, nil
}
<commit_msg>Fix serious bug with posix byte size selection.<commit_after>// +build !windows

package goserial

// #include <termios.h>
// #include <unistd.h>
import "C"

// TODO: Maybe change to using syscall package + ioctl instead of cgo

import (
	"errors"
	"fmt"
	"io"
	"os"
	"syscall"
	//"unsafe"
)

func openPort(name string, c *Config) (rwc io.ReadWriteCloser, err error) {
	f, err := os.OpenFile(name, syscall.O_RDWR|syscall.O_NOCTTY|syscall.O_NONBLOCK, 0666)
	if err != nil {
		return
	}

	defer func() {
		if err != nil {
			f.Close()
		}
	}()

	fd := C.int(f.Fd())
	if C.isatty(fd) != 1 {
		return nil, errors.New("File is not a tty")
	}

	var st C.struct_termios
	_, err = C.tcgetattr(fd, &st)
	if err != nil {
		return nil, err
	}
	var speed C.speed_t
	switch c.Baud {
	case 115200:
		speed = C.B115200
	case 57600:
		speed = C.B57600
	case 38400:
		speed = C.B38400
	case 19200:
		speed = C.B19200
	case 9600:
		speed = C.B9600
	default:
		return nil, fmt.Errorf("Unknown baud rate %v", c.Baud)
	}

	_, err = C.cfsetispeed(&st, speed)
	if err != nil {
		return nil, err
	}
	_, err = C.cfsetospeed(&st, speed)
	if err != nil {
		return nil, err
	}

	// Select local mode
	st.c_cflag |= C.CLOCAL | C.CREAD

	// Select stop bits
	switch c.StopBits {
	case StopBits1:
		st.c_cflag &^= C.CSTOPB
	case StopBits2:
		st.c_cflag |= C.CSTOPB
	default:
		panic(c.StopBits)
	}

	// Select character size
	st.c_cflag &^= C.CSIZE
	switch c.Size {
	case Byte5:
		st.c_cflag |= C.CS5
	case Byte6:
		st.c_cflag |= C.CS6
	case Byte7:
		st.c_cflag |= C.CS7
	case Byte8:
		st.c_cflag |= C.CS8
	default:
		panic(c.Size)
	}

	// Select parity mode
	switch c.Parity {
	case ParityNone:
		st.c_cflag &^= C.PARENB
	case ParityEven:
		st.c_cflag |= C.PARENB
		st.c_cflag &^= C.PARODD
	case ParityOdd:
		st.c_cflag |= C.PARENB
		st.c_cflag |= C.PARODD
	default:
		panic(c.Parity)
	}

	// Select CRLF translation
	if c.CRLFTranslate {
		st.c_iflag |= C.ICRNL
	} else {
		st.c_iflag &^= C.ICRNL
	}

	// Select raw mode
	st.c_lflag &^= C.ICANON | C.ECHO | C.ECHOE | C.ISIG
	st.c_oflag &^= C.OPOST

	_, err = C.tcsetattr(fd, C.TCSANOW, &st)
	if err != nil {
		return nil, err
	}

	//fmt.Println("Tweaking", name)
	r1, _, e := syscall.Syscall(syscall.SYS_FCNTL,
		uintptr(f.Fd()),
		uintptr(syscall.F_SETFL),
		uintptr(0))
	if e != 0 || r1 != 0 {
		s := fmt.Sprint("Clearing NONBLOCK syscall error:", e, r1)
		return nil, errors.New(s)
	}

	/*
				r1, _, e = syscall.Syscall(syscall.SYS_IOCTL,
			                uintptr(f.Fd()),
			                uintptr(0x80045402), // IOSSIOSPEED
			                uintptr(unsafe.Pointer(&baud)));
			        if e != 0 || r1 != 0 {
			                s := fmt.Sprint("Baudrate syscall error:", e, r1)
		                        return nil, os.NewError(s)
				}
	*/

	return f, nil
}
<|endoftext|>