<commit_before>package main

import (
	"fmt"
	"net"
	"net/url"
	"os"
	"os/exec"
	"path"
	"strings"
	"syscall"

	"github.com/lxc/lxd"
	"github.com/lxc/lxd/i18n"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/gnuflag"
	"github.com/lxc/lxd/shared/logging"
)

var configPath string

func main() {
	if err := run(); err != nil {
		// The action we take depends on the error we get.
		msg := fmt.Sprintf(i18n.G("error: %v"), err)
		switch t := err.(type) {
		case *url.Error:
			switch u := t.Err.(type) {
			case *net.OpError:
				if u.Op == "dial" && u.Net == "unix" {
					switch errno := u.Err.(type) {
					case syscall.Errno:
						switch errno {
						case syscall.ENOENT:
							msg = i18n.G("LXD socket not found; is LXD running?")
						case syscall.ECONNREFUSED:
							msg = i18n.G("Connection refused; is LXD running?")
						case syscall.EACCES:
							msg = i18n.G("Permisson denied, are you in the lxd group?")
						default:
							msg = fmt.Sprintf("%d %s", uintptr(errno), errno.Error())
						}
					}
				}
			}
		}

		fmt.Fprintln(os.Stderr, fmt.Sprintf("%s", msg))
		os.Exit(1)
	}
}

func run() error {
	verbose := gnuflag.Bool("verbose", false, i18n.G("Enables verbose mode."))
	debug := gnuflag.Bool("debug", false, i18n.G("Enables debug mode."))
	forceLocal := gnuflag.Bool("force-local", false, i18n.G("Force using the local unix socket."))
	noAlias := gnuflag.Bool("no-alias", false, i18n.G("Ignore aliases when determining what command to run."))

	configDir := "$HOME/.config/lxc"
	if os.Getenv("LXD_CONF") != "" {
		configDir = os.Getenv("LXD_CONF")
	}
	configPath = os.ExpandEnv(path.Join(configDir, "config.yml"))

	if len(os.Args) >= 3 && os.Args[1] == "config" && os.Args[2] == "profile" {
		fmt.Fprintf(os.Stderr, i18n.G("`lxc config profile` is deprecated, please use `lxc profile`")+"\n")
		os.Args = append(os.Args[:1], os.Args[2:]...)
	}

	if len(os.Args) >= 2 && (os.Args[1] == "-h" || os.Args[1] == "--help") {
		os.Args[1] = "help"
	}

	if len(os.Args) >= 2 && (os.Args[1] == "--all") {
		os.Args[1] = "help"
		os.Args = append(os.Args, "--all")
	}

	if len(os.Args) == 2 && os.Args[1] == "--version" {
		os.Args[1] = "version"
	}

	if len(os.Args) < 2 {
		commands["help"].run(nil, nil)
		os.Exit(1)
	}

	var config *lxd.Config
	var err error

	if *forceLocal {
		config = &lxd.DefaultConfig
	} else {
		config, err = lxd.LoadConfig(configPath)
		if err != nil {
			return err
		}

		// One time migration from old config
		if config.DefaultRemote == "" {
			_, ok := config.Remotes["local"]
			if !ok {
				config.Remotes["local"] = lxd.LocalRemote
			}
			config.DefaultRemote = "local"
			lxd.SaveConfig(config, configPath)
		}
	}

	// This is quite impolite, but it seems gnuflag needs us to shift our
	// own exename out of the arguments before parsing them. However, this
	// is useful for execIfAlias, which wants to know exactly the command
	// line we received, and in some cases is called before this shift, and
	// in others after. So, let's save the original args.
	origArgs := os.Args
	name := os.Args[1]

	/* at this point we haven't parsed the args, so we have to look for
	 * --no-alias by hand.
	 */
	if !shared.StringInSlice("--no-alias", origArgs) {
		execIfAliases(config, origArgs)
	}
	cmd, ok := commands[name]
	if !ok {
		commands["help"].run(nil, nil)
		fmt.Fprintf(os.Stderr, "\n"+i18n.G("error: unknown command: %s")+"\n", name)
		os.Exit(1)
	}
	cmd.flags()
	gnuflag.Usage = func() {
		fmt.Fprintf(os.Stderr, i18n.G("Usage: %s")+"\n\n"+i18n.G("Options:")+"\n\n", strings.TrimSpace(cmd.usage()))
		gnuflag.PrintDefaults()
	}

	os.Args = os.Args[1:]
	gnuflag.Parse(true)

	shared.Log, err = logging.GetLogger("", "", *verbose, *debug, nil)
	if err != nil {
		return err
	}

	certf := config.ConfigPath("client.crt")
	keyf := config.ConfigPath("client.key")

	if !*forceLocal && os.Args[0] != "help" && os.Args[0] != "version" && (!shared.PathExists(certf) || !shared.PathExists(keyf)) {
		fmt.Fprintf(os.Stderr, i18n.G("Generating a client certificate. This may take a minute...")+"\n")

		err = shared.FindOrGenCert(certf, keyf)
		if err != nil {
			return err
		}

		fmt.Fprintf(os.Stderr, i18n.G("If this is your first run, you will need to import images using the 'lxd-images' script.")+"\n")
		fmt.Fprintf(os.Stderr, i18n.G("For example: 'lxd-images import ubuntu --alias ubuntu'.")+"\n")
	}

	err = cmd.run(config, gnuflag.Args())
	if err == errArgs {
		/* If we got an error about invalid arguments, let's try to
		 * expand this as an alias
		 */
		if !*noAlias {
			execIfAliases(config, origArgs)
		}
		fmt.Fprintf(os.Stderr, "%s\n\n"+i18n.G("error: %v")+"\n", cmd.usage(), err)
		os.Exit(1)
	}
	return err
}

type command interface {
	usage() string
	flags()
	showByDefault() bool
	run(config *lxd.Config, args []string) error
}

var commands = map[string]command{
	"config":   &configCmd{},
	"copy":     &copyCmd{},
	"delete":   &deleteCmd{},
	"exec":     &execCmd{},
	"file":     &fileCmd{},
	"finger":   &fingerCmd{},
	"help":     &helpCmd{},
	"image":    &imageCmd{},
	"info":     &infoCmd{},
	"init":     &initCmd{},
	"launch":   &launchCmd{},
	"list":     &listCmd{},
	"monitor":  &monitorCmd{},
	"move":     &moveCmd{},
	"pause":    &actionCmd{shared.Freeze, false, false, "pause"},
	"profile":  &profileCmd{},
	"publish":  &publishCmd{},
	"remote":   &remoteCmd{},
	"restart":  &actionCmd{shared.Restart, true, true, "restart"},
	"restore":  &restoreCmd{},
	"snapshot": &snapshotCmd{},
	"start":    &actionCmd{shared.Start, false, true, "start"},
	"stop":     &actionCmd{shared.Stop, true, true, "stop"},
	"version":  &versionCmd{},
}

var errArgs = fmt.Errorf(i18n.G("wrong number of subcommand arguments"))

func expandAlias(config *lxd.Config, origArgs []string) ([]string, bool) {
	foundAlias := false
	aliasKey := []string{}
	aliasValue := []string{}

	for k, v := range config.Aliases {
		matches := false
		for i, key := range strings.Split(k, " ") {
			if len(origArgs) <= i+1 {
				break
			}

			if origArgs[i+1] == key {
				matches = true
				aliasKey = strings.Split(k, " ")
				aliasValue = strings.Split(v, " ")
				break
			}
		}

		if !matches {
			continue
		}

		foundAlias = true
		break

		break
	}

	if !foundAlias {
		return []string{}, false
	}

	newArgs := []string{origArgs[0]}
	hasReplacedArgsVar := false

	for i, aliasArg := range aliasValue {
		if aliasArg == "@ARGS@" && len(origArgs) > i {
			newArgs = append(newArgs, origArgs[i+1:]...)
			hasReplacedArgsVar = true
		} else {
			newArgs = append(newArgs, aliasArg)
		}
	}

	if !hasReplacedArgsVar {
		/* add the rest of the arguments */
		newArgs = append(newArgs, origArgs[len(aliasKey)+1:]...)
	}

	/* don't re-do aliases the next time; this allows us to have recursive
	 * aliases, e.g. `lxc list` to `lxc list -c n`
	 */
	newArgs = append(newArgs[:2], append([]string{"--no-alias"}, newArgs[2:]...)...)

	return newArgs, true
}

func execIfAliases(config *lxd.Config, origArgs []string) {
	newArgs, expanded := expandAlias(config, origArgs)
	if !expanded {
		return
	}

	path, err := exec.LookPath(origArgs[0])
	if err != nil {
		fmt.Fprintf(os.Stderr, i18n.G("processing aliases failed %s\n"), err)
		os.Exit(5)
	}
	ret := syscall.Exec(path, newArgs, syscall.Environ())
	fmt.Fprintf(os.Stderr, i18n.G("processing aliases failed %s\n"), ret)
	os.Exit(5)
}
<commit_msg>remove unreachable code<commit_after>package main

import (
	"fmt"
	"net"
	"net/url"
	"os"
	"os/exec"
	"path"
	"strings"
	"syscall"

	"github.com/lxc/lxd"
	"github.com/lxc/lxd/i18n"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/gnuflag"
	"github.com/lxc/lxd/shared/logging"
)

var configPath string

func main() {
	if err := run(); err != nil {
		// The action we take depends on the error we get.
		msg := fmt.Sprintf(i18n.G("error: %v"), err)
		switch t := err.(type) {
		case *url.Error:
			switch u := t.Err.(type) {
			case *net.OpError:
				if u.Op == "dial" && u.Net == "unix" {
					switch errno := u.Err.(type) {
					case syscall.Errno:
						switch errno {
						case syscall.ENOENT:
							msg = i18n.G("LXD socket not found; is LXD running?")
						case syscall.ECONNREFUSED:
							msg = i18n.G("Connection refused; is LXD running?")
						case syscall.EACCES:
							msg = i18n.G("Permisson denied, are you in the lxd group?")
						default:
							msg = fmt.Sprintf("%d %s", uintptr(errno), errno.Error())
						}
					}
				}
			}
		}

		fmt.Fprintln(os.Stderr, fmt.Sprintf("%s", msg))
		os.Exit(1)
	}
}

func run() error {
	verbose := gnuflag.Bool("verbose", false, i18n.G("Enables verbose mode."))
	debug := gnuflag.Bool("debug", false, i18n.G("Enables debug mode."))
	forceLocal := gnuflag.Bool("force-local", false, i18n.G("Force using the local unix socket."))
	noAlias := gnuflag.Bool("no-alias", false, i18n.G("Ignore aliases when determining what command to run."))

	configDir := "$HOME/.config/lxc"
	if os.Getenv("LXD_CONF") != "" {
		configDir = os.Getenv("LXD_CONF")
	}
	configPath = os.ExpandEnv(path.Join(configDir, "config.yml"))

	if len(os.Args) >= 3 && os.Args[1] == "config" && os.Args[2] == "profile" {
		fmt.Fprintf(os.Stderr, i18n.G("`lxc config profile` is deprecated, please use `lxc profile`")+"\n")
		os.Args = append(os.Args[:1], os.Args[2:]...)
	}

	if len(os.Args) >= 2 && (os.Args[1] == "-h" || os.Args[1] == "--help") {
		os.Args[1] = "help"
	}

	if len(os.Args) >= 2 && (os.Args[1] == "--all") {
		os.Args[1] = "help"
		os.Args = append(os.Args, "--all")
	}

	if len(os.Args) == 2 && os.Args[1] == "--version" {
		os.Args[1] = "version"
	}

	if len(os.Args) < 2 {
		commands["help"].run(nil, nil)
		os.Exit(1)
	}

	var config *lxd.Config
	var err error

	if *forceLocal {
		config = &lxd.DefaultConfig
	} else {
		config, err = lxd.LoadConfig(configPath)
		if err != nil {
			return err
		}

		// One time migration from old config
		if config.DefaultRemote == "" {
			_, ok := config.Remotes["local"]
			if !ok {
				config.Remotes["local"] = lxd.LocalRemote
			}
			config.DefaultRemote = "local"
			lxd.SaveConfig(config, configPath)
		}
	}

	// This is quite impolite, but it seems gnuflag needs us to shift our
	// own exename out of the arguments before parsing them. However, this
	// is useful for execIfAlias, which wants to know exactly the command
	// line we received, and in some cases is called before this shift, and
	// in others after. So, let's save the original args.
	origArgs := os.Args
	name := os.Args[1]

	/* at this point we haven't parsed the args, so we have to look for
	 * --no-alias by hand.
	 */
	if !shared.StringInSlice("--no-alias", origArgs) {
		execIfAliases(config, origArgs)
	}
	cmd, ok := commands[name]
	if !ok {
		commands["help"].run(nil, nil)
		fmt.Fprintf(os.Stderr, "\n"+i18n.G("error: unknown command: %s")+"\n", name)
		os.Exit(1)
	}
	cmd.flags()
	gnuflag.Usage = func() {
		fmt.Fprintf(os.Stderr, i18n.G("Usage: %s")+"\n\n"+i18n.G("Options:")+"\n\n", strings.TrimSpace(cmd.usage()))
		gnuflag.PrintDefaults()
	}

	os.Args = os.Args[1:]
	gnuflag.Parse(true)

	shared.Log, err = logging.GetLogger("", "", *verbose, *debug, nil)
	if err != nil {
		return err
	}

	certf := config.ConfigPath("client.crt")
	keyf := config.ConfigPath("client.key")

	if !*forceLocal && os.Args[0] != "help" && os.Args[0] != "version" && (!shared.PathExists(certf) || !shared.PathExists(keyf)) {
		fmt.Fprintf(os.Stderr, i18n.G("Generating a client certificate. This may take a minute...")+"\n")

		err = shared.FindOrGenCert(certf, keyf)
		if err != nil {
			return err
		}

		fmt.Fprintf(os.Stderr, i18n.G("If this is your first run, you will need to import images using the 'lxd-images' script.")+"\n")
		fmt.Fprintf(os.Stderr, i18n.G("For example: 'lxd-images import ubuntu --alias ubuntu'.")+"\n")
	}

	err = cmd.run(config, gnuflag.Args())
	if err == errArgs {
		/* If we got an error about invalid arguments, let's try to
		 * expand this as an alias
		 */
		if !*noAlias {
			execIfAliases(config, origArgs)
		}
		fmt.Fprintf(os.Stderr, "%s\n\n"+i18n.G("error: %v")+"\n", cmd.usage(), err)
		os.Exit(1)
	}
	return err
}

type command interface {
	usage() string
	flags()
	showByDefault() bool
	run(config *lxd.Config, args []string) error
}

var commands = map[string]command{
	"config":   &configCmd{},
	"copy":     &copyCmd{},
	"delete":   &deleteCmd{},
	"exec":     &execCmd{},
	"file":     &fileCmd{},
	"finger":   &fingerCmd{},
	"help":     &helpCmd{},
	"image":    &imageCmd{},
	"info":     &infoCmd{},
	"init":     &initCmd{},
	"launch":   &launchCmd{},
	"list":     &listCmd{},
	"monitor":  &monitorCmd{},
	"move":     &moveCmd{},
	"pause":    &actionCmd{shared.Freeze, false, false, "pause"},
	"profile":  &profileCmd{},
	"publish":  &publishCmd{},
	"remote":   &remoteCmd{},
	"restart":  &actionCmd{shared.Restart, true, true, "restart"},
	"restore":  &restoreCmd{},
	"snapshot": &snapshotCmd{},
	"start":    &actionCmd{shared.Start, false, true, "start"},
	"stop":     &actionCmd{shared.Stop, true, true, "stop"},
	"version":  &versionCmd{},
}

var errArgs = fmt.Errorf(i18n.G("wrong number of subcommand arguments"))

func expandAlias(config *lxd.Config, origArgs []string) ([]string, bool) {
	foundAlias := false
	aliasKey := []string{}
	aliasValue := []string{}

	for k, v := range config.Aliases {
		matches := false
		for i, key := range strings.Split(k, " ") {
			if len(origArgs) <= i+1 {
				break
			}

			if origArgs[i+1] == key {
				matches = true
				aliasKey = strings.Split(k, " ")
				aliasValue = strings.Split(v, " ")
				break
			}
		}

		if !matches {
			continue
		}

		foundAlias = true
		break
	}

	if !foundAlias {
		return []string{}, false
	}

	newArgs := []string{origArgs[0]}
	hasReplacedArgsVar := false

	for i, aliasArg := range aliasValue {
		if aliasArg == "@ARGS@" && len(origArgs) > i {
			newArgs = append(newArgs, origArgs[i+1:]...)
			hasReplacedArgsVar = true
		} else {
			newArgs = append(newArgs, aliasArg)
		}
	}

	if !hasReplacedArgsVar {
		/* add the rest of the arguments */
		newArgs = append(newArgs, origArgs[len(aliasKey)+1:]...)
	}

	/* don't re-do aliases the next time; this allows us to have recursive
	 * aliases, e.g. `lxc list` to `lxc list -c n`
	 */
	newArgs = append(newArgs[:2], append([]string{"--no-alias"}, newArgs[2:]...)...)

	return newArgs, true
}

func execIfAliases(config *lxd.Config, origArgs []string) {
	newArgs, expanded := expandAlias(config, origArgs)
	if !expanded {
		return
	}

	path, err := exec.LookPath(origArgs[0])
	if err != nil {
		fmt.Fprintf(os.Stderr, i18n.G("processing aliases failed %s\n"), err)
		os.Exit(5)
	}
	ret := syscall.Exec(path, newArgs, syscall.Environ())
	fmt.Fprintf(os.Stderr, i18n.G("processing aliases failed %s\n"), ret)
	os.Exit(5)
}
<|endoftext|>


<commit_before>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package platformvm

import (
	"fmt"

	"github.com/ava-labs/gecko/vms/components/verify"

	"github.com/ava-labs/gecko/database"
	"github.com/ava-labs/gecko/database/versiondb"
	"github.com/ava-labs/gecko/ids"
	"github.com/ava-labs/gecko/utils/hashing"
)

// UnsignedAddDefaultSubnetDelegatorTx is an unsigned addDefaultSubnetDelegatorTx
type UnsignedAddDefaultSubnetDelegatorTx struct {
	vm *VM

	// Metadata, inputs and outputs
	CommonTx `serialize:"true"`

	// Describes the delegatee
	DurationValidator `serialize:"true"`

	// Where to send staked AVA after done validating
	Destination ids.ShortID `serialize:"true"`
}

// addDefaultSubnetDelegatorTx is a transaction that, if it is in a
// ProposalBlock that is accepted and followed by a Commit block, adds a
// delegator to the pending validator set of the default subnet. (That is, the
// validator in the tx will have their weight increase at some point in the
// future.) The transaction fee will be paid from the account who signed the
// transaction.
type addDefaultSubnetDelegatorTx struct {
	UnsignedAddDefaultSubnetDelegatorTx `serialize:"true"`

	// Credentials that authorize the inputs to be spent
	Credentials []verify.Verifiable `serialize:"true"`
}

// Creds returns this transactions credentials
func (tx *addDefaultSubnetDelegatorTx) Creds() []verify.Verifiable {
	return tx.Credentials
}

// initialize [tx]
func (tx *addDefaultSubnetDelegatorTx) initialize(vm *VM) error {
	tx.vm = vm
	var err error
	tx.unsignedBytes, err = Codec.Marshal(interface{}(tx.UnsignedAddDefaultSubnetDelegatorTx))
	if err != nil {
		return fmt.Errorf("couldn't marshal UnsignedAddDefaultSubnetDelegatorTx: %w", err)
	}
	tx.bytes, err = Codec.Marshal(tx) // byte representation of the signed transaction
	if err != nil {
		return fmt.Errorf("couldn't marshal addDefaultSubnetDelegatorTx: %w", err)
	}
	tx.id = ids.NewID(hashing.ComputeHash256Array(tx.bytes))
	return nil
}

// SyntacticVerify return nil iff [tx] is valid
// If [tx] is valid, sets [tx.accountID]
// TODO: Only do syntactic Verify once
func (tx *addDefaultSubnetDelegatorTx) SyntacticVerify() error {
	switch {
	case tx == nil:
		return tempError{errNilTx}
	case tx.id.IsZero():
		return tempError{errInvalidID}
	case tx.NetworkID != tx.vm.Ctx.NetworkID:
		return permError{errWrongNetworkID}
	case tx.NodeID.IsZero():
		return tempError{errInvalidID}
	case tx.Wght < MinimumStakeAmount: // Ensure validator is staking at least the minimum amount
		return permError{errWeightTooSmall}
	}

	// Ensure staking length is not too short or long,
	// and that the inputs/outputs of this tx are syntactically valid
	stakingDuration := tx.Duration()
	if stakingDuration < MinimumStakingDuration {
		return permError{errStakeTooShort}
	} else if stakingDuration > MaximumStakingDuration {
		return permError{errStakeTooLong}
	} else if err := syntacticVerifySpend(tx, tx.vm.txFee, tx.vm.avaxAssetID); err != nil {
		return err
	}

	return nil
}

// SemanticVerify this transaction is valid.
// TODO make sure the ins and outs are semantically valid
func (tx *addDefaultSubnetDelegatorTx) SemanticVerify(db database.Database) (*versiondb.Database, *versiondb.Database, func(), func(), TxError) {
	if err := tx.SyntacticVerify(); err != nil {
		return nil, nil, nil, nil, permError{err}
	}

	// Verify inputs/outputs and update the UTXO set
	if err := tx.vm.semanticVerifySpend(db, tx); err != nil {
		return nil, nil, nil, nil, tempError{fmt.Errorf("couldn't verify tx: %w", err)}
	}

	// Ensure the proposed validator starts after the current timestamp
	currentTimestamp, err := tx.vm.getTimestamp(db)
	if err != nil {
		return nil, nil, nil, nil, permError{err}
	}
	validatorStartTime := tx.StartTime()
	if !currentTimestamp.Before(validatorStartTime) {
		return nil, nil, nil, nil, permError{fmt.Errorf("chain timestamp (%s) not before validator's start time (%s)",
			currentTimestamp,
			validatorStartTime)}
	}

	// Ensure that the period this validator validates the specified subnet is a subnet of the time they validate the default subnet
	// First, see if they're currently validating the default subnet
	currentEvents, err := tx.vm.getCurrentValidators(db, DefaultSubnetID)
	if err != nil {
		return nil, nil, nil, nil, permError{fmt.Errorf("couldn't get current validators of default subnet: %v", err)}
	}
	if dsValidator, err := currentEvents.getDefaultSubnetStaker(tx.NodeID); err == nil {
		if !tx.DurationValidator.BoundedBy(dsValidator.StartTime(), dsValidator.EndTime()) {
			return nil, nil, nil, nil, permError{errDSValidatorSubset}
		}
	} else {
		// They aren't currently validating the default subnet.
		// See if they will validate the default subnet in the future.
		pendingDSValidators, err := tx.vm.getPendingValidators(db, DefaultSubnetID)
		if err != nil {
			return nil, nil, nil, nil, permError{fmt.Errorf("couldn't get pending validators of default subnet: %v", err)}
		}
		dsValidator, err := pendingDSValidators.getDefaultSubnetStaker(tx.NodeID)
		if err != nil {
			return nil, nil, nil, nil, permError{errDSValidatorSubset}
		}
		if !tx.DurationValidator.BoundedBy(dsValidator.StartTime(), dsValidator.EndTime()) {
			return nil, nil, nil, nil, permError{errDSValidatorSubset}
		}
	}

	pendingEvents, err := tx.vm.getPendingValidators(db, DefaultSubnetID)
	if err != nil {
		return nil, nil, nil, nil, permError{err}
	}

	pendingEvents.Add(tx) // add validator to set of pending validators

	// If this proposal is committed, update the pending validator set to include the validator,
	// update the validator's account by removing the staked $AVA
	onCommitDB := versiondb.New(db)
	if err := tx.vm.putPendingValidators(onCommitDB, pendingEvents, DefaultSubnetID); err != nil {
		return nil, nil, nil, nil, permError{err}
	}
	/* TODO: Add this (or something like it) back
	if err := tx.vm.putAccount(onCommitDB, newAccount); err != nil {
		return nil, nil, nil, nil, permError{err}
	}
	*/

	// If this proposal is aborted, chain state doesn't change
	onAbortDB := versiondb.New(db)

	return onCommitDB, onAbortDB, nil, nil, nil
}

// InitiallyPrefersCommit returns true if the proposed validators start time is
// after the current wall clock time,
func (tx *addDefaultSubnetDelegatorTx) InitiallyPrefersCommit() bool {
	return tx.StartTime().After(tx.vm.clock.Time())
}

// TODO: Implement
/*
func (vm *VM) newAddDefaultSubnetDelegatorTx(
	nonce,
	weight,
	startTime,
	endTime uint64,
	nodeID ids.ShortID,
	destination ids.ShortID,
	networkID uint32,
	key *crypto.PrivateKeySECP256K1R,
) (*addDefaultSubnetDelegatorTx, error) {
	// Get UTXOs of sender
	addr := key.PublicKey().Address()

	tx := &addDefaultSubnetDelegatorTx{
		UnsignedAddDefaultSubnetDelegatorTx: UnsignedAddDefaultSubnetDelegatorTx{
			DurationValidator: DurationValidator{
				Validator: Validator{
					NodeID: nodeID,
					Wght:   weight,
				},
				Start: startTime,
				End:   endTime,
			},
			NetworkID:   networkID,
			Destination: destination,
		},
	}

	unsignedIntf := interface{}(&tx.UnsignedAddDefaultSubnetDelegatorTx)
	unsignedBytes, err := Codec.Marshal(&unsignedIntf) // byte repr. of unsigned tx
	if err != nil {
		return nil, err
	}

	sig, err := key.Sign(unsignedBytes)
	if err != nil {
		return nil, err
	}
	copy(tx.Sig[:], sig)

	return tx, tx.initialize(vm)
}
*/
<commit_msg>update addDefaultSUbnetDelegator<commit_after>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package platformvm

import (
	"fmt"

	"github.com/ava-labs/gecko/vms/components/verify"

	"github.com/ava-labs/gecko/database"
	"github.com/ava-labs/gecko/database/versiondb"
	"github.com/ava-labs/gecko/ids"
	"github.com/ava-labs/gecko/utils/hashing"
)

// UnsignedAddDefaultSubnetDelegatorTx is an unsigned addDefaultSubnetDelegatorTx
type UnsignedAddDefaultSubnetDelegatorTx struct {
	vm *VM

	// Metadata, inputs and outputs
	CommonTx `serialize:"true"`

	// Describes the delegatee
	DurationValidator `serialize:"true"`

	// Where to send staked AVA after done validating
	Destination ids.ShortID `serialize:"true"`
}

// addDefaultSubnetDelegatorTx is a transaction that, if it is in a
// ProposalBlock that is accepted and followed by a Commit block, adds a
// delegator to the pending validator set of the default subnet. (That is, the
// validator in the tx will have their weight increase at some point in the
// future.) The transaction fee will be paid from the account who signed the
// transaction.
type addDefaultSubnetDelegatorTx struct {
	UnsignedAddDefaultSubnetDelegatorTx `serialize:"true"`

	// Credentials that authorize the inputs to be spent
	Credentials []verify.Verifiable `serialize:"true"`
}

// Creds returns this transactions credentials
func (tx *addDefaultSubnetDelegatorTx) Creds() []verify.Verifiable {
	return tx.Credentials
}

// initialize [tx]
func (tx *addDefaultSubnetDelegatorTx) initialize(vm *VM) error {
	tx.vm = vm
	var err error
	tx.unsignedBytes, err = Codec.Marshal(interface{}(tx.UnsignedAddDefaultSubnetDelegatorTx))
	if err != nil {
		return fmt.Errorf("couldn't marshal UnsignedAddDefaultSubnetDelegatorTx: %w", err)
	}
	tx.bytes, err = Codec.Marshal(tx) // byte representation of the signed transaction
	if err != nil {
		return fmt.Errorf("couldn't marshal addDefaultSubnetDelegatorTx: %w", err)
	}
	tx.id = ids.NewID(hashing.ComputeHash256Array(tx.bytes))
	return nil
}

// SyntacticVerify return nil iff [tx] is valid
// If [tx] is valid, sets [tx.accountID]
// TODO: Only do syntactic Verify once
func (tx *addDefaultSubnetDelegatorTx) SyntacticVerify() error {
	switch {
	case tx == nil:
		return tempError{errNilTx}
	case tx.id.IsZero():
		return tempError{errInvalidID}
	case tx.NetworkID != tx.vm.Ctx.NetworkID:
		return permError{errWrongNetworkID}
	case tx.NodeID.IsZero():
		return tempError{errInvalidID}
	case tx.Wght < MinimumStakeAmount: // Ensure validator is staking at least the minimum amount
		return permError{errWeightTooSmall}
	}
	// Ensure staking length is not too short or long,
	// and that the inputs/outputs of this tx are syntactically valid
	stakingDuration := tx.Duration()
	if stakingDuration < MinimumStakingDuration {
		return permError{errStakeTooShort}
	} else if stakingDuration > MaximumStakingDuration {
		return permError{errStakeTooLong}
	} else if err := syntacticVerifySpend(tx, tx.vm.txFee, tx.vm.avaxAssetID); err != nil {
		return err
	}
	return nil
}

// SemanticVerify this transaction is valid.
// TODO make sure the ins and outs are semantically valid
func (tx *addDefaultSubnetDelegatorTx) SemanticVerify(db database.Database) (*versiondb.Database, *versiondb.Database, func(), func(), TxError) {
	if err := tx.SyntacticVerify(); err != nil {
		return nil, nil, nil, nil, permError{err}
	}

	// Verify inputs/outputs and update the UTXO set
	if err := tx.vm.semanticVerifySpend(db, tx); err != nil {
		return nil, nil, nil, nil, tempError{fmt.Errorf("couldn't verify tx: %w", err)}
	}

	// Ensure the proposed validator starts after the current timestamp
	if currentTimestamp, err := tx.vm.getTimestamp(db); err != nil {
		return nil, nil, nil, nil, permError{err}
	} else if validatorStartTime := tx.StartTime(); !currentTimestamp.Before(validatorStartTime) {
		return nil, nil, nil, nil, permError{fmt.Errorf("chain timestamp (%s) not before validator's start time (%s)",
			currentTimestamp,
			validatorStartTime)}
	}

	// Ensure that the period this validator validates the specified subnet is a subnet of the time they validate the default subnet
	// First, see if they're currently validating the default subnet
	currentValidatorHeap, err := tx.vm.getCurrentValidators(db, DefaultSubnetID)
	if err != nil {
		return nil, nil, nil, nil, permError{fmt.Errorf("couldn't get current validators of default subnet: %v", err)}
	}
	if dsValidator, err := currentValidatorHeap.getDefaultSubnetStaker(tx.NodeID); err == nil {
		if !tx.DurationValidator.BoundedBy(dsValidator.StartTime(), dsValidator.EndTime()) {
			return nil, nil, nil, nil, permError{errDSValidatorSubset}
		}
	} else {
		// They aren't currently validating the default subnet.
		// See if they will validate the default subnet in the future.
		pendingDSValidators, err := tx.vm.getPendingValidators(db, DefaultSubnetID)
		if err != nil {
			return nil, nil, nil, nil, permError{fmt.Errorf("couldn't get pending validators of default subnet: %v", err)}
		}
		dsValidator, err := pendingDSValidators.getDefaultSubnetStaker(tx.NodeID)
		if err != nil {
			return nil, nil, nil, nil, permError{errDSValidatorSubset}
		}
		if !tx.DurationValidator.BoundedBy(dsValidator.StartTime(), dsValidator.EndTime()) {
			return nil, nil, nil, nil, permError{errDSValidatorSubset}
		}
	}

	pendingValidatorHeap, err := tx.vm.getPendingValidators(db, DefaultSubnetID)
	if err != nil {
		return nil, nil, nil, nil, permError{err}
	}
	pendingValidatorHeap.Add(tx) // add validator to set of pending validators

	// If this proposal is committed, update the pending validator set to include the validator,
	// update the validator's account by removing the staked AVAX
	onCommitDB := versiondb.New(db)
	if err := tx.vm.putPendingValidators(onCommitDB, pendingValidatorHeap, DefaultSubnetID); err != nil {
		return nil, nil, nil, nil, permError{err}
	}

	// If this proposal is aborted, chain state doesn't change
	onAbortDB := versiondb.New(db)

	return onCommitDB, onAbortDB, nil, nil, nil
}

// InitiallyPrefersCommit returns true if the proposed validators start time is
// after the current wall clock time,
func (tx *addDefaultSubnetDelegatorTx) InitiallyPrefersCommit() bool {
	return tx.StartTime().After(tx.vm.clock.Time())
}

// TODO: Implement
/*
func (vm *VM) newAddDefaultSubnetDelegatorTx(
	nonce,
	weight,
	startTime,
	endTime uint64,
	nodeID ids.ShortID,
	destination ids.ShortID,
	networkID uint32,
	key *crypto.PrivateKeySECP256K1R,
) (*addDefaultSubnetDelegatorTx, error) {
	// Get UTXOs of sender
	addr := key.PublicKey().Address()

	tx := &addDefaultSubnetDelegatorTx{
		UnsignedAddDefaultSubnetDelegatorTx: UnsignedAddDefaultSubnetDelegatorTx{
			DurationValidator: DurationValidator{
				Validator: Validator{
					NodeID: nodeID,
					Wght:   weight,
				},
				Start: startTime,
				End:   endTime,
			},
			NetworkID:   networkID,
			Destination: destination,
		},
	}

	unsignedIntf := interface{}(&tx.UnsignedAddDefaultSubnetDelegatorTx)
	unsignedBytes, err := Codec.Marshal(&unsignedIntf) // byte repr. of unsigned tx
	if err != nil {
		return nil, err
	}

	sig, err := key.Sign(unsignedBytes)
	if err != nil {
		return nil, err
	}
	copy(tx.Sig[:], sig)

	return tx, tx.initialize(vm)
}
*/
<|endoftext|>


<commit_before>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cluster

import (
	"context"
	"fmt"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/coreos/etcd/clientv3"
	"github.com/coreos/etcd/embed"
	"github.com/coreos/etcd/pkg/transport"
	"github.com/gyuho/db/pkg/types"
)

// nodeState defines current Node state.
type nodeState uint8

const (
	stateNone nodeState = iota
	stateStarted
	stateStopped
)

// node contains *embed.Etcd and its state.
type node struct {
	srv        *embed.Etcd
	cfg        *embed.Config
	state      nodeState
	lastUpdate time.Time
}

// Cluster contains all embedded etcd nodes in the same cluster.
// Configuration is meant to be auto-generated.
type Cluster struct {
	mu             sync.Mutex
	rootDir        string
	size           int
	nodes          []*node
	updateInterval time.Duration
}

// Config defines etcd local cluster Configuration.
type Config struct {
	Size     int
	RootDir  string
	RootPort int

	ClientTLSInfo transport.TLSInfo
	ClientAutoTLS bool
	PeerTLSInfo   transport.TLSInfo
	PeerAutoTLS   bool

	// UpdateInterval is the minimum duration to allow updates on nodes.
	// This is to rate limit the nodes stop and restart operations.
	UpdateInterval time.Duration
}

var minUpdateInterval = time.Second

// Start starts embedded etcd cluster.
func Start(ccfg Config) (c *Cluster, err error) {
	plog.Printf("starting %d nodes (root directory %s, root port :%d)", ccfg.Size, ccfg.RootDir, ccfg.RootPort)

	if ccfg.UpdateInterval < minUpdateInterval {
		ccfg.UpdateInterval = minUpdateInterval
	}

	c = &Cluster{
		rootDir:        ccfg.RootDir,
		size:           ccfg.Size,
		nodes:          make([]*node, ccfg.Size),
		updateInterval: ccfg.UpdateInterval,
	}

	if !existFileOrDir(ccfg.RootDir) {
		if err = mkdirAll(ccfg.RootDir); err != nil {
			return nil, err
		}
	}

	// client TLS
	if !ccfg.ClientTLSInfo.Empty() && ccfg.ClientAutoTLS {
		return nil, fmt.Errorf("choose either auto TLS or manual client TLS")
	}
	clientScheme := "https"
	if ccfg.ClientTLSInfo.Empty() && !ccfg.ClientAutoTLS {
		clientScheme = "http"
	}

	// peer TLS
	if !ccfg.PeerTLSInfo.Empty() && ccfg.PeerAutoTLS {
		return nil, fmt.Errorf("choose either auto TLS or manual peer TLS")
	}
	peerScheme := "https"
	if ccfg.PeerTLSInfo.Empty() && !ccfg.PeerAutoTLS {
		peerScheme = "http"
	}

	startPort := ccfg.RootPort
	for i := 0; i < ccfg.Size; i++ {
		cfg := embed.NewConfig()

		cfg.Name = fmt.Sprintf("name%d", i)
		cfg.Dir = filepath.Join(ccfg.RootDir, cfg.Name+".etcd")
		cfg.WalDir = filepath.Join(cfg.Dir, "wal")

		clientURL := url.URL{Scheme: clientScheme, Host: fmt.Sprintf("localhost:%d", startPort)}
		cfg.LCUrls, cfg.ACUrls = []url.URL{clientURL}, []url.URL{clientURL}

		peerURL := url.URL{Scheme: peerScheme, Host: fmt.Sprintf("localhost:%d", startPort+1)}
		cfg.LPUrls, cfg.APUrls = []url.URL{peerURL}, []url.URL{peerURL}

		cfg.ClientAutoTLS = ccfg.ClientAutoTLS
		cfg.ClientTLSInfo = ccfg.ClientTLSInfo
		cfg.PeerAutoTLS = ccfg.PeerAutoTLS
		cfg.PeerTLSInfo = ccfg.PeerTLSInfo

		c.nodes[i] = &node{cfg: cfg}

		startPort += 2
	}

	inits := make([]string, ccfg.Size)
	for i := 0; i < ccfg.Size; i++ {
		inits[i] = c.nodes[i].cfg.Name + "=" + c.nodes[i].cfg.APUrls[0].String()
	}
	ic := strings.Join(inits, ",")

	for i := 0; i < ccfg.Size; i++ {
		c.nodes[i].cfg.InitialCluster = ic

		// start server
		var srv *embed.Etcd
		srv, err = embed.StartEtcd(c.nodes[i].cfg)
		if err != nil {
			return nil, err
		}
		c.nodes[i].srv = srv

		// copy and overwrite with internal configuration
		// in case it was configured with auto TLS
		nc := c.nodes[i].srv.Config()
		c.nodes[i].cfg = &nc
	}

	var wg sync.WaitGroup
	wg.Add(c.size)
	for i := range c.nodes {
		go func(i int) {
			defer wg.Done()

			<-c.nodes[i].srv.Server.ReadyNotify()

			c.nodes[i].state = stateStarted
			c.nodes[i].lastUpdate = time.Now()

			plog.Printf("started %s (client %s, peer %s)", c.nodes[i].cfg.Name, c.nodes[i].cfg.LCUrls[0].String(), c.nodes[i].cfg.LPUrls[0].String())
		}(i)
	}
	wg.Wait()

	time.Sleep(time.Second)

	plog.Print("checking leader")
	errc := make(chan error)
	for _, nd := range c.nodes {
		go func(n *node) {
			ep := n.cfg.LCUrls[0].Host
			ccfg := clientv3.Config{
				Endpoints:   []string{ep},
				DialTimeout: 3 * time.Second,
			}

			switch {
			case !n.cfg.ClientTLSInfo.Empty():
				tlsConfig, err := n.cfg.ClientTLSInfo.ClientConfig()
				if err != nil {
					errc <- err
					return
				}
				ccfg.TLS = tlsConfig

			case !n.cfg.ClientTLSInfo.Empty():
				tlsConfig, err := n.cfg.ClientTLSInfo.ClientConfig()
				if err != nil {
					errc <- err
					return
				}
				ccfg.TLS = tlsConfig
			}

			for {
				cli, err := clientv3.New(ccfg)
				if err != nil {
					plog.Warning(err)
					continue
				}
				defer cli.Close()

				ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
				resp, err := cli.Status(ctx, ep)
				cancel()
				if err != nil {
					plog.Warning(err)
					continue
				}

				if resp.Leader == uint64(0) {
					plog.Printf("%s %s has no leader yet", n.cfg.Name, types.ID(resp.Header.MemberId))
					time.Sleep(500 * time.Millisecond)
					continue
				}

				plog.Printf("%s %s has leader %s", n.cfg.Name, types.ID(resp.Header.MemberId), types.ID(resp.Leader))
				break
			}

			errc <- nil
		}(nd)
	}

	cn := 0
	for err := range errc {
		if err != nil {
			plog.Warning(err)
			return nil, err
		}

		cn++
		if cn == c.size {
			close(errc)
		}
	}

	plog.Printf("successfully started %d nodes", ccfg.Size)
	return c, nil
}

// Client creates the client.
func (c *Cluster) Client(i int, scheme, allEndpoints bool, dialTimeout time.Duration) (*clientv3.Client, error) {
	c.mu.Lock()
	defer c.mu.Unlock()

	eps := []string{c.nodes[i].cfg.LCUrls[0].Host}
	if allEndpoints {
		eps = c.allEndpoints(scheme)
	}
	ccfg := clientv3.Config{
		Endpoints:   eps,
		DialTimeout: dialTimeout,
	}

	switch {
	case !c.nodes[i].cfg.ClientTLSInfo.Empty():
		tlsConfig, err := c.nodes[i].cfg.ClientTLSInfo.ClientConfig()
		if err != nil {
			return nil, err
		}
		ccfg.TLS = tlsConfig

	case !c.nodes[i].cfg.ClientTLSInfo.Empty():
		tlsConfig, err := c.nodes[i].cfg.ClientTLSInfo.ClientConfig()
		if err != nil {
			return nil, err
		}
		ccfg.TLS = tlsConfig
	}

	return clientv3.New(ccfg)
}

// Stop stops a node.
func (c *Cluster) Stop(i int) {
	plog.Printf("stopping %s", c.nodes[i].cfg.Name)

	c.mu.Lock()
	defer c.mu.Unlock()

	if c.nodes[i].state == stateStopped {
		plog.Warningf("%s is already stopped", c.nodes[i].cfg.Name)
		return
	}

	for {
		it := time.Since(c.nodes[i].lastUpdate)
		if it > c.updateInterval {
			break
		}

		more := c.updateInterval - it + 100*time.Millisecond
		plog.Printf("rate-limiting stopping %s (sleeping %v)", c.nodes[i].cfg.Name, more)

		time.Sleep(more)
	}

	c.nodes[i].state = stateStopped
	c.nodes[i].lastUpdate = time.Now()

	c.nodes[i].srv.Close()
	<-c.nodes[i].srv.Err()

	plog.Printf("stopped %s", c.nodes[i].cfg.Name)
}

// Restart restarts a node.
func (c *Cluster) Restart(i int) error {
	plog.Printf("restarting %s", c.nodes[i].cfg.Name)

	c.mu.Lock()
	defer c.mu.Unlock()

	if c.nodes[i].state == stateStarted {
		plog.Warningf("%s is already started", c.nodes[i].cfg.Name)
		return nil
	}

	for {
		it := time.Since(c.nodes[i].lastUpdate)
		if it > c.updateInterval {
			break
		}

		more := c.updateInterval - it + 100*time.Millisecond
		plog.Printf("rate-limiting restarting %s (sleeping %v)", c.nodes[i].cfg.Name, more)

		time.Sleep(more)
	}

	c.nodes[i].cfg.ClusterState = "existing"

	// start server
	srv, err := embed.StartEtcd(c.nodes[i].cfg)
	if err != nil {
		return err
	}
	c.nodes[i].srv = srv

	nc := c.nodes[i].srv.Config()
	c.nodes[i].cfg = &nc

	<-c.nodes[i].srv.Server.ReadyNotify()

	c.nodes[i].state = stateStarted
	c.nodes[i].lastUpdate = time.Now()

	plog.Printf("restarted %s", c.nodes[i].cfg.Name)
	return nil
}

// Shutdown stops all nodes and deletes all data directories.
func (c *Cluster) Shutdown() {
	plog.Println("shutting down all nodes")

	c.mu.Lock()
	defer c.mu.Unlock()

	var wg sync.WaitGroup
	wg.Add(c.size)
	for i := range c.nodes {
		go func(i int) {
			defer wg.Done()

			if c.nodes[i].state == stateStopped {
				plog.Warningf("%s is already stopped", c.nodes[i].cfg.Name)
				return
			}

			c.nodes[i].state = stateStopped
			c.nodes[i].lastUpdate = time.Now()

			c.nodes[i].srv.Close()
			<-c.nodes[i].srv.Err()
		}(i)
	}
	wg.Wait()

	os.RemoveAll(c.rootDir)
	plog.Printf("deleted %s (done!)", c.rootDir)
}

// AllEndpoints returns all endpoints of clients.
func (c *Cluster) AllEndpoints(scheme bool) []string {
	c.mu.Lock()
	defer c.mu.Unlock()

	return c.allEndpoints(scheme)
}

func (c *Cluster) allEndpoints(scheme bool) []string {
	eps := make([]string, c.size)
	for i := 0; i < c.size; i++ {
		if scheme {
			eps[i] = c.nodes[i].cfg.LCUrls[0].String()
		} else {
			eps[i] = c.nodes[i].cfg.LCUrls[0].Host
		}
	}
	return eps
}
<commit_msg>cluster: fix locks<commit_after>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cluster

import (
	"context"
	"fmt"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/coreos/etcd/clientv3"
	"github.com/coreos/etcd/embed"
	"github.com/coreos/etcd/pkg/transport"
	"github.com/gyuho/db/pkg/types"
)

// nodeState defines current Node state.
type nodeState uint8

const (
	stateNone nodeState = iota
	stateStarted
	stateStopped
)

// node contains *embed.Etcd and its state.
type node struct {
	srv        *embed.Etcd
	cfg        *embed.Config
	state      nodeState
	lastUpdate time.Time
}

// Cluster contains all embedded etcd nodes in the same cluster.
// Configuration is meant to be auto-generated.
type Cluster struct {
	mu             sync.Mutex
	rootDir        string
	size           int
	nodes          []*node
	updateInterval time.Duration
}

// Config defines etcd local cluster Configuration.
type Config struct {
	Size     int
	RootDir  string
	RootPort int

	ClientTLSInfo transport.TLSInfo
	ClientAutoTLS bool
	PeerTLSInfo   transport.TLSInfo
	PeerAutoTLS   bool

	// UpdateInterval is the minimum duration to allow updates on nodes.
	// This is to rate limit the nodes stop and restart operations.
	UpdateInterval time.Duration
}

var minUpdateInterval = time.Second

// Start starts embedded etcd cluster.
func Start(ccfg Config) (c *Cluster, err error) {
	plog.Printf("starting %d nodes (root directory %s, root port :%d)", ccfg.Size, ccfg.RootDir, ccfg.RootPort)

	if ccfg.UpdateInterval < minUpdateInterval {
		ccfg.UpdateInterval = minUpdateInterval
	}

	c = &Cluster{
		rootDir:        ccfg.RootDir,
		size:           ccfg.Size,
		nodes:          make([]*node, ccfg.Size),
		updateInterval: ccfg.UpdateInterval,
	}

	if !existFileOrDir(ccfg.RootDir) {
		if err = mkdirAll(ccfg.RootDir); err != nil {
			return nil, err
		}
	}

	// client TLS
	if !ccfg.ClientTLSInfo.Empty() && ccfg.ClientAutoTLS {
		return nil, fmt.Errorf("choose either auto TLS or manual client TLS")
	}
	clientScheme := "https"
	if ccfg.ClientTLSInfo.Empty() && !ccfg.ClientAutoTLS {
		clientScheme = "http"
	}

	// peer TLS
	if !ccfg.PeerTLSInfo.Empty() && ccfg.PeerAutoTLS {
		return nil, fmt.Errorf("choose either auto TLS or manual peer TLS")
	}
	peerScheme := "https"
	if ccfg.PeerTLSInfo.Empty() && !ccfg.PeerAutoTLS {
		peerScheme = "http"
	}

	startPort := ccfg.RootPort
	for i := 0; i < ccfg.Size; i++ {
		cfg := embed.NewConfig()

		cfg.Name = fmt.Sprintf("name%d", i)
		cfg.Dir = filepath.Join(ccfg.RootDir, cfg.Name+".etcd")
		cfg.WalDir = filepath.Join(cfg.Dir, "wal")

		clientURL := url.URL{Scheme: clientScheme, Host: fmt.Sprintf("localhost:%d", startPort)}
		cfg.LCUrls, cfg.ACUrls = []url.URL{clientURL}, []url.URL{clientURL}

		peerURL := url.URL{Scheme: peerScheme, Host: fmt.Sprintf("localhost:%d", startPort+1)}
		cfg.LPUrls, cfg.APUrls = []url.URL{peerURL}, []url.URL{peerURL}

		cfg.ClientAutoTLS = ccfg.ClientAutoTLS
		cfg.ClientTLSInfo = ccfg.ClientTLSInfo
		cfg.PeerAutoTLS = ccfg.PeerAutoTLS
		cfg.PeerTLSInfo = ccfg.PeerTLSInfo

		c.nodes[i] = &node{cfg: cfg}

		startPort += 2
	}

	inits := make([]string, ccfg.Size)
	for i := 0; i < ccfg.Size; i++ {
		inits[i] = c.nodes[i].cfg.Name + "=" + c.nodes[i].cfg.APUrls[0].String()
	}
	ic := strings.Join(inits, ",")

	for i := 0; i < ccfg.Size; i++ {
		c.nodes[i].cfg.InitialCluster = ic

		// start server
		var srv *embed.Etcd
		srv, err = embed.StartEtcd(c.nodes[i].cfg)
		if err != nil {
			return nil, err
		}
		c.nodes[i].srv = srv

		// copy and overwrite with internal configuration
		// in case it was configured with auto TLS
		nc := c.nodes[i].srv.Config()
		c.nodes[i].cfg = &nc
	}

	var wg sync.WaitGroup
	wg.Add(c.size)
	for i := range c.nodes {
		go func(i int) {
			defer wg.Done()

			<-c.nodes[i].srv.Server.ReadyNotify()

			c.nodes[i].state = stateStarted
			c.nodes[i].lastUpdate = time.Now()

			plog.Printf("started %s (client %s, peer %s)", c.nodes[i].cfg.Name, c.nodes[i].cfg.LCUrls[0].String(), c.nodes[i].cfg.LPUrls[0].String())
		}(i)
	}
	wg.Wait()

	time.Sleep(time.Second)

	plog.Print("checking leader")
	errc := make(chan error)
	for i := range c.nodes {
		go func(i int) {
			for {
				cli, err := c.client(i, false, false, 3*time.Second)
				if err != nil {
					plog.Warning(err)
					continue
				}
				defer cli.Close()

				ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
				resp, err := cli.Status(ctx, c.nodes[i].cfg.LCUrls[0].Host)
				cancel()
				if err != nil {
					plog.Warning(err)
					continue
				}

				if resp.Leader == uint64(0) {
					plog.Printf("%s %s has no leader yet", c.nodes[i].cfg.Name, types.ID(resp.Header.MemberId))
					time.Sleep(time.Second)
					continue
				}

				plog.Printf("%s %s has leader %s", c.nodes[i].cfg.Name, types.ID(resp.Header.MemberId), types.ID(resp.Leader))
				break
			}

			errc <- nil
		}(i)
	}

	cn := 0
	for err := range errc {
		if err != nil {
			plog.Warning(err)
			return nil, err
		}

		cn++
		if cn == c.size {
			close(errc)
		}
	}

	plog.Printf("successfully started %d nodes", ccfg.Size)
	return c, nil
}

// Client creates the client.
func (c *Cluster) Client(i int, scheme, allEndpoints bool, dialTimeout time.Duration) (*clientv3.Client, error) {
	c.mu.Lock()
	defer c.mu.Unlock()
	return c.client(i, scheme, allEndpoints, dialTimeout)
}

func (c *Cluster) client(i int, scheme, allEndpoints bool, dialTimeout time.Duration) (*clientv3.Client, error) {
	eps := []string{c.nodes[i].cfg.LCUrls[0].Host}
	if allEndpoints {
		eps = c.allEndpoints(scheme)
	}
	ccfg := clientv3.Config{
		Endpoints:   eps,
		DialTimeout: dialTimeout,
	}

	switch {
	case !c.nodes[i].cfg.ClientTLSInfo.Empty():
		tlsConfig, err := c.nodes[i].cfg.ClientTLSInfo.ClientConfig()
		if err != nil {
			return nil, err
		}
		ccfg.TLS = tlsConfig

	case !c.nodes[i].cfg.ClientTLSInfo.Empty():
		tlsConfig, err := c.nodes[i].cfg.ClientTLSInfo.ClientConfig()
		if err != nil {
			return nil, err
		}
		ccfg.TLS = tlsConfig
	}

	return clientv3.New(ccfg)
}

// Stop stops a node.
func (c *Cluster) Stop(i int) {
	plog.Printf("stopping %s", c.nodes[i].cfg.Name)

	c.mu.Lock()
	defer c.mu.Unlock()

	if c.nodes[i].state == stateStopped {
		plog.Warningf("%s is already stopped", c.nodes[i].cfg.Name)
		return
	}

	for {
		it := time.Since(c.nodes[i].lastUpdate)
		if it > c.updateInterval {
			break
		}

		more := c.updateInterval - it + 100*time.Millisecond
		plog.Printf("rate-limiting stopping %s (sleeping %v)", c.nodes[i].cfg.Name, more)

		time.Sleep(more)
	}

	c.nodes[i].state = stateStopped
	c.nodes[i].lastUpdate = time.Now()

	c.nodes[i].srv.Close()
	<-c.nodes[i].srv.Err()

	plog.Printf("stopped %s", c.nodes[i].cfg.Name)
}

// Restart restarts a node.
func (c *Cluster) Restart(i int) error {
	plog.Printf("restarting %s", c.nodes[i].cfg.Name)

	c.mu.Lock()
	defer c.mu.Unlock()

	if c.nodes[i].state == stateStarted {
		plog.Warningf("%s is already started", c.nodes[i].cfg.Name)
		return nil
	}

	for {
		it := time.Since(c.nodes[i].lastUpdate)
		if it > c.updateInterval {
			break
		}

		more := c.updateInterval - it + 100*time.Millisecond
		plog.Printf("rate-limiting restarting %s (sleeping %v)", c.nodes[i].cfg.Name, more)

		time.Sleep(more)
	}

	c.nodes[i].cfg.ClusterState = "existing"

	// start server
	srv, err := embed.StartEtcd(c.nodes[i].cfg)
	if err != nil {
		return err
	}
	c.nodes[i].srv = srv

	nc := c.nodes[i].srv.Config()
	c.nodes[i].cfg = &nc

	<-c.nodes[i].srv.Server.ReadyNotify()

	c.nodes[i].state = stateStarted
	c.nodes[i].lastUpdate = time.Now()

	plog.Printf("restarted %s", c.nodes[i].cfg.Name)
	return nil
}

// Shutdown stops all nodes and deletes all data directories.
func (c *Cluster) Shutdown() {
	plog.Println("shutting down all nodes")

	c.mu.Lock()
	defer c.mu.Unlock()

	var wg sync.WaitGroup
	wg.Add(c.size)
	for i := range c.nodes {
		go func(i int) {
			defer wg.Done()

			if c.nodes[i].state == stateStopped {
				plog.Warningf("%s is already stopped", c.nodes[i].cfg.Name)
				return
			}

			c.nodes[i].state = stateStopped
			c.nodes[i].lastUpdate = time.Now()

			c.nodes[i].srv.Close()
			<-c.nodes[i].srv.Err()
		}(i)
	}
	wg.Wait()

	os.RemoveAll(c.rootDir)
	plog.Printf("deleted %s (done!)", c.rootDir)
}

// AllEndpoints returns all endpoints of clients.
func (c *Cluster) AllEndpoints(scheme bool) []string {
	c.mu.Lock()
	defer c.mu.Unlock()

	return c.allEndpoints(scheme)
}

func (c *Cluster) allEndpoints(scheme bool) []string {
	eps := make([]string, c.size)
	for i := 0; i < c.size; i++ {
		if scheme {
			eps[i] = c.nodes[i].cfg.LCUrls[0].String()
		} else {
			eps[i] = c.nodes[i].cfg.LCUrls[0].Host
		}
	}
	return eps
}
<|endoftext|>


<commit_before>package cluster

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"
	"text/tabwriter"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"github.com/flexiant/concerto/config"
	"github.com/flexiant/concerto/utils"
	"github.com/flexiant/concerto/webservice"
)

type Cluster struct {
	Id                string   `json:"id"`
	Name              string   `json:"name"`
	State             string   `json:"state"`
	MasterCount       int      `json:"master_count"`
	SlaveCount        int      `json:"slave_count"`
	WorkspaceId       string   `json:"workspace_id"`
	FirewallProfileId string   `json:"firewall_profile_id"`
	MasterTemplateId  string   `json:"master_template_id"`
	SlaveTemplateId   string   `json:"slave_template_id"`
	Masters           []string `json:"masters"`
}

func cmdCreate(c *cli.Context) {
	utils.FlagsRequired(c, []string{"cluster"})

	webservice, err := webservice.NewWebService()
	utils.CheckError(err)

	v := make(map[string]string)

	v["name"] = c.String("cluster")
	if c.IsSet("domain_id") {
		v["domain_id"] = c.String("domain_id")
	}

	json, err := json.Marshal(v)
	utils.CheckError(err)

	err, _, code := webservice.Post("/v1/kaas/fleets", json)
	utils.CheckError(err)
	utils.CheckReturnCode(code)

}

func cmdDelete(c *cli.Context) {
	utils.FlagsRequired(c, []string{"id"})

	webservice, err := webservice.NewWebService()
	utils.CheckError(err)

	err, _, res := webservice.Delete(fmt.Sprintf("/v1/kaas/fleets/%s", c.String("id")))
	utils.CheckError(err)
	utils.CheckReturnCode(res)

}

func cmdStart(c *cli.Context) {
	utils.FlagsRequired(c, []string{"id"})

	webservice, err := webservice.NewWebService()
	utils.CheckError(err)

	err, _, res := webservice.Put(fmt.Sprintf("/v1/kaas/fleets/%s/start", c.String("id")), nil)
	utils.CheckError(err)
	utils.CheckReturnCode(res)

}

func cmdStop(c *cli.Context) {
	utils.FlagsRequired(c, []string{"id"})

	webservice, err := webservice.NewWebService()
	utils.CheckError(err)

	err, _, res := webservice.Put(fmt.Sprintf("/v1/kaas/fleets/%s/stop", c.String("id")), nil)
	utils.CheckError(err)
	utils.CheckReturnCode(res)

}

func cmdEmpty(c *cli.Context) {
	utils.FlagsRequired(c, []string{"id"})

	webservice, err := webservice.NewWebService()
	utils.CheckError(err)

	err, _, res := webservice.Put(fmt.Sprintf("/v1/kaas/fleets/%s/empty", c.String("id")), nil)
	utils.CheckError(err)
	utils.CheckReturnCode(res)

}

func cmdAttachNet(c *cli.Context) {
	utils.FlagsRequired(c, []string{"id"})

	webservice, err := webservice.NewWebService()
	utils.CheckError(err)

	err, _, res := webservice.Put(fmt.Sprintf("/v1/kaas/fleets/%s/attach_network", c.String("id")), nil)
	utils.CheckError(err)
	utils.CheckReturnCode(res)

}

func cmdList(c *cli.Context) {
	var clusters []Cluster

	webservice, err := webservice.NewWebService()
	utils.CheckError(err)

	data, err := webservice.Get("/v1/kaas/fleets")
	utils.CheckError(err)

	err = json.Unmarshal(data, &clusters)
	utils.CheckError(err)

	w := tabwriter.NewWriter(os.Stdout, 15, 1, 3, ' ', 0)
	fmt.Fprintln(w, "CLUSTER\tID\tSTATE\tMASTER COUNT\tSLAVE COUNT")

	for _, cluster := range clusters {
		fmt.Fprintf(w, "%s\t%s\t%s\t%d\t%d\n", cluster.Name, cluster.Id, cluster.State, cluster.MasterCount, cluster.SlaveCount)
	}

	w.Flush()
}

func cmdKubectlHijack(c *cli.Context) {
	var clusters []Cluster
	var cluster Cluster

	discovered := false

	utils.FlagsRequired(c, []string{"cluster"})

	clusterName := c.String("cluster")

	var firstArgument string
	if c.Args().Present() {
		firstArgument = c.Args().First()
	} else {
		firstArgument = "help"
	}

	webservice, err := webservice.NewWebService()
	utils.CheckError(err)

	data, err := webservice.Get("/v1/kaas/fleets")
	utils.CheckError(err)

	err = json.Unmarshal(data, &clusters)
	utils.CheckError(err)

	// Validating if cluster exist
	for _, element := range clusters {
		if (element.Name == clusterName) || (element.Id == clusterName) {
			discovered = true
			cluster = element
		}
	}

	if discovered == true {
		//Discover where kubectl is located
		output, err := exec.Command("whereis", "kubectl").Output()
		utils.CheckError(err)

		kubeLocation := strings.TrimSpace(string(output))

		if !(len(kubeLocation) > 0) {
			log.Debug("Not found kubectl with whereis going to try which")
			//Discover where kubectl is located
			output, err = exec.Command("which", "kubectl").Output()
			utils.CheckError(err)

			kubeLocation = strings.TrimSpace(string(output))
		}

		if len(kubeLocation) > 0 {
			log.Debug(fmt.Sprintf("Found kubectl at %s", kubeLocation))
			config, err := config.ConcertoServerConfiguration()
			utils.CheckError(err)

			clusterParameters := fmt.Sprintf("--server=https://%s:6443", cluster.Masters[0])
			clientCertificate := fmt.Sprintf("--client-certificate=%s", config.Certificate.Cert)
			clientKey := fmt.Sprintf("--client-key=%s", config.Certificate.Key)
			clientCA := fmt.Sprintf("--certificate-authority=%s", config.Certificate.Ca)

			arguments := append([]string{clusterParameters, "--api-version=v1", clientCertificate, clientKey, clientCA, firstArgument}, c.Args().Tail()...)

			log.Debug(fmt.Sprintf("Going to execute %s %s", kubeLocation, arguments))

			cmd := exec.Command(kubeLocation, arguments...)

			stdout, err := cmd.StdoutPipe()
			utils.CheckError(err)

			stderr, err := cmd.StderrPipe()
			utils.CheckError(err)

			// Start command
			err = cmd.Start()
			utils.CheckError(err)
			defer cmd.Wait()

			go io.Copy(os.Stderr, stderr)

			ls := bufio.NewReader(stdout)

			for {
				line, isPrefix, err := ls.ReadLine()
				if isPrefix {
					log.Errorf("%s", errors.New("isPrefix: true"))
				}
				if err != nil {
					if err != io.EOF {
						log.Errorf("%s", err.Error())
					}
					break
				}
				fmt.Printf("%s\n", strings.Replace(string(line), "kubectl", fmt.Sprintf("concerto cluster kubectl --cluster %s", clusterName), -1))
			}

			go func() {
				time.Sleep(30 * time.Second)
				log.Fatal(fmt.Sprintf("Timeout out. Check conectivity to %s", clusterParameters))
			}()

			return
		} else {
			log.Warn(fmt.Sprintf("We could not find kubectl in your enviroment. Please install it. Thank you."))
			os.Exit(1)
		}
	} else {
		log.Warn(fmt.Sprintf("Cluster \"%s\" is not in your account please create it. Thank you.", clusterName))
		os.Exit(1)
	}

}

func SubCommands() []cli.Command {
	return []cli.Command{
		{
			Name:   "list",
			Usage:  "Lists all available Clusters",
			Action: cmdList,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "cluster",
					Usage: "Cluster Name to Attach Ship",
				},
				cli.StringFlag{
					Name:  "name",
					Usage: "Name of Host",
				},
				cli.StringFlag{
					Name:  "fqdn",
					Usage: "Full Qualify Domain Name of Host",
				},
				cli.StringFlag{
					Name:  "plan",
					Usage: "Server Plan to Use to Create Host",
				},
			},
		},
		{
			Name:   "start",
			Usage:  "Starts a given Cluster",
			Action: cmdStart,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "id",
					Usage: "Cluster Id",
				},
			},
		},
		{
			Name:   "stop",
			Usage:  "Stops a given Cluster",
			Action: cmdStop,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "id",
					Usage: "Cluster Id",
				},
			},
		},
		{
			Name:   "empty",
			Usage:  "Empties a given Cluster",
			Action: cmdEmpty,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "id",
					Usage: "Cluster Id",
				},
			},
		},
		{
			Name:   "attach_net",
			Usage:  "Attaches network to a given Cluster",
			Action: cmdAttachNet,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "id",
					Usage: "Cluster Id",
				},
			},
		},
		{
			Name:   "create",
			Usage:  "Creates a Cluster",
			Action: cmdCreate,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "cluster",
					Usage: "Cluster Name",
				},
				cli.StringFlag{
					Name:  "domain_id",
					Usage: "Domain Id",
				},
			},
		},
		{
			Name:   "delete",
			Usage:  "Deletes a given Cluster",
			Action: cmdDelete,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "id",
					Usage: "Cluster Id",
				},
			},
		},
		{
			Name:   "kubectl",
			Usage:  "Kubectl command line wrapper",
			Action: cmdKubectlHijack,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "cluster",
					Usage: "Cluster Name",
				},
			},
		},
	}
}
<commit_msg>[CARM-373] Change message level when kubectl is not found<commit_after>package cluster

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"
	"text/tabwriter"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"github.com/flexiant/concerto/config"
	"github.com/flexiant/concerto/utils"
	"github.com/flexiant/concerto/webservice"
)

type Cluster struct {
	Id                string   `json:"id"`
	Name              string   `json:"name"`
	State             string   `json:"state"`
	MasterCount       int      `json:"master_count"`
	SlaveCount        int      `json:"slave_count"`
	WorkspaceId       string   `json:"workspace_id"`
	FirewallProfileId string   `json:"firewall_profile_id"`
	MasterTemplateId  string   `json:"master_template_id"`
	SlaveTemplateId   string   `json:"slave_template_id"`
	Masters           []string `json:"masters"`
}

func cmdCreate(c *cli.Context) {
	utils.FlagsRequired(c, []string{"cluster"})

	webservice, err := webservice.NewWebService()
	utils.CheckError(err)

	v := make(map[string]string)

	v["name"] = c.String("cluster")
	if c.IsSet("domain_id") {
		v["domain_id"] = c.String("domain_id")
	}

	json, err := json.Marshal(v)
	utils.CheckError(err)

	err, _, code := webservice.Post("/v1/kaas/fleets", json)
	utils.CheckError(err)
	utils.CheckReturnCode(code)

}

func cmdDelete(c *cli.Context) {
	utils.FlagsRequired(c, []string{"id"})

	webservice, err := webservice.NewWebService()
	utils.CheckError(err)

	err, _, res := webservice.Delete(fmt.Sprintf("/v1/kaas/fleets/%s", c.String("id")))
	utils.CheckError(err)
	utils.CheckReturnCode(res)

}

func cmdStart(c *cli.Context) {
	utils.FlagsRequired(c, []string{"id"})

	webservice, err := webservice.NewWebService()
	utils.CheckError(err)

	err, _, res := webservice.Put(fmt.Sprintf("/v1/kaas/fleets/%s/start", c.String("id")), nil)
	utils.CheckError(err)
	utils.CheckReturnCode(res)

}

func cmdStop(c *cli.Context) {
	utils.FlagsRequired(c, []string{"id"})

	webservice, err := webservice.NewWebService()
	utils.CheckError(err)

	err, _, res := webservice.Put(fmt.Sprintf("/v1/kaas/fleets/%s/stop", c.String("id")), nil)
	utils.CheckError(err)
	utils.CheckReturnCode(res)

}

func cmdEmpty(c *cli.Context) {
	utils.FlagsRequired(c, []string{"id"})

	webservice, err := webservice.NewWebService()
	utils.CheckError(err)

	err, _, res := webservice.Put(fmt.Sprintf("/v1/kaas/fleets/%s/empty", c.String("id")), nil)
	utils.CheckError(err)
	utils.CheckReturnCode(res)

}

func cmdAttachNet(c *cli.Context) {
	utils.FlagsRequired(c, []string{"id"})

	webservice, err := webservice.NewWebService()
	utils.CheckError(err)

	err, _, res := webservice.Put(fmt.Sprintf("/v1/kaas/fleets/%s/attach_network", c.String("id")), nil)
	utils.CheckError(err)
	utils.CheckReturnCode(res)

}

func cmdList(c *cli.Context) {
	var clusters []Cluster

	webservice, err := webservice.NewWebService()
	utils.CheckError(err)

	data, err := webservice.Get("/v1/kaas/fleets")
	utils.CheckError(err)

	err = json.Unmarshal(data, &clusters)
	utils.CheckError(err)

	w := tabwriter.NewWriter(os.Stdout, 15, 1, 3, ' ', 0)
	fmt.Fprintln(w, "CLUSTER\tID\tSTATE\tMASTER COUNT\tSLAVE COUNT")

	for _, cluster := range clusters {
		fmt.Fprintf(w, "%s\t%s\t%s\t%d\t%d\n", cluster.Name, cluster.Id, cluster.State, cluster.MasterCount, cluster.SlaveCount)
	}

	w.Flush()
}

func cmdKubectlHijack(c *cli.Context) {
	var clusters []Cluster
	var cluster Cluster

	discovered := false

	utils.FlagsRequired(c, []string{"cluster"})

	clusterName := c.String("cluster")

	var firstArgument string
	if c.Args().Present() {
		firstArgument = c.Args().First()
	} else {
		firstArgument = "help"
	}

	webservice, err := webservice.NewWebService()
	utils.CheckError(err)

	data, err := webservice.Get("/v1/kaas/fleets")
	utils.CheckError(err)

	err = json.Unmarshal(data, &clusters)
	utils.CheckError(err)

	// Validating if cluster exist
	for _, element := range clusters {
		if (element.Name == clusterName) || (element.Id == clusterName) {
			discovered = true
			cluster = element
		}
	}

	if discovered == true {
		//Discover where kubectl is located
		output, err := exec.Command("whereis", "kubectl").Output()
		utils.CheckError(err)

		kubeLocation := strings.TrimSpace(string(output))

		if !(len(kubeLocation) > 0) {
			log.Info("Not found kubectl with whereis going to try which")
			//Discover where kubectl is located
			output, err = exec.Command("which", "kubectl").Output()
			utils.CheckError(err)

			kubeLocation = strings.TrimSpace(string(output))
		}

		if len(kubeLocation) > 0 {
			log.Debug(fmt.Sprintf("Found kubectl at %s", kubeLocation))
			config, err := config.ConcertoServerConfiguration()
			utils.CheckError(err)

			clusterParameters := fmt.Sprintf("--server=https://%s:6443", cluster.Masters[0])
			clientCertificate := fmt.Sprintf("--client-certificate=%s", config.Certificate.Cert)
			clientKey := fmt.Sprintf("--client-key=%s", config.Certificate.Key)
			clientCA := fmt.Sprintf("--certificate-authority=%s", config.Certificate.Ca)

			arguments := append([]string{clusterParameters, "--api-version=v1", clientCertificate, clientKey, clientCA, firstArgument}, c.Args().Tail()...)

			log.Debug(fmt.Sprintf("Going to execute %s %s", kubeLocation, arguments))

			cmd := exec.Command(kubeLocation, arguments...)

			stdout, err := cmd.StdoutPipe()
			utils.CheckError(err)

			stderr, err := cmd.StderrPipe()
			utils.CheckError(err)

			// Start command
			err = cmd.Start()
			utils.CheckError(err)
			defer cmd.Wait()

			go io.Copy(os.Stderr, stderr)

			ls := bufio.NewReader(stdout)

			for {
				line, isPrefix, err := ls.ReadLine()
				if isPrefix {
					log.Errorf("%s", errors.New("isPrefix: true"))
				}
				if err != nil {
					if err != io.EOF {
						log.Errorf("%s", err.Error())
					}
					break
				}
				fmt.Printf("%s\n", strings.Replace(string(line), "kubectl", fmt.Sprintf("concerto cluster kubectl --cluster %s", clusterName), -1))
			}

			go func() {
				time.Sleep(30 * time.Second)
				log.Fatal(fmt.Sprintf("Timeout out. Check conectivity to %s", clusterParameters))
			}()

			return
		} else {
			log.Warn(fmt.Sprintf("We could not find kubectl in your enviroment. Please install it. Thank you."))
			os.Exit(1)
		}
	} else {
		log.Warn(fmt.Sprintf("Cluster \"%s\" is not in your account please create it. Thank you.", clusterName))
		os.Exit(1)
	}

}

func SubCommands() []cli.Command {
	return []cli.Command{
		{
			Name:   "list",
			Usage:  "Lists all available Clusters",
			Action: cmdList,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "cluster",
					Usage: "Cluster Name to Attach Ship",
				},
				cli.StringFlag{
					Name:  "name",
					Usage: "Name of Host",
				},
				cli.StringFlag{
					Name:  "fqdn",
					Usage: "Full Qualify Domain Name of Host",
				},
				cli.StringFlag{
					Name:  "plan",
					Usage: "Server Plan to Use to Create Host",
				},
			},
		},
		{
			Name:   "start",
			Usage:  "Starts a given Cluster",
			Action: cmdStart,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "id",
					Usage: "Cluster Id",
				},
			},
		},
		{
			Name:   "stop",
			Usage:  "Stops a given Cluster",
			Action: cmdStop,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "id",
					Usage: "Cluster Id",
				},
			},
		},
		{
			Name:   "empty",
			Usage:  "Empties a given Cluster",
			Action: cmdEmpty,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "id",
					Usage: "Cluster Id",
				},
			},
		},
		{
			Name:   "attach_net",
			Usage:  "Attaches network to a given Cluster",
			Action: cmdAttachNet,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "id",
					Usage: "Cluster Id",
				},
			},
		},
		{
			Name:   "create",
			Usage:  "Creates a Cluster",
			Action: cmdCreate,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "cluster",
					Usage: "Cluster Name",
				},
				cli.StringFlag{
					Name:  "domain_id",
					Usage: "Domain Id",
				},
			},
		},
		{
			Name:   "delete",
			Usage:  "Deletes a given Cluster",
			Action: cmdDelete,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "id",
					Usage: "Cluster Id",
				},
			},
		},
		{
			Name:   "kubectl",
			Usage:  "Kubectl command line wrapper",
			Action: cmdKubectlHijack,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "cluster",
					Usage: "Cluster Name",
				},
			},
		},
	}
}
<|endoftext|>


<commit_before>package core

import (
//	"encoding/json"
)

// BagItProfile is a slightly modified version of bagit-profiles at
// https://github.com/ruebot/bagit-profiles.
// In this version, the Tag-Files-Required list is not a list of
// strings, but a list of TagFile objects, like the BagInfo object
// in bagit-profiles. This lets us validate the presence and value
// of specific tags in any tag file the same way bagit-profile lets
// us validate tags in the bagit.txt file.
type BagItProfile struct {
	// AcceptBagItVersion is a list of BagIt versions to accept.
	// For example, ["0.96", "0.97"]
	AcceptBagItVersion []string `json:"Accept-BagIt-Version"`
	// AcceptSerialization is a list of BagIt serialization formats
	// to accept. For example, ["application/zip", "application/tar"]
	AcceptSerialization []string `json:"Accept-BagIt-Serialization"`
	// AllowFetchTxt indicates whether we allow a fetch.txt file in the bag.
	AllowFetchTxt bool `json:"Allow-Fetch.txt"`
	// BagItProfileInfo contains descriptive information about this
	// BagIt profile.
	BagItProfileInfo BagItProfileInfo `json:"BagIt-Profile-Info"`
	// ManifestsRequired is a list of payload manifests that must be
	// in the bag. Values in this list are the algoritm names. So, to
	// require manifest-md5.txt and manifest-sha256.txt, the list
	// should contain just ["md5", "sha256"].
	ManifestsRequired []string `json:"Manifests-Required"`
	// Serialization can be "required", "optional" or "forbidden."
	Serialization string `json:"Serialization"`
	// TagFiles is a list of TagFile objects, each of which describes
	// a tag file in the bag. Here, we differ from the bagit-profiles
	// specification in that ALL tag files in the list are objects
	// instead of strings, and the objects describe tags we expect to
	// find in the files. Since TagFile objects have a Required property,
	// we omit bagit-profiles' TagFilesRequired, because that would be
	// redundant.
	TagFilesRequired []*TagFile `json:"Tag-Files-Required"`
	// TagManifestsRequired is a list of required tag manifests. Like
	// ManifestsRequired, the list contains only the names of the
	// required hashing algorithms. E.g. ["md5", "sha256"]
	TagManifestsRequired []string `json:"Tag-Manifests-Required"`
}

// BagItProfileInfo contains some basic info about the bagit profile
// and the sender, adhering to the structure of bagit-profiles at
// https://github.com/ruebot/bagit-profiles.
type BagItProfileInfo struct {
	// BagItProfileIdentifier is the URL where this bagit profile can be found.
	BagItProfileIdentifier string `json:"BagIt-Profile-Info"`
	// ContactEmail is the email address of the person maintaining this
	// bagit profile.
	ContactEmail string `json:"Contact-Email"`
	// ContactName is the name of the person maintaining this profile.
	ContactName string `json:"Contact-Name"`
	// ExternamDescription describes what this profile is for. For example,
	// "BagIt profile for ingesting content into APTrust."
	ExternalDescription string `json:"External-Description"`
	// SourceOrganization is the name of the organization maintaining this
	// profile.
	SourceOrganization string `json:"Source-Organization"`
	// Version is the version number of this profile. E.g "1.2".
	Version string `json:"Version"`
}

// LoadBagItProfile loads a BagItProfile from the specified file path.
func LoadBagItProfile(filePath string) (*BagItProfile, error) {
	return nil, nil
}

func (profile *BagItProfile) Validate() []error {
	return nil
}
<commit_msg>Added a profile options for misc files and dirs<commit_after>package core

import (
//	"encoding/json"
)

// BagItProfile is a slightly modified version of bagit-profiles at
// https://github.com/ruebot/bagit-profiles.
// In this version, the Tag-Files-Required list is not a list of
// strings, but a list of TagFile objects, like the BagInfo object
// in bagit-profiles. This lets us validate the presence and value
// of specific tags in any tag file the same way bagit-profile lets
// us validate tags in the bagit.txt file.
type BagItProfile struct {
	// AcceptBagItVersion is a list of BagIt versions to accept.
	// For example, ["0.96", "0.97"]
	AcceptBagItVersion []string `json:"Accept-BagIt-Version"`
	// AcceptSerialization is a list of BagIt serialization formats
	// to accept. For example, ["application/zip", "application/tar"]
	AcceptSerialization []string `json:"Accept-BagIt-Serialization"`
	// AllowFetchTxt indicates whether we allow a fetch.txt file in the bag.
	AllowFetchTxt bool `json:"Allow-Fetch.txt"`
	// AllowMiscTopLevelFiles indicates whether we allow files in the top-level
	// directory other than payload manifests and tag manifests.
	AllowMiscTopLevelFiles bool `json:"Allow-Misc-Top-Level-Files"`
	// AllowMiscDirectories indicates whether we allow miscellaneous
	// directories to exist outside the data directory. These non-data
	// directories often contain custom tag files whose checksums may
	// not appear in any manifest.
	AllowMiscDirectories bool `json:"Allow-Misc-Directories"`
	// BagItProfileInfo contains descriptive information about this
	// BagIt profile.
	BagItProfileInfo BagItProfileInfo `json:"BagIt-Profile-Info"`
	// ManifestsRequired is a list of payload manifests that must be
	// in the bag. Values in this list are the algoritm names. So, to
	// require manifest-md5.txt and manifest-sha256.txt, the list
	// should contain just ["md5", "sha256"].
	ManifestsRequired []string `json:"Manifests-Required"`
	// Serialization can be "required", "optional" or "forbidden."
	Serialization string `json:"Serialization"`
	// TagFiles is a list of TagFile objects, each of which describes
	// a tag file in the bag. Here, we differ from the bagit-profiles
	// specification in that ALL tag files in the list are objects
	// instead of strings, and the objects describe tags we expect to
	// find in the files. Since TagFile objects have a Required property,
	// we omit bagit-profiles' TagFilesRequired, because that would be
	// redundant.
	TagFilesRequired []*TagFile `json:"Tag-Files-Required"`
	// TagManifestsRequired is a list of required tag manifests. Like
	// ManifestsRequired, the list contains only the names of the
	// required hashing algorithms. E.g. ["md5", "sha256"]
	TagManifestsRequired []string `json:"Tag-Manifests-Required"`
}

// BagItProfileInfo contains some basic info about the bagit profile
// and the sender, adhering to the structure of bagit-profiles at
// https://github.com/ruebot/bagit-profiles.
type BagItProfileInfo struct {
	// BagItProfileIdentifier is the URL where this bagit profile can be found.
	BagItProfileIdentifier string `json:"BagIt-Profile-Info"`
	// ContactEmail is the email address of the person maintaining this
	// bagit profile.
	ContactEmail string `json:"Contact-Email"`
	// ContactName is the name of the person maintaining this profile.
	ContactName string `json:"Contact-Name"`
	// ExternamDescription describes what this profile is for. For example,
	// "BagIt profile for ingesting content into APTrust."
	ExternalDescription string `json:"External-Description"`
	// SourceOrganization is the name of the organization maintaining this
	// profile.
	SourceOrganization string `json:"Source-Organization"`
	// Version is the version number of this profile. E.g "1.2".
	Version string `json:"Version"`
}

// LoadBagItProfile loads a BagItProfile from the specified file path.
func LoadBagItProfile(filePath string) (*BagItProfile, error) {
	return nil, nil
}

func (profile *BagItProfile) Validate() []error {
	return nil
}
<|endoftext|>


<commit_before>package cmd

import (
	"testing"
	"time"

	"github.com/kr/pretty"
	"github.com/stretchr/testify/assert"
	"github.com/zrepl/zrepl/util"
	"github.com/zrepl/zrepl/zfs"
)

func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {

	paths := []string{
		"./sampleconf/localbackup/host1.yml",
		"./sampleconf/pullbackup/backuphost.yml",
		"./sampleconf/pullbackup/productionhost.yml",
		"./sampleconf/random/debugging.yml",
		"./sampleconf/random/logging_and_monitoring.yml",
	}

	for _, p := range paths {

		c, err := ParseConfig(p)
		if err != nil {
			t.Errorf("error parsing %s:\n%+v", p, err)
		}

		t.Logf("file: %s", p)
		t.Log(pretty.Sprint(c))

	}

}

func TestParseRetentionGridStringParsing(t *testing.T) {

	intervals, err := parseRetentionGridIntervalsString("2x10m(keep=2) | 1x1h | 3x1w")

	assert.Nil(t, err)
	assert.Len(t, intervals, 6)
	proto := util.RetentionInterval{
		KeepCount: 2,
		Length:    10 * time.Minute,
	}
	assert.EqualValues(t, proto, intervals[0])
	assert.EqualValues(t, proto, intervals[1])

	proto.KeepCount = 1
	proto.Length = 1 * time.Hour
	assert.EqualValues(t, proto, intervals[2])

	proto.Length = 7 * 24 * time.Hour
	assert.EqualValues(t, proto, intervals[3])
	assert.EqualValues(t, proto, intervals[4])
	assert.EqualValues(t, proto, intervals[5])

	intervals, err = parseRetentionGridIntervalsString("|")
	assert.Error(t, err)
	intervals, err = parseRetentionGridIntervalsString("2x10m")
	assert.NoError(t, err)

	intervals, err = parseRetentionGridIntervalsString("1x10m(keep=all)")
	assert.NoError(t, err)
	assert.Len(t, intervals, 1)
	assert.EqualValues(t, util.RetentionGridKeepCountAll, intervals[0].KeepCount)

}

func TestDatasetMapFilter(t *testing.T) {

	expectMapping := func(m map[string]string, from, to string) {
		dmf, err := parseDatasetMapFilter(m, false)
		if err != nil {
			t.Logf("expect test map to be valid: %s", err)
			t.FailNow()
		}
		fromPath, err := zfs.NewDatasetPath(from)
		if err != nil {
			t.Logf("expect test from path to be valid: %s", err)
			t.FailNow()
		}

		res, err := dmf.Map(fromPath)
		if to == "" {
			assert.Nil(t, res)
			assert.Nil(t, err)
			t.Logf("%s => NOT MAPPED", fromPath.ToString())
			return
		}

		assert.Nil(t, err)
		toPath, err := zfs.NewDatasetPath(to)
		if err != nil {
			t.Logf("expect test to path to be valid: %s", err)
			t.FailNow()
		}
		assert.True(t, res.Equal(toPath))
	}

	expectFilter := func(m map[string]string, path string, pass bool) {
		dmf, err := parseDatasetMapFilter(m, true)
		if err != nil {
			t.Logf("expect test filter to be valid: %s", err)
			t.FailNow()
		}
		p, err := zfs.NewDatasetPath(path)
		if err != nil {
			t.Logf("expect test path to be valid: %s", err)
			t.FailNow()
		}
		res, err := dmf.Filter(p)
		assert.Nil(t, err)
		assert.Equal(t, pass, res)
	}

	map1 := map[string]string{
		"a/b/c<":     "root1",
		"a/b<":       "root2",
		"<":          "root3/b/c",
		"b":          "!",
		"a/b/c/d/e<": "!",
		"q<":         "root4/1/2",
	}

	expectMapping(map1, "a/b/c", "root1")
	expectMapping(map1, "a/b/c/d", "root1/d")
	expectMapping(map1, "a/b/c/d/e", "")
	expectMapping(map1, "a/b/e", "root2/e")
	expectMapping(map1, "a/b", "root2")
	expectMapping(map1, "x", "root3/b/c/x")
	expectMapping(map1, "x/y", "root3/b/c/x/y")
	expectMapping(map1, "q", "root4/1/2")
	expectMapping(map1, "b", "")
	expectMapping(map1, "q/r", "root4/1/2/r")

	map2 := map[string]string{ // identity mapping
		"<":"",
	}
	expectMapping(map2, "foo/bar", "foo/bar")

	map3 := map[string]string{ // subtree to local mapping, need that for Invert()
		"foo/bar<": "",
	}
	{
		m, _ := parseDatasetMapFilter(map3, false)
		p, _ := zfs.NewDatasetPath("foo/bar")
		tp, err := m.Map(p)
		assert.Nil(t, err)
		assert.True(t, tp.Empty())

		expectMapping(map3, "foo/bar/x", "x")
		expectMapping(map3, "x", "")
	}

	filter1 := map[string]string{
		"<":    "!",
		"a<":   "ok",
		"a/b<": "!",
	}

	expectFilter(filter1, "b", false)
	expectFilter(filter1, "a", true)
	expectFilter(filter1, "a/d", true)
	expectFilter(filter1, "a/b", false)
	expectFilter(filter1, "a/b/c", false)

	filter2 := map[string]string{}
	expectFilter(filter2, "foo", false) // default to omit

}

func TestDatasetMapFilter_AsFilter(t *testing.T) {

	mapspec := map[string]string{
		"a/b/c<":     "root1",
		"a/b<":       "root2",
		"<":          "root3/b/c",
		"b":          "!",
		"a/b/c/d/e<": "!",
		"q<":         "root4/1/2",
	}

	m, err := parseDatasetMapFilter(mapspec, false)
	assert.Nil(t, err)

	f := m.AsFilter()

	t.Logf("Mapping:\n%s\nFilter:\n%s", pretty.Sprint(m), pretty.Sprint(f))

	tf := func(f zfs.DatasetFilter, path string, pass bool) {
		p, err := zfs.NewDatasetPath(path)
		assert.Nil(t, err)
		r, err := f.Filter(p)
		assert.Nil(t, err)
		assert.Equal(t, pass, r)
	}

	tf(f, "a/b/c", true)
	tf(f, "a/b", true)
	tf(f, "b", false)
	tf(f, "a/b/c/d/e", false)
	tf(f, "a/b/c/d/e/f", false)
	tf(f, "a", true)

}

func TestDatasetMapFilter_InvertedFilter(t *testing.T) {
	mapspec := map[string]string{
		"a/b":      "1/2",
		"a/b/c<":   "3",
		"a/b/c/d<": "1/2/a",
		"a/b/d":    "!",
	}

	m, err := parseDatasetMapFilter(mapspec, false)
	assert.Nil(t, err)

	inv, err := m.InvertedFilter()
	assert.Nil(t, err)

	t.Log(pretty.Sprint(inv))

	expectMapping := func(m *DatasetMapFilter, ps string, expRes bool) {
		p, err := zfs.NewDatasetPath(ps)
		assert.Nil(t, err)
		r, err := m.Filter(p)
		assert.Nil(t, err)
		assert.Equal(t, expRes, r)
	}

	expectMapping(inv, "4", false)
	expectMapping(inv, "3", true)
	expectMapping(inv, "3/x", true)
	expectMapping(inv, "1", false)
	expectMapping(inv, "1/2", true)
	expectMapping(inv, "1/2/3", false)
	expectMapping(inv, "1/2/a/b", true)

}

func TestDatasetMapFilter_Invert(t *testing.T) {

	mapspec := map[string]string{
		"<": "foo/bar",
	}

	m, err := parseDatasetMapFilter(mapspec, false)
	assert.NoError(t, err)

	inv, err := m.Invert()
	assert.NoError(t, err)

	expectMapping := func(m *DatasetMapFilter, input, expect string, expErr bool, expEmpty bool) {
		p, err := zfs.NewDatasetPath(input)
		assert.Nil(t, err)
		r, err := m.Map(p)
		if expErr {
			assert.Nil(t, r)
			assert.Error(t, err)
			return
		}
		if expEmpty {
			assert.Nil(t, err)
			assert.True(t, r.Empty())
		} else if expect == "" {
			assert.Nil(t, r)
			assert.Nil(t, err)
		} else {
			assert.Nil(t, err)
			assert.NotNil(t, r)
			assert.Equal(t, expect, r.ToString())
		}
	}

	expectMapping(inv, "x", "", false, false)
	expectMapping(inv, "foo/bar", "", false, true)
	expectMapping(inv, "foo/bar/bee", "bee", false, false)

}<commit_msg>fixup 70aad0940f37e611c20165a0d4c300e6e209d304: fix broken config_test.go<commit_after>package cmd

import (
	"testing"
	"time"

	"github.com/kr/pretty"
	"github.com/stretchr/testify/assert"
	"github.com/zrepl/zrepl/util"
	"github.com/zrepl/zrepl/zfs"
)

func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {

	paths := []string{
		"./sampleconf/localbackup/host1.yml",
		"./sampleconf/pullbackup/backuphost.yml",
		"./sampleconf/pullbackup/productionhost.yml",
		"./sampleconf/random/debugging.yml",
		"./sampleconf/random/logging_and_monitoring.yml",
	}

	for _, p := range paths {

		c, err := ParseConfig(p)
		if err != nil {
			t.Errorf("error parsing %s:\n%+v", p, err)
		}

		t.Logf("file: %s", p)
		t.Log(pretty.Sprint(c))

	}

}

func TestParseRetentionGridStringParsing(t *testing.T) {

	intervals, err := parseRetentionGridIntervalsString("2x10m(keep=2) | 1x1h | 3x1w")

	assert.Nil(t, err)
	assert.Len(t, intervals, 6)
	proto := util.RetentionInterval{
		KeepCount: 2,
		Length:    10 * time.Minute,
	}
	assert.EqualValues(t, proto, intervals[0])
	assert.EqualValues(t, proto, intervals[1])

	proto.KeepCount = 1
	proto.Length = 1 * time.Hour
	assert.EqualValues(t, proto, intervals[2])

	proto.Length = 7 * 24 * time.Hour
	assert.EqualValues(t, proto, intervals[3])
	assert.EqualValues(t, proto, intervals[4])
	assert.EqualValues(t, proto, intervals[5])

	intervals, err = parseRetentionGridIntervalsString("|")
	assert.Error(t, err)
	intervals, err = parseRetentionGridIntervalsString("2x10m")
	assert.NoError(t, err)

	intervals, err = parseRetentionGridIntervalsString("1x10m(keep=all)")
	assert.NoError(t, err)
	assert.Len(t, intervals, 1)
	assert.EqualValues(t, util.RetentionGridKeepCountAll, intervals[0].KeepCount)

}

func TestDatasetMapFilter(t *testing.T) {

	expectMapping := func(m map[string]string, from, to string) {
		dmf, err := parseDatasetMapFilter(m, false)
		if err != nil {
			t.Logf("expect test map to be valid: %s", err)
			t.FailNow()
		}
		fromPath, err := zfs.NewDatasetPath(from)
		if err != nil {
			t.Logf("expect test from path to be valid: %s", err)
			t.FailNow()
		}

		res, err := dmf.Map(fromPath)
		if to == "" {
			assert.Nil(t, res)
			assert.Nil(t, err)
			t.Logf("%s => NOT MAPPED", fromPath.ToString())
			return
		}

		assert.Nil(t, err)
		toPath, err := zfs.NewDatasetPath(to)
		if err != nil {
			t.Logf("expect test to path to be valid: %s", err)
			t.FailNow()
		}
		assert.True(t, res.Equal(toPath))
	}

	expectFilter := func(m map[string]string, path string, pass bool) {
		dmf, err := parseDatasetMapFilter(m, true)
		if err != nil {
			t.Logf("expect test filter to be valid: %s", err)
			t.FailNow()
		}
		p, err := zfs.NewDatasetPath(path)
		if err != nil {
			t.Logf("expect test path to be valid: %s", err)
			t.FailNow()
		}
		res, err := dmf.Filter(p)
		assert.Nil(t, err)
		assert.Equal(t, pass, res)
	}

	map1 := map[string]string{
		"a/b/c<":     "root1",
		"a/b<":       "root2",
		"<":          "root3/b/c",
		"b":          "!",
		"a/b/c/d/e<": "!",
		"q<":         "root4/1/2",
	}

	expectMapping(map1, "a/b/c", "root1")
	expectMapping(map1, "a/b/c/d", "root1/d")
	expectMapping(map1, "a/b/c/d/e", "")
	expectMapping(map1, "a/b/e", "root2/e")
	expectMapping(map1, "a/b", "root2")
	expectMapping(map1, "x", "root3/b/c/x")
	expectMapping(map1, "x/y", "root3/b/c/x/y")
	expectMapping(map1, "q", "root4/1/2")
	expectMapping(map1, "b", "")
	expectMapping(map1, "q/r", "root4/1/2/r")

	map2 := map[string]string{ // identity mapping
		"<":"",
	}
	expectMapping(map2, "foo/bar", "foo/bar")

	map3 := map[string]string{ // subtree to local mapping, need that for Invert()
		"foo/bar<": "",
	}
	{
		m, _ := parseDatasetMapFilter(map3, false)
		p, _ := zfs.NewDatasetPath("foo/bar")
		tp, err := m.Map(p)
		assert.Nil(t, err)
		assert.True(t, tp.Empty())

		expectMapping(map3, "foo/bar/x", "x")
		expectMapping(map3, "x", "")
	}

	filter1 := map[string]string{
		"<":    "!",
		"a<":   "ok",
		"a/b<": "!",
	}

	expectFilter(filter1, "b", false)
	expectFilter(filter1, "a", true)
	expectFilter(filter1, "a/d", true)
	expectFilter(filter1, "a/b", false)
	expectFilter(filter1, "a/b/c", false)

	filter2 := map[string]string{}
	expectFilter(filter2, "foo", false) // default to omit

}

func TestDatasetMapFilter_AsFilter(t *testing.T) {

	mapspec := map[string]string{
		"a/b/c<":     "root1",
		"a/b<":       "root2",
		"<":          "root3/b/c",
		"b":          "!",
		"a/b/c/d/e<": "!",
		"q<":         "root4/1/2",
	}

	m, err := parseDatasetMapFilter(mapspec, false)
	assert.Nil(t, err)

	f := m.AsFilter()

	t.Logf("Mapping:\n%s\nFilter:\n%s", pretty.Sprint(m), pretty.Sprint(f))

	tf := func(f zfs.DatasetFilter, path string, pass bool) {
		p, err := zfs.NewDatasetPath(path)
		assert.Nil(t, err)
		r, err := f.Filter(p)
		assert.Nil(t, err)
		assert.Equal(t, pass, r)
	}

	tf(f, "a/b/c", true)
	tf(f, "a/b", true)
	tf(f, "b", false)
	tf(f, "a/b/c/d/e", false)
	tf(f, "a/b/c/d/e/f", false)
	tf(f, "a", true)

}

func TestDatasetMapFilter_InvertedFilter(t *testing.T) {
	mapspec := map[string]string{
		"a/b":      "1/2",
		"a/b/c<":   "3",
		"a/b/c/d<": "1/2/a",
		"a/b/d":    "!",
	}

	m, err := parseDatasetMapFilter(mapspec, false)
	assert.Nil(t, err)

	inv, err := m.InvertedFilter()
	assert.Nil(t, err)

	t.Log(pretty.Sprint(inv))

	expectMapping := func(m *DatasetMapFilter, ps string, expRes bool) {
		p, err := zfs.NewDatasetPath(ps)
		assert.Nil(t, err)
		r, err := m.Filter(p)
		assert.Nil(t, err)
		assert.Equal(t, expRes, r)
	}

	expectMapping(inv, "4", false)
	expectMapping(inv, "3", true)
	expectMapping(inv, "3/x", true)
	expectMapping(inv, "1", false)
	expectMapping(inv, "1/2", true)
	expectMapping(inv, "1/2/3", false)
	expectMapping(inv, "1/2/a/b", true)

}

func TestDatasetMapFilter_Invert(t *testing.T) {

	mapspec := map[string]string{
		"<": "foo/bar",
	}

	m, err := parseDatasetMapFilter(mapspec, false)
	assert.NoError(t, err)

	invI, err := m.Invert()
	assert.NoError(t, err)
	inv, ok := invI.(*DatasetMapFilter)
	assert.True(t, ok)

	expectMapping := func(m *DatasetMapFilter, input, expect string, expErr bool, expEmpty bool) {
		p, err := zfs.NewDatasetPath(input)
		assert.Nil(t, err)
		r, err := m.Map(p)
		if expErr {
			assert.Nil(t, r)
			assert.Error(t, err)
			return
		}
		if expEmpty {
			assert.Nil(t, err)
			assert.True(t, r.Empty())
		} else if expect == "" {
			assert.Nil(t, r)
			assert.Nil(t, err)
		} else {
			assert.Nil(t, err)
			assert.NotNil(t, r)
			assert.Equal(t, expect, r.ToString())
		}
	}

	expectMapping(inv, "x", "", false, false)
	expectMapping(inv, "foo/bar", "", false, true)
	expectMapping(inv, "foo/bar/bee", "bee", false, false)

}<|endoftext|>


<commit_before>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package broker

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"sort"
	"time"

	pb "github.com/TheThingsNetwork/ttn/api/broker"
	pb_discovery "github.com/TheThingsNetwork/ttn/api/discovery"
	"github.com/TheThingsNetwork/ttn/api/gateway"
	"github.com/TheThingsNetwork/ttn/api/networkserver"
	pb_lorawan "github.com/TheThingsNetwork/ttn/api/protocol/lorawan"
	"github.com/TheThingsNetwork/ttn/core/fcnt"
	"github.com/TheThingsNetwork/ttn/core/types"
	"github.com/TheThingsNetwork/ttn/utils/errors"
	"github.com/apex/log"
	"github.com/brocaar/lorawan"
)

const maxFCntGap = 16384

func (b *broker) HandleUplink(uplink *pb.UplinkMessage) error {
	ctx := b.Ctx.WithField("GatewayID", uplink.GatewayMetadata.GatewayId)
	var err error
	start := time.Now()
	defer func() {
		if err != nil {
			ctx.WithError(err).Warn("Could not handle uplink")
		} else {
			ctx.WithField("Duration", time.Now().Sub(start)).Info("Handled uplink")
		}
	}()

	time := time.Now()

	// De-duplicate uplink messages
	duplicates := b.deduplicateUplink(uplink)
	if len(duplicates) == 0 {
		return nil
	}

	base := duplicates[0]

	if base.ProtocolMetadata.GetLorawan() == nil {
		err = errors.NewErrInvalidArgument("Uplink", "does not contain LoRaWAN metadata")
		return err
	}

	// LoRaWAN: Unmarshal
	var phyPayload lorawan.PHYPayload
	err = phyPayload.UnmarshalBinary(base.Payload)
	if err != nil {
		return err
	}
	macPayload, ok := phyPayload.MACPayload.(*lorawan.MACPayload)
	if !ok {
		err = errors.NewErrInvalidArgument("Uplink", "does not contain a MAC payload")
		return err
	}

	// Request devices from NS
	devAddr := types.DevAddr(macPayload.FHDR.DevAddr)
	ctx = ctx.WithField("DevAddr", devAddr)
	var getDevicesResp *networkserver.DevicesResponse
	getDevicesResp, err = b.ns.GetDevices(b.Component.GetContext(b.nsToken), &networkserver.DevicesRequest{
		DevAddr: &devAddr,
		FCnt:    macPayload.FHDR.FCnt,
	})
	if err != nil {
		return errors.BuildGRPCError(errors.Wrap(errors.FromGRPCError(err), "NetworkServer did not return devices"))
	}
	if len(getDevicesResp.Results) == 0 {
		err = errors.NewErrNotFound(fmt.Sprintf("Device with DevAddr %s and FCnt <= %d", devAddr, macPayload.FHDR.FCnt))
		return err
	}
	ctx = ctx.WithField("DevAddrResults", len(getDevicesResp.Results))

	// Sort by FCntUp to optimize the number of MIC checks
	sort.Sort(ByFCntUp(getDevicesResp.Results))

	// Find AppEUI/DevEUI through MIC check
	var device *pb_lorawan.Device
	var micChecks int
	for _, candidate := range getDevicesResp.Results {
		nwkSKey := lorawan.AES128Key(*candidate.NwkSKey)
		if candidate.Uses32BitFCnt {
			macPayload.FHDR.FCnt = fcnt.GetFull(candidate.FCntUp, uint16(macPayload.FHDR.FCnt))
		}
		micChecks++
		ok, err = phyPayload.ValidateMIC(nwkSKey)
		if err != nil {
			return err
		}
		if ok {
			device = candidate
			break
		}
	}
	if device == nil {
		err = errors.NewErrNotFound("device that validates MIC")
		return err
	}
	ctx = ctx.WithFields(log.Fields{
		"MICChecks": micChecks,
		"DevEUI":    device.DevEui,
		"AppEUI":    device.AppEui,
		"AppID":     device.AppId,
		"DevID":     device.DevId,
	})

	if device.DisableFCntCheck {
		// TODO: Add warning to message?
	} else if device.FCntUp == 0 {

	} else if macPayload.FHDR.FCnt <= device.FCntUp || macPayload.FHDR.FCnt-device.FCntUp > maxFCntGap {
		// Replay attack or FCnt gap too big
		err = errors.NewErrNotFound("device with matching FCnt")
		return err
	}

	// Add FCnt to Metadata (because it's not marshaled in lorawan payload)
	base.ProtocolMetadata.GetLorawan().FCnt = macPayload.FHDR.FCnt

	// Collect GatewayMetadata and DownlinkOptions
	var gatewayMetadata []*gateway.RxMetadata
	var downlinkOptions []*pb.DownlinkOption
	var downlinkMessage *pb.DownlinkMessage
	for _, duplicate := range duplicates {
		gatewayMetadata = append(gatewayMetadata, duplicate.GatewayMetadata)
		downlinkOptions = append(downlinkOptions, duplicate.DownlinkOptions...)
	}

	// Select best DownlinkOption
	if len(downlinkOptions) > 0 {
		downlinkMessage = &pb.DownlinkMessage{
			DownlinkOption: selectBestDownlink(downlinkOptions),
		}
	}

	// Build Uplink
	deduplicatedUplink := &pb.DeduplicatedUplinkMessage{
		Payload:          base.Payload,
		DevEui:           device.DevEui,
		DevId:            device.DevId,
		AppEui:           device.AppEui,
		AppId:            device.AppId,
		ProtocolMetadata: base.ProtocolMetadata,
		GatewayMetadata:  gatewayMetadata,
		ServerTime:       time.UnixNano(),
		ResponseTemplate: downlinkMessage,
	}

	// Pass Uplink through NS
	deduplicatedUplink, err = b.ns.Uplink(b.Component.GetContext(b.nsToken), deduplicatedUplink)
	if err != nil {
		return errors.BuildGRPCError(errors.Wrap(errors.FromGRPCError(err), "NetworkServer did not handle uplink"))
	}

	var announcements []*pb_discovery.Announcement
	announcements, err = b.Discovery.GetAllHandlersForAppID(device.AppId)
	if err != nil {
		return err
	}
	if len(announcements) == 0 {
		err = errors.NewErrNotFound(fmt.Sprintf("Handler for AppID %s", device.AppId))
		return err
	}
	if len(announcements) > 1 {
		err = errors.NewErrInternal(fmt.Sprintf("Multiple Handlers for AppID %s", device.AppId))
		return err
	}

	var handler chan<- *pb.DeduplicatedUplinkMessage
	handler, err = b.getHandler(announcements[0].Id)
	if err != nil {
		return err
	}

	handler <- deduplicatedUplink

	return nil
}

func (b *broker) deduplicateUplink(duplicate *pb.UplinkMessage) (uplinks []*pb.UplinkMessage) {
	sum := md5.Sum(duplicate.Payload)
	key := hex.EncodeToString(sum[:])
	list := b.uplinkDeduplicator.Deduplicate(key, duplicate)
	if len(list) == 0 {
		return
	}
	for _, duplicate := range list {
		uplinks = append(uplinks, duplicate.(*pb.UplinkMessage))
	}
	return
}

func selectBestDownlink(options []*pb.DownlinkOption) *pb.DownlinkOption {
	sort.Sort(ByScore(options))
	return options[0]
}

// ByFCntUp implements sort.Interface for []*pb_lorawan.Device based on FCnt
type ByFCntUp []*pb_lorawan.Device

func (a ByFCntUp) Len() int      { return len(a) }
func (a ByFCntUp) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a ByFCntUp) Less(i, j int) bool {
	// Devices that disable the FCnt check have low priority
	if a[i].DisableFCntCheck {
		return 2*int(a[i].FCntUp)+100 < int(a[j].FCntUp)
	}
	return int(a[i].FCntUp) < int(a[j].FCntUp)
}
<commit_msg>Add extra MIC check for devices with DisableFCntCheck<commit_after>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package broker

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"sort"
	"time"

	pb "github.com/TheThingsNetwork/ttn/api/broker"
	pb_discovery "github.com/TheThingsNetwork/ttn/api/discovery"
	"github.com/TheThingsNetwork/ttn/api/gateway"
	"github.com/TheThingsNetwork/ttn/api/networkserver"
	pb_lorawan "github.com/TheThingsNetwork/ttn/api/protocol/lorawan"
	"github.com/TheThingsNetwork/ttn/core/fcnt"
	"github.com/TheThingsNetwork/ttn/core/types"
	"github.com/TheThingsNetwork/ttn/utils/errors"
	"github.com/apex/log"
	"github.com/brocaar/lorawan"
)

const maxFCntGap = 16384

func (b *broker) HandleUplink(uplink *pb.UplinkMessage) error {
	ctx := b.Ctx.WithField("GatewayID", uplink.GatewayMetadata.GatewayId)
	var err error
	start := time.Now()
	defer func() {
		if err != nil {
			ctx.WithError(err).Warn("Could not handle uplink")
		} else {
			ctx.WithField("Duration", time.Now().Sub(start)).Info("Handled uplink")
		}
	}()

	time := time.Now()

	// De-duplicate uplink messages
	duplicates := b.deduplicateUplink(uplink)
	if len(duplicates) == 0 {
		return nil
	}

	base := duplicates[0]

	if base.ProtocolMetadata.GetLorawan() == nil {
		err = errors.NewErrInvalidArgument("Uplink", "does not contain LoRaWAN metadata")
		return err
	}

	// LoRaWAN: Unmarshal
	var phyPayload lorawan.PHYPayload
	err = phyPayload.UnmarshalBinary(base.Payload)
	if err != nil {
		return err
	}
	macPayload, ok := phyPayload.MACPayload.(*lorawan.MACPayload)
	if !ok {
		err = errors.NewErrInvalidArgument("Uplink", "does not contain a MAC payload")
		return err
	}

	// Request devices from NS
	devAddr := types.DevAddr(macPayload.FHDR.DevAddr)
	ctx = ctx.WithField("DevAddr", devAddr)
	var getDevicesResp *networkserver.DevicesResponse
	getDevicesResp, err = b.ns.GetDevices(b.Component.GetContext(b.nsToken), &networkserver.DevicesRequest{
		DevAddr: &devAddr,
		FCnt:    macPayload.FHDR.FCnt,
	})
	if err != nil {
		return errors.BuildGRPCError(errors.Wrap(errors.FromGRPCError(err), "NetworkServer did not return devices"))
	}
	if len(getDevicesResp.Results) == 0 {
		err = errors.NewErrNotFound(fmt.Sprintf("Device with DevAddr %s and FCnt <= %d", devAddr, macPayload.FHDR.FCnt))
		return err
	}
	ctx = ctx.WithField("DevAddrResults", len(getDevicesResp.Results))

	// Sort by FCntUp to optimize the number of MIC checks
	sort.Sort(ByFCntUp(getDevicesResp.Results))

	// Find AppEUI/DevEUI through MIC check
	var device *pb_lorawan.Device
	var micChecks int
	for _, candidate := range getDevicesResp.Results {
		nwkSKey := lorawan.AES128Key(*candidate.NwkSKey)
		if candidate.Uses32BitFCnt {
			if candidate.DisableFCntCheck {
				// We should first check with the 16 bit counter
				micChecks++
				ok, err = phyPayload.ValidateMIC(nwkSKey)
				if err != nil {
					return err
				}
				if ok {
					device = candidate
					break
				}
			}
			// Then set the full 32 bit counter and check again
			macPayload.FHDR.FCnt = fcnt.GetFull(candidate.FCntUp, uint16(macPayload.FHDR.FCnt))
		}
		micChecks++
		ok, err = phyPayload.ValidateMIC(nwkSKey)
		if err != nil {
			return err
		}
		if ok {
			device = candidate
			break
		}
	}
	if device == nil {
		err = errors.NewErrNotFound("device that validates MIC")
		return err
	}
	ctx = ctx.WithFields(log.Fields{
		"MICChecks": micChecks,
		"DevEUI":    device.DevEui,
		"AppEUI":    device.AppEui,
		"AppID":     device.AppId,
		"DevID":     device.DevId,
	})

	if device.DisableFCntCheck {
		// TODO: Add warning to message?
	} else if device.FCntUp == 0 {

	} else if macPayload.FHDR.FCnt <= device.FCntUp || macPayload.FHDR.FCnt-device.FCntUp > maxFCntGap {
		// Replay attack or FCnt gap too big
		err = errors.NewErrNotFound("device with matching FCnt")
		return err
	}

	// Add FCnt to Metadata (because it's not marshaled in lorawan payload)
	base.ProtocolMetadata.GetLorawan().FCnt = macPayload.FHDR.FCnt

	// Collect GatewayMetadata and DownlinkOptions
	var gatewayMetadata []*gateway.RxMetadata
	var downlinkOptions []*pb.DownlinkOption
	var downlinkMessage *pb.DownlinkMessage
	for _, duplicate := range duplicates {
		gatewayMetadata = append(gatewayMetadata, duplicate.GatewayMetadata)
		downlinkOptions = append(downlinkOptions, duplicate.DownlinkOptions...)
	}

	// Select best DownlinkOption
	if len(downlinkOptions) > 0 {
		downlinkMessage = &pb.DownlinkMessage{
			DownlinkOption: selectBestDownlink(downlinkOptions),
		}
	}

	// Build Uplink
	deduplicatedUplink := &pb.DeduplicatedUplinkMessage{
		Payload:          base.Payload,
		DevEui:           device.DevEui,
		DevId:            device.DevId,
		AppEui:           device.AppEui,
		AppId:            device.AppId,
		ProtocolMetadata: base.ProtocolMetadata,
		GatewayMetadata:  gatewayMetadata,
		ServerTime:       time.UnixNano(),
		ResponseTemplate: downlinkMessage,
	}

	// Pass Uplink through NS
	deduplicatedUplink, err = b.ns.Uplink(b.Component.GetContext(b.nsToken), deduplicatedUplink)
	if err != nil {
		return errors.BuildGRPCError(errors.Wrap(errors.FromGRPCError(err), "NetworkServer did not handle uplink"))
	}

	var announcements []*pb_discovery.Announcement
	announcements, err = b.Discovery.GetAllHandlersForAppID(device.AppId)
	if err != nil {
		return err
	}
	if len(announcements) == 0 {
		err = errors.NewErrNotFound(fmt.Sprintf("Handler for AppID %s", device.AppId))
		return err
	}
	if len(announcements) > 1 {
		err = errors.NewErrInternal(fmt.Sprintf("Multiple Handlers for AppID %s", device.AppId))
		return err
	}

	var handler chan<- *pb.DeduplicatedUplinkMessage
	handler, err = b.getHandler(announcements[0].Id)
	if err != nil {
		return err
	}

	handler <- deduplicatedUplink

	return nil
}

func (b *broker) deduplicateUplink(duplicate *pb.UplinkMessage) (uplinks []*pb.UplinkMessage) {
	sum := md5.Sum(duplicate.Payload)
	key := hex.EncodeToString(sum[:])
	list := b.uplinkDeduplicator.Deduplicate(key, duplicate)
	if len(list) == 0 {
		return
	}
	for _, duplicate := range list {
		uplinks = append(uplinks, duplicate.(*pb.UplinkMessage))
	}
	return
}

func selectBestDownlink(options []*pb.DownlinkOption) *pb.DownlinkOption {
	sort.Sort(ByScore(options))
	return options[0]
}

// ByFCntUp implements sort.Interface for []*pb_lorawan.Device based on FCnt
type ByFCntUp []*pb_lorawan.Device

func (a ByFCntUp) Len() int      { return len(a) }
func (a ByFCntUp) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a ByFCntUp) Less(i, j int) bool {
	// Devices that disable the FCnt check have low priority
	if a[i].DisableFCntCheck {
		return 2*int(a[i].FCntUp)+100 < int(a[j].FCntUp)
	}
	return int(a[i].FCntUp) < int(a[j].FCntUp)
}
<|endoftext|>