<commit_before>package cmds

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"sort"

	"github.com/pachyderm/pachyderm/src/client/pkg/config"
	"github.com/pachyderm/pachyderm/src/server/pkg/cmdutil"

	"github.com/gogo/protobuf/jsonpb"
	"github.com/spf13/cobra"
)

const (
	listContextHeader = "ACITVE\tNAME"
)

func readContext() (*config.Context, error) {
	var buf bytes.Buffer
	var decoder *json.Decoder
	var result config.Context

	contextReader := io.TeeReader(os.Stdin, &buf)
	fmt.Println("Reading from stdin.")
	decoder = json.NewDecoder(contextReader)

	if err := jsonpb.UnmarshalNext(decoder, &result); err != nil {
		if err == io.EOF {
			return nil, err
		}
		return nil, fmt.Errorf("malformed context: %s", err)
	}
	return &result, nil
}

// Cmds returns a slice containing admin commands.
func Cmds() []*cobra.Command {
	marshaller := &jsonpb.Marshaler{
		Indent:   "  ",
		OrigName: true,
	}

	var commands []*cobra.Command

	getMetrics := &cobra.Command{
		Short: "Gets whether metrics are enabled.",
		Long:  "Gets whether metrics are enabled.",
		Run: cmdutil.Run(func(args []string) (retErr error) {
			cfg, err := config.Read()
			if err != nil {
				return err
			}
			fmt.Printf("%v\n", cfg.V2.Metrics)
			return nil
		}),
	}
	commands = append(commands, cmdutil.CreateAlias(getMetrics, "config get metrics"))

	setMetrics := &cobra.Command{
		Short: "Sets whether metrics are enabled.",
		Long:  "Sets whether metrics are enabled.",
		Run: cmdutil.RunFixedArgs(1, func(args []string) (retErr error) {
			metrics := true
			if args[0] == "false" {
				metrics = false
			} else if args[0] != "true" {
				return errors.New("invalid argument; use either `true` or `false`")
			}

			cfg, err := config.Read()
			if err != nil {
				return err
			}

			cfg.V2.Metrics = metrics
			return cfg.Write()
		}),
	}
	commands = append(commands, cmdutil.CreateAlias(setMetrics, "config set metrics"))

	getActiveContext := &cobra.Command{
		Short: "Gets the currently active context.",
		Long:  "Gets the currently active context.",
		Run: cmdutil.Run(func(args []string) (retErr error) {
			cfg, err := config.Read()
			if err != nil {
				return err
			}
			fmt.Printf("%s\n", cfg.V2.ActiveContext)
			return nil
		}),
	}
	commands = append(commands, cmdutil.CreateAlias(getActiveContext, "config get active-context"))

	setActiveContext := &cobra.Command{
		Short: "Sets the currently active context.",
		Long:  "Sets the currently active context.",
		Run: cmdutil.RunFixedArgs(1, func(args []string) (retErr error) {
			cfg, err := config.Read()
			if err != nil {
				return err
			}
			if _, ok := cfg.V2.Contexts[args[0]]; !ok {
				return fmt.Errorf("context does not exist: %s", args[0])
			}
			cfg.V2.ActiveContext = args[0]
			return cfg.Write()
		}),
	}
	commands = append(commands, cmdutil.CreateAlias(setActiveContext, "config set active-context"))

	getContext := &cobra.Command{
		Short: "Gets a context.",
		Long:  "Gets the config of a context by its name.",
		Run: cmdutil.RunFixedArgs(1, func(args []string) (retErr error) {
			cfg, err := config.Read()
			if err != nil {
				return err
			}

			context, ok := cfg.V2.Contexts[args[0]]
			if !ok {
				return fmt.Errorf("context does not exist: %s", args[0])
			}

			if err = marshaller.Marshal(os.Stdout, context); err != nil {
				return err
			}

			fmt.Println()
			return nil
		}),
	}
	commands = append(commands, cmdutil.CreateAlias(getContext, "config get context"))

	var overwrite bool
	setContext := &cobra.Command{
		Short: "Set a context.",
		Long:  "Set a context config from a given name and JSON stdin.",
		Run: cmdutil.RunFixedArgs(1, func(args []string) (retErr error) {
			cfg, err := config.Read()
			if err != nil {
				return err
			}

			context, err := readContext()
			if err != nil {
				return err
			}

			if !overwrite {
				if _, ok := cfg.V2.Contexts[args[0]]; ok {
					return fmt.Errorf("context '%s' already exists, use `--overwrite` if you wish to replace it", args[0])
				}
			}

			cfg.V2.Contexts[args[0]] = context
			return cfg.Write()
		}),
	}
	setContext.Flags().BoolVar(&overwrite, "overwrite", false, "Overwrite a context if it already exists.")
	commands = append(commands, cmdutil.CreateAlias(setContext, "config set context"))

	var pachdAddress string
	updateContext := &cobra.Command{
		Short: "Updates a context.",
		Long:  "Updates an existing context config from a given name.",
		Run: cmdutil.RunCmdFixedArgs(1, func(cmd *cobra.Command, args []string) (retErr error) {
			cfg, err := config.Read()
			if err != nil {
				return err
			}

			context, ok := cfg.V2.Contexts[args[0]]
			if !ok {
				return fmt.Errorf("context does not exist: %s", args[0])
			}

			if cmd.Flags().Changed("pachd-address") {
				// Use this method since we want to differentiate between no
				// `pachd-address` flag being set (the value shouldn't be
				// changed) vs the flag being an empty string (meaning we want
				// to set the value to an empty string)
				context.PachdAddress = pachdAddress
			}

			return cfg.Write()
		}),
	}
	updateContext.Flags().StringVar(&pachdAddress, "pachd-address", "", "Set a new name pachd address.")
	commands = append(commands, cmdutil.CreateAlias(updateContext, "config update context"))

	deleteContext := &cobra.Command{
		Short: "Deletes a context.",
		Long:  "Deletes a context.",
		Run: cmdutil.RunFixedArgs(1, func(args []string) (retErr error) {
			cfg, err := config.Read()
			if err != nil {
				return err
			}
			if _, ok := cfg.V2.Contexts[args[0]]; !ok {
				return fmt.Errorf("context does not exist: %s", args[0])
			}
			if cfg.V2.ActiveContext == args[0] {
				return errors.New("cannot delete an active context")
			}
			delete(cfg.V2.Contexts, args[0])
			return cfg.Write()
		}),
	}
	commands = append(commands, cmdutil.CreateAlias(deleteContext, "config delete context"))

	listContext := &cobra.Command{
		Short: "Lists contexts.",
		Long:  "Lists contexts.",
		Run: cmdutil.Run(func(args []string) (retErr error) {
			cfg, err := config.Read()
			if err != nil {
				return err
			}

			keys := make([]string, len(cfg.V2.Contexts))
			i := 0
			for key := range cfg.V2.Contexts {
				keys[i] = key
				i++
			}
			sort.Strings(keys)

			fmt.Println(listContextHeader)
			for _, key := range keys {
				if key == cfg.V2.ActiveContext {
					fmt.Printf("*\t%s\n", key)
				} else {
					fmt.Printf("\t%s\n", key)
				}
			}
			return nil
		}),
	}
	commands = append(commands, cmdutil.CreateAlias(listContext, "config list context"))

	configDocs := &cobra.Command{
		Short: "Manages the pachyderm config.",
		Long:  "Gets/sets pachyderm config values.",
	}
	commands = append(commands, cmdutil.CreateDocsAlias(configDocs, "config", "^pachctl config "))

	configGetRoot := &cobra.Command{
		Short: "Commands for getting pachyderm config values",
		Long:  "Commands for getting pachyderm config values",
	}
	commands = append(commands, cmdutil.CreateAlias(configGetRoot, "config get"))

	configSetRoot := &cobra.Command{
		Short: "Commands for setting pachyderm config values",
		Long:  "Commands for setting pachyderm config values",
	}
	commands = append(commands, cmdutil.CreateAlias(configSetRoot, "config set"))

	configUpdateRoot := &cobra.Command{
		Short: "Commands for updating pachyderm config values",
		Long:  "Commands for updating pachyderm config values",
	}
	commands = append(commands, cmdutil.CreateAlias(configUpdateRoot, "config update"))

	configDeleteRoot := &cobra.Command{
		Short: "Commands for deleting pachyderm config values",
		Long:  "Commands for deleting pachyderm config values",
	}
	commands = append(commands, cmdutil.CreateAlias(configDeleteRoot, "config delete"))

	configListRoot := &cobra.Command{
		Short: "Commands for listing pachyderm config values",
		Long:  "Commands for listing pachyderm config values",
	}
	commands = append(commands, cmdutil.CreateAlias(configListRoot, "config list"))

	return commands
}
<commit_msg>Fix a typo (#3973)<commit_after>package cmds

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"sort"

	"github.com/pachyderm/pachyderm/src/client/pkg/config"
	"github.com/pachyderm/pachyderm/src/server/pkg/cmdutil"

	"github.com/gogo/protobuf/jsonpb"
	"github.com/spf13/cobra"
)

const (
	listContextHeader = "ACTIVE\tNAME"
)

func readContext() (*config.Context, error) {
	var buf bytes.Buffer
	var decoder *json.Decoder
	var result config.Context

	contextReader := io.TeeReader(os.Stdin, &buf)
	fmt.Println("Reading from stdin.")
	decoder = json.NewDecoder(contextReader)

	if err := jsonpb.UnmarshalNext(decoder, &result); err != nil {
		if err == io.EOF {
			return nil, err
		}
		return nil, fmt.Errorf("malformed context: %s", err)
	}
	return &result, nil
}

// Cmds returns a slice containing admin commands.
func Cmds() []*cobra.Command {
	marshaller := &jsonpb.Marshaler{
		Indent:   "  ",
		OrigName: true,
	}

	var commands []*cobra.Command

	getMetrics := &cobra.Command{
		Short: "Gets whether metrics are enabled.",
		Long:  "Gets whether metrics are enabled.",
		Run: cmdutil.Run(func(args []string) (retErr error) {
			cfg, err := config.Read()
			if err != nil {
				return err
			}
			fmt.Printf("%v\n", cfg.V2.Metrics)
			return nil
		}),
	}
	commands = append(commands, cmdutil.CreateAlias(getMetrics, "config get metrics"))

	setMetrics := &cobra.Command{
		Short: "Sets whether metrics are enabled.",
		Long:  "Sets whether metrics are enabled.",
		Run: cmdutil.RunFixedArgs(1, func(args []string) (retErr error) {
			metrics := true
			if args[0] == "false" {
				metrics = false
			} else if args[0] != "true" {
				return errors.New("invalid argument; use either `true` or `false`")
			}

			cfg, err := config.Read()
			if err != nil {
				return err
			}

			cfg.V2.Metrics = metrics
			return cfg.Write()
		}),
	}
	commands = append(commands, cmdutil.CreateAlias(setMetrics, "config set metrics"))

	getActiveContext := &cobra.Command{
		Short: "Gets the currently active context.",
		Long:  "Gets the currently active context.",
		Run: cmdutil.Run(func(args []string) (retErr error) {
			cfg, err := config.Read()
			if err != nil {
				return err
			}
			fmt.Printf("%s\n", cfg.V2.ActiveContext)
			return nil
		}),
	}
	commands = append(commands, cmdutil.CreateAlias(getActiveContext, "config get active-context"))

	setActiveContext := &cobra.Command{
		Short: "Sets the currently active context.",
		Long:  "Sets the currently active context.",
		Run: cmdutil.RunFixedArgs(1, func(args []string) (retErr error) {
			cfg, err := config.Read()
			if err != nil {
				return err
			}
			if _, ok := cfg.V2.Contexts[args[0]]; !ok {
				return fmt.Errorf("context does not exist: %s", args[0])
			}
			cfg.V2.ActiveContext = args[0]
			return cfg.Write()
		}),
	}
	commands = append(commands, cmdutil.CreateAlias(setActiveContext, "config set active-context"))

	getContext := &cobra.Command{
		Short: "Gets a context.",
		Long:  "Gets the config of a context by its name.",
		Run: cmdutil.RunFixedArgs(1, func(args []string) (retErr error) {
			cfg, err := config.Read()
			if err != nil {
				return err
			}

			context, ok := cfg.V2.Contexts[args[0]]
			if !ok {
				return fmt.Errorf("context does not exist: %s", args[0])
			}

			if err = marshaller.Marshal(os.Stdout, context); err != nil {
				return err
			}

			fmt.Println()
			return nil
		}),
	}
	commands = append(commands, cmdutil.CreateAlias(getContext, "config get context"))

	var overwrite bool
	setContext := &cobra.Command{
		Short: "Set a context.",
		Long:  "Set a context config from a given name and JSON stdin.",
		Run: cmdutil.RunFixedArgs(1, func(args []string) (retErr error) {
			cfg, err := config.Read()
			if err != nil {
				return err
			}

			context, err := readContext()
			if err != nil {
				return err
			}

			if !overwrite {
				if _, ok := cfg.V2.Contexts[args[0]]; ok {
					return fmt.Errorf("context '%s' already exists, use `--overwrite` if you wish to replace it", args[0])
				}
			}

			cfg.V2.Contexts[args[0]] = context
			return cfg.Write()
		}),
	}
	setContext.Flags().BoolVar(&overwrite, "overwrite", false, "Overwrite a context if it already exists.")
	commands = append(commands, cmdutil.CreateAlias(setContext, "config set context"))

	var pachdAddress string
	updateContext := &cobra.Command{
		Short: "Updates a context.",
		Long:  "Updates an existing context config from a given name.",
		Run: cmdutil.RunCmdFixedArgs(1, func(cmd *cobra.Command, args []string) (retErr error) {
			cfg, err := config.Read()
			if err != nil {
				return err
			}

			context, ok := cfg.V2.Contexts[args[0]]
			if !ok {
				return fmt.Errorf("context does not exist: %s", args[0])
			}

			if cmd.Flags().Changed("pachd-address") {
				// Use this method since we want to differentiate between no
				// `pachd-address` flag being set (the value shouldn't be
				// changed) vs the flag being an empty string (meaning we want
				// to set the value to an empty string)
				context.PachdAddress = pachdAddress
			}

			return cfg.Write()
		}),
	}
	updateContext.Flags().StringVar(&pachdAddress, "pachd-address", "", "Set a new name pachd address.")
	commands = append(commands, cmdutil.CreateAlias(updateContext, "config update context"))

	deleteContext := &cobra.Command{
		Short: "Deletes a context.",
		Long:  "Deletes a context.",
		Run: cmdutil.RunFixedArgs(1, func(args []string) (retErr error) {
			cfg, err := config.Read()
			if err != nil {
				return err
			}
			if _, ok := cfg.V2.Contexts[args[0]]; !ok {
				return fmt.Errorf("context does not exist: %s", args[0])
			}
			if cfg.V2.ActiveContext == args[0] {
				return errors.New("cannot delete an active context")
			}
			delete(cfg.V2.Contexts, args[0])
			return cfg.Write()
		}),
	}
	commands = append(commands, cmdutil.CreateAlias(deleteContext, "config delete context"))

	listContext := &cobra.Command{
		Short: "Lists contexts.",
		Long:  "Lists contexts.",
		Run: cmdutil.Run(func(args []string) (retErr error) {
			cfg, err := config.Read()
			if err != nil {
				return err
			}

			keys := make([]string, len(cfg.V2.Contexts))
			i := 0
			for key := range cfg.V2.Contexts {
				keys[i] = key
				i++
			}
			sort.Strings(keys)

			fmt.Println(listContextHeader)
			for _, key := range keys {
				if key == cfg.V2.ActiveContext {
					fmt.Printf("*\t%s\n", key)
				} else {
					fmt.Printf("\t%s\n", key)
				}
			}
			return nil
		}),
	}
	commands = append(commands, cmdutil.CreateAlias(listContext, "config list context"))

	configDocs := &cobra.Command{
		Short: "Manages the pachyderm config.",
		Long:  "Gets/sets pachyderm config values.",
	}
	commands = append(commands, cmdutil.CreateDocsAlias(configDocs, "config", "^pachctl config "))

	configGetRoot := &cobra.Command{
		Short: "Commands for getting pachyderm config values",
		Long:  "Commands for getting pachyderm config values",
	}
	commands = append(commands, cmdutil.CreateAlias(configGetRoot, "config get"))

	configSetRoot := &cobra.Command{
		Short: "Commands for setting pachyderm config values",
		Long:  "Commands for setting pachyderm config values",
	}
	commands = append(commands, cmdutil.CreateAlias(configSetRoot, "config set"))

	configUpdateRoot := &cobra.Command{
		Short: "Commands for updating pachyderm config values",
		Long:  "Commands for updating pachyderm config values",
	}
	commands = append(commands, cmdutil.CreateAlias(configUpdateRoot, "config update"))

	configDeleteRoot := &cobra.Command{
		Short: "Commands for deleting pachyderm config values",
		Long:  "Commands for deleting pachyderm config values",
	}
	commands = append(commands, cmdutil.CreateAlias(configDeleteRoot, "config delete"))

	configListRoot := &cobra.Command{
		Short: "Commands for listing pachyderm config values",
		Long:  "Commands for listing pachyderm config values",
	}
	commands = append(commands, cmdutil.CreateAlias(configListRoot, "config list"))

	return commands
}
<|endoftext|>


<commit_before>package obj

import (
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/url"
	"path"
	"strings"
	"time"

	"github.com/cenkalti/backoff"
	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"
	log "github.com/sirupsen/logrus"
	"golang.org/x/net/context"
)

// Client is an interface to object storage.
type Client interface {
	// Writer returns a writer which writes to an object.
	// It should error if the object already exists or we don't have sufficient
	// permissions to write it.
	Writer(name string) (io.WriteCloser, error)
	// Reader returns a reader which reads from an object.
	// If `size == 0`, the reader should read from the offset till the end of the object.
	// It should error if the object doesn't exist or we don't have sufficient
	// permission to read it.
	Reader(name string, offset uint64, size uint64) (io.ReadCloser, error)
	// Delete deletes an object.
	// It should error if the object doesn't exist or we don't have sufficient
	// permission to delete it.
	Delete(name string) error
	// Walk calls `fn` with the names of objects which can be found under `prefix`.
	Walk(prefix string, fn func(name string) error) error
	// Exsits checks if a given object already exists
	Exists(name string) bool
	// isRetryable determines if an operation should be retried given an error
	isRetryable(err error) bool
	// IsNotExist returns true if err is a non existence error
	IsNotExist(err error) bool
	// IsIgnorable returns true if the error can be ignored
	IsIgnorable(err error) bool
}

// NewGoogleClient creates a google client with the given bucket name.
func NewGoogleClient(ctx context.Context, bucket string) (Client, error) {
	return newGoogleClient(ctx, bucket)
}

// NewGoogleClientFromSecret creates a google client by reading credentials
// from a mounted GoogleSecret. You may pass "" for bucket in which case it
// will read the bucket from the secret.
func NewGoogleClientFromSecret(ctx context.Context, bucket string) (Client, error) {
	if bucket == "" {
		_bucket, err := ioutil.ReadFile("/google-secret/bucket")
		if err != nil {
			return nil, err
		}
		bucket = string(_bucket)
	}
	return NewGoogleClient(ctx, bucket)
}

// NewMicrosoftClient creates a microsoft client:
//	container   - Azure Blob Container name
//	accountName - Azure Storage Account name
// 	accountKey  - Azure Storage Account key
func NewMicrosoftClient(container string, accountName string, accountKey string) (Client, error) {
	return newMicrosoftClient(container, accountName, accountKey)
}

// NewMicrosoftClientFromSecret creates a microsoft client by reading
// credentials from a mounted MicrosoftSecret. You may pass "" for container in
// which case it will read the container from the secret.
func NewMicrosoftClientFromSecret(container string) (Client, error) {
	if container == "" {
		_container, err := ioutil.ReadFile("/microsoft-secret/container")
		if err != nil {
			return nil, err
		}
		container = string(_container)
	}
	id, err := ioutil.ReadFile("/microsoft-secret/id")
	if err != nil {
		return nil, err
	}
	secret, err := ioutil.ReadFile("/microsoft-secret/secret")
	if err != nil {
		return nil, err
	}
	return NewMicrosoftClient(container, string(id), string(secret))
}

// NewMinioClient creates an s3 compatible client with the following credentials:
//   endpoint - S3 compatible endpoint
//   bucket - S3 bucket name
//   id     - AWS access key id
//   secret - AWS secret access key
//   secure - Set to true if connection is secure.
func NewMinioClient(endpoint, bucket, id, secret string, secure bool) (Client, error) {
	return newMinioClient(endpoint, bucket, id, secret, secure)
}

// NewAmazonClient creates an amazon client with the following credentials:
//   bucket - S3 bucket name
//   distribution - cloudfront distribution ID
//   id     - AWS access key id
//   secret - AWS secret access key
//   token  - AWS access token
//   region - AWS region
func NewAmazonClient(bucket string, distribution string, id string, secret string, token string,
	region string) (Client, error) {
	return newAmazonClient(bucket, distribution, id, secret, token, region)
}

// NewMinioClientFromSecret constructs an s3 compatible client by reading
// credentials from a mounted AmazonSecret. You may pass "" for bucket in which case it
// will read the bucket from the secret.
func NewMinioClientFromSecret(bucket string) (Client, error) {
	if bucket == "" {
		_bucket, err := ioutil.ReadFile("/minio-secret/bucket")
		if err != nil {
			return nil, err
		}
		bucket = string(_bucket)
	}
	endpoint, err := ioutil.ReadFile("/minio-secret/endpoint")
	if err != nil {
		return nil, err
	}
	id, err := ioutil.ReadFile("/minio-secret/id")
	if err != nil {
		return nil, err
	}
	secret, err := ioutil.ReadFile("/minio-secret/secret")
	if err != nil {
		return nil, err
	}
	secure, err := ioutil.ReadFile("/minio-secret/secure")
	if err != nil {
		return nil, err
	}
	return NewMinioClient(string(endpoint), bucket, string(id), string(secret), string(secure) == "1")
}

// NewAmazonClientFromSecret constructs an amazon client by reading credentials
// from a mounted AmazonSecret. You may pass "" for bucket in which case it
// will read the bucket from the secret.
func NewAmazonClientFromSecret(bucket string) (Client, error) {
	var distribution []byte
	if bucket == "" {
		_bucket, err := ioutil.ReadFile("/amazon-secret/bucket")
		if err != nil {
			return nil, err
		}
		bucket = string(_bucket)
		distribution, err = ioutil.ReadFile("/amazon-secret/distribution")
		if err != nil {
			// Distribution is not required, but we can log a warning
			log.Warnln("AWS deployed without cloudfront distribution\n")
		} else {
			log.Infof("AWS deployed with cloudfront distribution at %v\n", string(distribution))
		}
	}
	id, err := ioutil.ReadFile("/amazon-secret/id")
	if err != nil {
		return nil, err
	}
	secret, err := ioutil.ReadFile("/amazon-secret/secret")
	if err != nil {
		return nil, err
	}
	token, err := ioutil.ReadFile("/amazon-secret/token")
	if err != nil {
		return nil, err
	}
	region, err := ioutil.ReadFile("/amazon-secret/region")
	if err != nil {
		return nil, err
	}
	return NewAmazonClient(bucket, string(distribution), string(id), string(secret), string(token), string(region))
}

// NewClientFromURLAndSecret constructs a client by parsing `URL` and then
// constructing the correct client for that URL using secrets.
func NewClientFromURLAndSecret(ctx context.Context, url *ObjectStoreURL) (Client, error) {
	switch url.Store {
	case "s3":
		return NewAmazonClientFromSecret(url.Bucket)
	case "gcs":
		fallthrough
	case "gs":
		return NewGoogleClientFromSecret(ctx, url.Bucket)
	case "as":
		fallthrough
	case "wasb":
		// In Azure, the first part of the path is the container name.
		return NewMicrosoftClientFromSecret(url.Bucket)
	}
	return nil, fmt.Errorf("unrecognized object store: %s", url.Bucket)
}

// ObjectStoreURL represents a parsed URL to an object in an object store.
type ObjectStoreURL struct {
	// The object store, e.g. s3, gcs, as...
	Store string
	// The "bucket" (in AWS parlance) or the "container" (in Azure parlance).
	Bucket string
	// The object itself.
	Object string
}

func ParseURL(urlStr string) (ret *ObjectStoreURL, retErr error) {
	url, err := url.Parse(urlStr)
	if err != nil {
		return nil, fmt.Errorf("error parsing url %v: %v", urlStr, err)
	}
	switch url.Scheme {
	case "s3", "gcs", "gs":
		return &ObjectStoreURL{
			Store:  url.Scheme,
			Bucket: url.Host,
			Object: strings.Trim(url.Path, "/"),
		}, nil
	case "as", "wasb":
		// In Azure, the first part of the path is the container name.
		parts := strings.Split(strings.Trim(url.Path, "/"), "/")
		if len(parts) < 1 {
			return nil, fmt.Errorf("malformed Azure URI: %v", urlStr)
		}
		return &ObjectStoreURL{
			Store:  url.Scheme,
			Bucket: parts[0],
			Object: strings.Trim(path.Join(parts[1:]...), "/"),
		}, nil
	}
	return nil, fmt.Errorf("unrecognized object store: %s", url.Scheme)
}

// NewExponentialBackOffConfig creates an exponential back-off config with
// longer wait times than the default.
func NewExponentialBackOffConfig() *backoff.ExponentialBackOff {
	config := backoff.NewExponentialBackOff()
	// We want to backoff more aggressively (i.e. wait longer) than the default
	config.InitialInterval = 1 * time.Second
	config.Multiplier = 2
	config.MaxInterval = 15 * time.Minute
	return config
}

// RetryError is used to log retry attempts.
type RetryError struct {
	Err               string
	TimeTillNextRetry string
	BytesProcessed    int
}

// BackoffReadCloser retries with exponential backoff in the case of failures
type BackoffReadCloser struct {
	client        Client
	reader        io.ReadCloser
	backoffConfig *backoff.ExponentialBackOff
}

func newBackoffReadCloser(client Client, reader io.ReadCloser) io.ReadCloser {
	return &BackoffReadCloser{
		client:        client,
		reader:        reader,
		backoffConfig: NewExponentialBackOffConfig(),
	}
}

func (b *BackoffReadCloser) Read(data []byte) (int, error) {
	bytesRead := 0
	var n int
	var err error
	backoff.RetryNotify(func() error {
		n, err = b.reader.Read(data[bytesRead:])
		bytesRead += n
		if err != nil && IsRetryable(b.client, err) {
			return err
		}
		return nil
	}, b.backoffConfig, func(err error, d time.Duration) {
		log.Infof("Error reading; retrying in %s: %#v", d, RetryError{
			Err:               err.Error(),
			TimeTillNextRetry: d.String(),
			BytesProcessed:    bytesRead,
		})
	})
	return bytesRead, err
}

// Close closes the ReaderCloser contained in b.
func (b *BackoffReadCloser) Close() error {
	return b.reader.Close()
}

// BackoffWriteCloser retries with exponential backoff in the case of failures
type BackoffWriteCloser struct {
	client        Client
	writer        io.WriteCloser
	backoffConfig *backoff.ExponentialBackOff
}

func newBackoffWriteCloser(client Client, writer io.WriteCloser) io.WriteCloser {
	return &BackoffWriteCloser{
		client:        client,
		writer:        writer,
		backoffConfig: NewExponentialBackOffConfig(),
	}
}

func (b *BackoffWriteCloser) Write(data []byte) (int, error) {
	bytesWritten := 0
	var n int
	var err error
	backoff.RetryNotify(func() error {
		n, err = b.writer.Write(data[bytesWritten:])
		bytesWritten += n
		if err != nil && IsRetryable(b.client, err) {
			return err
		}
		return nil
	}, b.backoffConfig, func(err error, d time.Duration) {
		log.Infof("Error writing; retrying in %s: %#v", d, RetryError{
			Err:               err.Error(),
			TimeTillNextRetry: d.String(),
			BytesProcessed:    bytesWritten,
		})
	})
	return bytesWritten, err
}

// Close closes the WriteCloser contained in b.
func (b *BackoffWriteCloser) Close() error {
	err := b.writer.Close()
	if b.client.IsIgnorable(err) {
		return nil
	}
	return err
}

// IsRetryable determines if an operation should be retried given an error
func IsRetryable(client Client, err error) bool {
	return isNetRetryable(err) || client.isRetryable(err)
}

func byteRange(offset uint64, size uint64) string {
	if offset == 0 && size == 0 {
		return ""
	} else if size == 0 {
		return fmt.Sprintf("%d-", offset)
	}
	return fmt.Sprintf("%d-%d", offset, offset+size-1)
}

func isNetRetryable(err error) bool {
	netErr, ok := err.(net.Error)
	return ok && netErr.Temporary()
}

// TestIsNotExist is a defensive method for checking to make sure IsNotExist is
// satisfying its semantics.
func TestIsNotExist(c Client) error {
	_, err := c.Reader(uuid.NewWithoutDashes(), 0, 0)
	if !c.IsNotExist(err) {
		return fmt.Errorf("storage is unable to discern NotExist errors, \"%s\" should count as NotExist", err.Error())
	}
	return nil
}
<commit_msg>Fix lint errors<commit_after>package obj

import (
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/url"
	"path"
	"strings"
	"time"

	"github.com/cenkalti/backoff"
	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"
	log "github.com/sirupsen/logrus"
	"golang.org/x/net/context"
)

// Client is an interface to object storage.
type Client interface {
	// Writer returns a writer which writes to an object.
	// It should error if the object already exists or we don't have sufficient
	// permissions to write it.
	Writer(name string) (io.WriteCloser, error)
	// Reader returns a reader which reads from an object.
	// If `size == 0`, the reader should read from the offset till the end of the object.
	// It should error if the object doesn't exist or we don't have sufficient
	// permission to read it.
	Reader(name string, offset uint64, size uint64) (io.ReadCloser, error)
	// Delete deletes an object.
	// It should error if the object doesn't exist or we don't have sufficient
	// permission to delete it.
	Delete(name string) error
	// Walk calls `fn` with the names of objects which can be found under `prefix`.
	Walk(prefix string, fn func(name string) error) error
	// Exsits checks if a given object already exists
	Exists(name string) bool
	// isRetryable determines if an operation should be retried given an error
	isRetryable(err error) bool
	// IsNotExist returns true if err is a non existence error
	IsNotExist(err error) bool
	// IsIgnorable returns true if the error can be ignored
	IsIgnorable(err error) bool
}

// NewGoogleClient creates a google client with the given bucket name.
func NewGoogleClient(ctx context.Context, bucket string) (Client, error) {
	return newGoogleClient(ctx, bucket)
}

// NewGoogleClientFromSecret creates a google client by reading credentials
// from a mounted GoogleSecret. You may pass "" for bucket in which case it
// will read the bucket from the secret.
func NewGoogleClientFromSecret(ctx context.Context, bucket string) (Client, error) {
	if bucket == "" {
		_bucket, err := ioutil.ReadFile("/google-secret/bucket")
		if err != nil {
			return nil, err
		}
		bucket = string(_bucket)
	}
	return NewGoogleClient(ctx, bucket)
}

// NewMicrosoftClient creates a microsoft client:
//	container   - Azure Blob Container name
//	accountName - Azure Storage Account name
// 	accountKey  - Azure Storage Account key
func NewMicrosoftClient(container string, accountName string, accountKey string) (Client, error) {
	return newMicrosoftClient(container, accountName, accountKey)
}

// NewMicrosoftClientFromSecret creates a microsoft client by reading
// credentials from a mounted MicrosoftSecret. You may pass "" for container in
// which case it will read the container from the secret.
func NewMicrosoftClientFromSecret(container string) (Client, error) {
	if container == "" {
		_container, err := ioutil.ReadFile("/microsoft-secret/container")
		if err != nil {
			return nil, err
		}
		container = string(_container)
	}
	id, err := ioutil.ReadFile("/microsoft-secret/id")
	if err != nil {
		return nil, err
	}
	secret, err := ioutil.ReadFile("/microsoft-secret/secret")
	if err != nil {
		return nil, err
	}
	return NewMicrosoftClient(container, string(id), string(secret))
}

// NewMinioClient creates an s3 compatible client with the following credentials:
//   endpoint - S3 compatible endpoint
//   bucket - S3 bucket name
//   id     - AWS access key id
//   secret - AWS secret access key
//   secure - Set to true if connection is secure.
func NewMinioClient(endpoint, bucket, id, secret string, secure bool) (Client, error) {
	return newMinioClient(endpoint, bucket, id, secret, secure)
}

// NewAmazonClient creates an amazon client with the following credentials:
//   bucket - S3 bucket name
//   distribution - cloudfront distribution ID
//   id     - AWS access key id
//   secret - AWS secret access key
//   token  - AWS access token
//   region - AWS region
func NewAmazonClient(bucket string, distribution string, id string, secret string, token string,
	region string) (Client, error) {
	return newAmazonClient(bucket, distribution, id, secret, token, region)
}

// NewMinioClientFromSecret constructs an s3 compatible client by reading
// credentials from a mounted AmazonSecret. You may pass "" for bucket in which case it
// will read the bucket from the secret.
func NewMinioClientFromSecret(bucket string) (Client, error) {
	if bucket == "" {
		_bucket, err := ioutil.ReadFile("/minio-secret/bucket")
		if err != nil {
			return nil, err
		}
		bucket = string(_bucket)
	}
	endpoint, err := ioutil.ReadFile("/minio-secret/endpoint")
	if err != nil {
		return nil, err
	}
	id, err := ioutil.ReadFile("/minio-secret/id")
	if err != nil {
		return nil, err
	}
	secret, err := ioutil.ReadFile("/minio-secret/secret")
	if err != nil {
		return nil, err
	}
	secure, err := ioutil.ReadFile("/minio-secret/secure")
	if err != nil {
		return nil, err
	}
	return NewMinioClient(string(endpoint), bucket, string(id), string(secret), string(secure) == "1")
}

// NewAmazonClientFromSecret constructs an amazon client by reading credentials
// from a mounted AmazonSecret. You may pass "" for bucket in which case it
// will read the bucket from the secret.
func NewAmazonClientFromSecret(bucket string) (Client, error) {
	var distribution []byte
	if bucket == "" {
		_bucket, err := ioutil.ReadFile("/amazon-secret/bucket")
		if err != nil {
			return nil, err
		}
		bucket = string(_bucket)
		distribution, err = ioutil.ReadFile("/amazon-secret/distribution")
		if err != nil {
			// Distribution is not required, but we can log a warning
			log.Warnln("AWS deployed without cloudfront distribution\n")
		} else {
			log.Infof("AWS deployed with cloudfront distribution at %v\n", string(distribution))
		}
	}
	id, err := ioutil.ReadFile("/amazon-secret/id")
	if err != nil {
		return nil, err
	}
	secret, err := ioutil.ReadFile("/amazon-secret/secret")
	if err != nil {
		return nil, err
	}
	token, err := ioutil.ReadFile("/amazon-secret/token")
	if err != nil {
		return nil, err
	}
	region, err := ioutil.ReadFile("/amazon-secret/region")
	if err != nil {
		return nil, err
	}
	return NewAmazonClient(bucket, string(distribution), string(id), string(secret), string(token), string(region))
}

// NewClientFromURLAndSecret constructs a client by parsing `URL` and then
// constructing the correct client for that URL using secrets.
func NewClientFromURLAndSecret(ctx context.Context, url *ObjectStoreURL) (Client, error) {
	switch url.Store {
	case "s3":
		return NewAmazonClientFromSecret(url.Bucket)
	case "gcs":
		fallthrough
	case "gs":
		return NewGoogleClientFromSecret(ctx, url.Bucket)
	case "as":
		fallthrough
	case "wasb":
		// In Azure, the first part of the path is the container name.
		return NewMicrosoftClientFromSecret(url.Bucket)
	}
	return nil, fmt.Errorf("unrecognized object store: %s", url.Bucket)
}

// ObjectStoreURL represents a parsed URL to an object in an object store.
type ObjectStoreURL struct {
	// The object store, e.g. s3, gcs, as...
	Store string
	// The "bucket" (in AWS parlance) or the "container" (in Azure parlance).
	Bucket string
	// The object itself.
	Object string
}

// ParseURL parses an URL into ObjectStoreURL.
func ParseURL(urlStr string) (*ObjectStoreURL, error) {
	url, err := url.Parse(urlStr)
	if err != nil {
		return nil, fmt.Errorf("error parsing url %v: %v", urlStr, err)
	}
	switch url.Scheme {
	case "s3", "gcs", "gs":
		return &ObjectStoreURL{
			Store:  url.Scheme,
			Bucket: url.Host,
			Object: strings.Trim(url.Path, "/"),
		}, nil
	case "as", "wasb":
		// In Azure, the first part of the path is the container name.
		parts := strings.Split(strings.Trim(url.Path, "/"), "/")
		if len(parts) < 1 {
			return nil, fmt.Errorf("malformed Azure URI: %v", urlStr)
		}
		return &ObjectStoreURL{
			Store:  url.Scheme,
			Bucket: parts[0],
			Object: strings.Trim(path.Join(parts[1:]...), "/"),
		}, nil
	}
	return nil, fmt.Errorf("unrecognized object store: %s", url.Scheme)
}

// NewExponentialBackOffConfig creates an exponential back-off config with
// longer wait times than the default.
func NewExponentialBackOffConfig() *backoff.ExponentialBackOff {
	config := backoff.NewExponentialBackOff()
	// We want to backoff more aggressively (i.e. wait longer) than the default
	config.InitialInterval = 1 * time.Second
	config.Multiplier = 2
	config.MaxInterval = 15 * time.Minute
	return config
}

// RetryError is used to log retry attempts.
type RetryError struct {
	Err               string
	TimeTillNextRetry string
	BytesProcessed    int
}

// BackoffReadCloser retries with exponential backoff in the case of failures
type BackoffReadCloser struct {
	client        Client
	reader        io.ReadCloser
	backoffConfig *backoff.ExponentialBackOff
}

func newBackoffReadCloser(client Client, reader io.ReadCloser) io.ReadCloser {
	return &BackoffReadCloser{
		client:        client,
		reader:        reader,
		backoffConfig: NewExponentialBackOffConfig(),
	}
}

func (b *BackoffReadCloser) Read(data []byte) (int, error) {
	bytesRead := 0
	var n int
	var err error
	backoff.RetryNotify(func() error {
		n, err = b.reader.Read(data[bytesRead:])
		bytesRead += n
		if err != nil && IsRetryable(b.client, err) {
			return err
		}
		return nil
	}, b.backoffConfig, func(err error, d time.Duration) {
		log.Infof("Error reading; retrying in %s: %#v", d, RetryError{
			Err:               err.Error(),
			TimeTillNextRetry: d.String(),
			BytesProcessed:    bytesRead,
		})
	})
	return bytesRead, err
}

// Close closes the ReaderCloser contained in b.
func (b *BackoffReadCloser) Close() error {
	return b.reader.Close()
}

// BackoffWriteCloser retries with exponential backoff in the case of failures
type BackoffWriteCloser struct {
	client        Client
	writer        io.WriteCloser
	backoffConfig *backoff.ExponentialBackOff
}

func newBackoffWriteCloser(client Client, writer io.WriteCloser) io.WriteCloser {
	return &BackoffWriteCloser{
		client:        client,
		writer:        writer,
		backoffConfig: NewExponentialBackOffConfig(),
	}
}

func (b *BackoffWriteCloser) Write(data []byte) (int, error) {
	bytesWritten := 0
	var n int
	var err error
	backoff.RetryNotify(func() error {
		n, err = b.writer.Write(data[bytesWritten:])
		bytesWritten += n
		if err != nil && IsRetryable(b.client, err) {
			return err
		}
		return nil
	}, b.backoffConfig, func(err error, d time.Duration) {
		log.Infof("Error writing; retrying in %s: %#v", d, RetryError{
			Err:               err.Error(),
			TimeTillNextRetry: d.String(),
			BytesProcessed:    bytesWritten,
		})
	})
	return bytesWritten, err
}

// Close closes the WriteCloser contained in b.
func (b *BackoffWriteCloser) Close() error {
	err := b.writer.Close()
	if b.client.IsIgnorable(err) {
		return nil
	}
	return err
}

// IsRetryable determines if an operation should be retried given an error
func IsRetryable(client Client, err error) bool {
	return isNetRetryable(err) || client.isRetryable(err)
}

func byteRange(offset uint64, size uint64) string {
	if offset == 0 && size == 0 {
		return ""
	} else if size == 0 {
		return fmt.Sprintf("%d-", offset)
	}
	return fmt.Sprintf("%d-%d", offset, offset+size-1)
}

func isNetRetryable(err error) bool {
	netErr, ok := err.(net.Error)
	return ok && netErr.Temporary()
}

// TestIsNotExist is a defensive method for checking to make sure IsNotExist is
// satisfying its semantics.
func TestIsNotExist(c Client) error {
	_, err := c.Reader(uuid.NewWithoutDashes(), 0, 0)
	if !c.IsNotExist(err) {
		return fmt.Errorf("storage is unable to discern NotExist errors, \"%s\" should count as NotExist", err.Error())
	}
	return nil
}
<|endoftext|>


<commit_before>package todoist

import (
	"strings"
)

type Project struct {
	HaveID
	HaveParentID
	HaveIndent
	Collapsed    int    `json:"collapsed"`
	Color        int    `json:"color"`
	HasMoreNotes bool   `json:"has_more_notes"`
	InboxProject bool   `json:"inbox_project"`
	IsArchived   int    `json:"is_archived"`
	IsDeleted    int    `json:"is_deleted"`
	ItemOrder    int    `json:"item_order"`
	Name         string `json:"name"`
	Shared       bool   `json:"shared"`
}

type Projects []Project

func (a Projects) Len() int           { return len(a) }
func (a Projects) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a Projects) Less(i, j int) bool { return a[i].ID < a[j].ID }

func (a Projects) At(i int) IDCarrier { return a[i] }

func (a Projects) GetIDByName(name string) int {
	for _, pjt := range a {
		if pjt.Name == name {
			return pjt.GetID()
		}
	}
	return 0
}

func (a Projects) GetIDsByName(name string, isAll bool) []int {
	ids := []int{}
	for _, pjt := range a {
		if strings.Contains(pjt.Name, name) {
			ids = append(ids, pjt.ID)
			if isAll {
				parentID := pjt.ID
				// Find all children which has the project as parent
				ids = append(ids, parentID)
				for _, id := range childProjectIDs(parentID, a) {
					ids = append(ids, id)
				}
			}
		}
	}
	return ids
}

func childProjectIDs(parentId int, projects Projects) []int {
	ids := []int{}
	for _, pjt := range projects {
		id, err := pjt.GetParentID()
		if err != nil {
			continue
		}

		if id == parentId {
			ids = append(ids, pjt.ID)
			for _, id := range childProjectIDs(pjt.ID, projects) {
				ids = append(ids, id)
			}
		}
	}
	return ids
}
<commit_msg>Compare with lower case<commit_after>package todoist

import (
	"strings"
)

type Project struct {
	HaveID
	HaveParentID
	HaveIndent
	Collapsed    int    `json:"collapsed"`
	Color        int    `json:"color"`
	HasMoreNotes bool   `json:"has_more_notes"`
	InboxProject bool   `json:"inbox_project"`
	IsArchived   int    `json:"is_archived"`
	IsDeleted    int    `json:"is_deleted"`
	ItemOrder    int    `json:"item_order"`
	Name         string `json:"name"`
	Shared       bool   `json:"shared"`
}

type Projects []Project

func (a Projects) Len() int           { return len(a) }
func (a Projects) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a Projects) Less(i, j int) bool { return a[i].ID < a[j].ID }

func (a Projects) At(i int) IDCarrier { return a[i] }

func (a Projects) GetIDByName(name string) int {
	for _, pjt := range a {
		if pjt.Name == name {
			return pjt.GetID()
		}
	}
	return 0
}

func (a Projects) GetIDsByName(name string, isAll bool) []int {
	ids := []int{}
	name = strings.ToLower(name)
	for _, pjt := range a {
		if strings.Contains(strings.ToLower(pjt.Name), name) {
			ids = append(ids, pjt.ID)
			if isAll {
				parentID := pjt.ID
				// Find all children which has the project as parent
				ids = append(ids, parentID)
				for _, id := range childProjectIDs(parentID, a) {
					ids = append(ids, id)
				}
			}
		}
	}
	return ids
}

func childProjectIDs(parentId int, projects Projects) []int {
	ids := []int{}
	for _, pjt := range projects {
		id, err := pjt.GetParentID()
		if err != nil {
			continue
		}

		if id == parentId {
			ids = append(ids, pjt.ID)
			for _, id := range childProjectIDs(pjt.ID, projects) {
				ids = append(ids, id)
			}
		}
	}
	return ids
}
<|endoftext|>


<commit_before>package lib

import (
	"encoding/json"
	"net/http"
	"net/http/httputil"
	"strings"
)

func backend(c *Config, r *http.Request) (string, string, bool) {
	var (
		pathToMatch string
	)
	if c.Version != "" {
		ps := strings.SplitN(r.URL.Path, "/", 3)
		if len(ps) != 3 {
			return "", "", false //expect URL of form /{version}/
		}
		pathToMatch = "/" + ps[2]
	} else {
		pathToMatch = r.URL.Path
	}
	for k, v := range c.Rules {
		if strings.Index(pathToMatch, k) == 0 {
			return v, pathToMatch, true
		}
	}
	return "", "", false
}

// New creates a new gateway.
func New(c *Config) http.HandlerFunc {
	return func(w http.ResponseWriter, req *http.Request) {
		b, url, ok := backend(c, req)
		if !ok {
			resp, _ := json.Marshal(c.NotFoundResponse)
			w.WriteHeader(http.StatusNotFound)
			w.Write(resp)
			return
		}
		(&httputil.ReverseProxy{
			Director: func(r *http.Request) {
				r.URL.Scheme = "http"
				r.URL.Host = b
				r.URL.Path = url
				r.Host = b
			},
		}).ServeHTTP(w, req)
	}
}
<commit_msg>Add content-type header for 404 responses<commit_after>package lib

import (
	"encoding/json"
	"net/http"
	"net/http/httputil"
	"strings"
)

func backend(c *Config, r *http.Request) (string, string, bool) {
	var (
		pathToMatch string
	)
	if c.Version != "" {
		ps := strings.SplitN(r.URL.Path, "/", 3)
		if len(ps) != 3 {
			return "", "", false //expect URL of form /{version}/
		}
		pathToMatch = "/" + ps[2]
	} else {
		pathToMatch = r.URL.Path
	}
	for k, v := range c.Rules {
		if strings.Index(pathToMatch, k) == 0 {
			return v, pathToMatch, true
		}
	}
	return "", "", false
}

// New creates a new gateway.
func New(c *Config) http.HandlerFunc {
	return func(w http.ResponseWriter, req *http.Request) {
		b, url, ok := backend(c, req)
		if !ok {
			resp, _ := json.Marshal(c.NotFoundResponse)
			w.WriteHeader(http.StatusNotFound)
			w.Header().Set("Content-type", "application/json")
			w.Write(resp)
			return
		}
		(&httputil.ReverseProxy{
			Director: func(r *http.Request) {
				r.URL.Scheme = "http"
				r.URL.Host = b
				r.URL.Path = url
				r.Host = b
			},
		}).ServeHTTP(w, req)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"fmt"
	"flag"
	"io"
	"log"
	"encoding/json"
	"link-select/add"
	"link-select/sel"
	"link-select/types"
)

var system types.ConfigRecord
var files types.ConfigRecord

var addLink string
var selectLink string

func init() {
	const (
		defaultAdd = "read"
		usageAdd = "add a link (to json file)"
		defaultSelect = "read"
		usageSelect = "select a link (from json file)"
	)

	flag.StringVar(&addLink, "add-link", defaultAdd, usageAdd)
	flag.StringVar(&addLink, "a", defaultAdd, usageAdd + " (shorthand)")
	flag.StringVar(&selectLink, "sel-link", defaultSelect, usageSelect)
	flag.StringVar(&selectLink, "s", defaultSelect, usageSelect + " (shorthand)")
}

func loadConfig() {
	configFile, err := os.Open("config.json")
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error while opening config JSON file\n")
		log.Fatal(err)
		os.Exit(-1)
	}

	dec := json.NewDecoder(configFile)
	for {
		var c types.Config
		if err := dec.Decode(&c); err == io.EOF {
			break
		} else if err != nil {
			fmt.Fprintf(os.Stderr, "Error while parsing config JSON file\n")
			log.Fatal(err)
			os.Exit(-1)
		}

		system = c["system"]
		files = c["files"]
	}

	/*
	// debug
	fmt.Println("system:", system)
	fmt.Println("files:", files)
	*/
}

func processArgs(arg *flag.Flag) {
	fmt.Println(selectLink)

	switch arg.Name {
	case "add-link":
		add.AddLink(arg)
	case "a":
		add.AddLink(arg)
	case "sel-link":
		sel.SelectLink(arg, files[selectLink], system["browser"])
	case "s":
		sel.SelectLink(arg, files[selectLink], system["browser"])
	default:

	}
}

func main() {
	if len(os.Args) != 2 {
		fmt.Fprintf(os.Stderr, "USAGE: link-select " +
			"[--add-link=<link-to-add> |" +
			" --sel-link=[read|watch|book]])\n")
		os.Exit(-1)
	}

	flag.Parse()
	if flag.Parsed() {
		loadConfig()
    	flag.Visit(processArgs)
	} else {
		fmt.Fprintf(os.Stderr, "Error while parsing command-line arguments\n")
		os.Exit(-1)
	}
}<commit_msg>Fix USAGE message<commit_after>package main

import (
	"os"
	"fmt"
	"flag"
	"io"
	"log"
	"encoding/json"
	"link-select/add"
	"link-select/sel"
	"link-select/types"
)

var system types.ConfigRecord
var files types.ConfigRecord

var addLink string
var selectLink string

func init() {
	const (
		defaultAdd = "read"
		usageAdd = "add a link (to json file)"
		defaultSelect = "read"
		usageSelect = "select a link (from json file)"
	)

	flag.StringVar(&addLink, "add-link", defaultAdd, usageAdd)
	flag.StringVar(&addLink, "a", defaultAdd, usageAdd + " (shorthand)")
	flag.StringVar(&selectLink, "sel-link", defaultSelect, usageSelect)
	flag.StringVar(&selectLink, "s", defaultSelect, usageSelect + " (shorthand)")
}

func loadConfig() {
	configFile, err := os.Open("config.json")
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error while opening config JSON file\n")
		log.Fatal(err)
		os.Exit(-1)
	}

	dec := json.NewDecoder(configFile)
	for {
		var c types.Config
		if err := dec.Decode(&c); err == io.EOF {
			break
		} else if err != nil {
			fmt.Fprintf(os.Stderr, "Error while parsing config JSON file\n")
			log.Fatal(err)
			os.Exit(-1)
		}

		system = c["system"]
		files = c["files"]
	}

	/*
	// debug
	fmt.Println("system:", system)
	fmt.Println("files:", files)
	*/
}

func processArgs(arg *flag.Flag) {
	fmt.Println(selectLink)

	switch arg.Name {
	case "add-link":
		add.AddLink(arg)
	case "a":
		add.AddLink(arg)
	case "sel-link":
		sel.SelectLink(arg, files[selectLink], system["browser"])
	case "s":
		sel.SelectLink(arg, files[selectLink], system["browser"])
	default:

	}
}

func main() {
	if len(os.Args) != 2 {
		fmt.Fprintf(os.Stderr, "USAGE: link-select " +
			"[--add-link=read|watch|book |" +
			" --sel-link=read|watch|book])\n")
		os.Exit(-1)
	}

	flag.Parse()
	if flag.Parsed() {
		loadConfig()
    	flag.Visit(processArgs)
	} else {
		fmt.Fprintf(os.Stderr, "Error while parsing command-line arguments\n")
		os.Exit(-1)
	}
}<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package memfs

import (
	"fmt"
	"os"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/syncutil"
)

// Common attributes for files and directories.
//
// TODO(jacobsa): Add tests for interacting with a file/directory after it has
// been unlinked, including creating a new file. Make sure we don't screw up
// and reuse an inode ID while it is still in use.
type inode struct {
	/////////////////////////
	// Constant data
	/////////////////////////

	// Is this a directory? If not, it is a file.
	dir bool

	/////////////////////////
	// Mutable state
	/////////////////////////

	mu syncutil.InvariantMutex

	// The current attributes of this inode.
	//
	// INVARIANT: No non-permission mode bits are set besides os.ModeDir
	// INVARIANT: If dir, then os.ModeDir is set
	// INVARIANT: If !dir, then os.ModeDir is not set
	attributes fuse.InodeAttributes // GUARDED_BY(mu)

	// For directories, entries describing the children of the directory.
	//
	// This array can never be shortened, nor can its elements be moved, because
	// we use its indices for Dirent.Offset, which is exposed to the user who
	// might be calling readdir in a loop while concurrently modifying the
	// directory. Unused entries can, however, be reused.
	//
	// TODO(jacobsa): Add good tests exercising concurrent modifications while
	// doing readdir, seekdir, etc. calls.
	//
	// INVARIANT: If dir is false, this is nil.
	// INVARIANT: For each i, entries[i].Offset == i+1
	// INVARIANT: Contains no duplicate names.
	entries []fuseutil.Dirent // GUARDED_BY(mu)

	// For files, the current contents of the file.
	//
	// INVARIANT: If dir is true, this is nil.
	contents []byte // GUARDED_BY(mu)
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func newInode(attrs fuse.InodeAttributes) (in *inode) {
	in = &inode{
		dir:        (attrs.Mode&os.ModeDir != 0),
		attributes: attrs,
	}

	in.mu = syncutil.NewInvariantMutex(in.checkInvariants)
	return
}

func (inode *inode) checkInvariants() {
	// No non-permission mode bits should be set besides os.ModeDir.
	if inode.attributes.Mode & ^(os.ModePerm|os.ModeDir) != 0 {
		panic(fmt.Sprintf("Unexpected mode: %v", inode.attributes.Mode))
	}

	// Check os.ModeDir.
	if inode.dir != (inode.attributes.Mode&os.ModeDir == os.ModeDir) {
		panic(
			fmt.Sprintf(
				"Unexpected mode: %v, dir: %v",
				inode.attributes.Mode,
				inode.dir))
	}

	// Check directory-specific stuff.
	if inode.dir {
		if inode.contents != nil {
			panic("Non-nil contents in a directory.")
		}

		childNames := make(map[string]struct{})
		for i, e := range inode.entries {
			if e.Offset != fuse.DirOffset(i+1) {
				panic(fmt.Sprintf("Unexpected offset: %v", e.Offset))
			}

			if _, ok := childNames[e.Name]; ok {
				panic(fmt.Sprintf("Duplicate name: %s", e.Name))
			}

			childNames[e.Name] = struct{}{}
		}
	}

	// Check file-specific stuff.
	if !inode.dir {
		if inode.entries != nil {
			panic("Non-nil entries in a file.")
		}
	}
}

////////////////////////////////////////////////////////////////////////
// Public methods
////////////////////////////////////////////////////////////////////////

// Find an entry for the given child name and return its inode ID.
//
// REQUIRES: inode.dir
// SHARED_LOCKS_REQUIRED(inode.mu)
func (inode *inode) LookUpChild(name string) (id fuse.InodeID, ok bool) {
	if !inode.dir {
		panic("LookUpChild called on non-directory.")
	}

	for _, e := range inode.entries {
		if e.Name == name {
			id = e.Inode
			ok = true
			return
		}
	}

	return
}

// Add an entry for a child.
//
// REQUIRES: inode.dir
// EXCLUSIVE_LOCKS_REQUIRED(inode.mu)
func (inode *inode) AddChild(
	id fuse.InodeID,
	name string,
	dt fuseutil.DirentType) {
	e := fuseutil.Dirent{
		Offset: fuse.DirOffset(len(inode.entries) + 1),
		Inode:  id,
		Name:   name,
		Type:   dt,
	}

	inode.entries = append(inode.entries, e)
}

// Remove an entry for a child.
//
// REQUIRES: inode.dir
// REQUIRES: An entry for the given name exists.
// EXCLUSIVE_LOCKS_REQUIRED(inode.mu)
func (inode *inode) RemoveChild(name string)

// Serve a ReadDir request.
//
// REQUIRED: inode.dir
// SHARED_LOCKS_REQUIRED(inode.mu)
func (inode *inode) ReadDir(offset int, size int) (data []byte, err error) {
	if !inode.dir {
		panic("ReadDir called on non-directory.")
	}

	for i := offset; i < len(inode.entries); i++ {
		data = fuseutil.AppendDirent(data, inode.entries[i])

		// Trim and stop early if we've exceeded the requested size.
		if len(data) > size {
			data = data[:size]
			break
		}
	}

	return
}
<commit_msg>Refactored inode.LookUpChild.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package memfs

import (
	"fmt"
	"os"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/syncutil"
)

// Common attributes for files and directories.
//
// TODO(jacobsa): Add tests for interacting with a file/directory after it has
// been unlinked, including creating a new file. Make sure we don't screw up
// and reuse an inode ID while it is still in use.
type inode struct {
	/////////////////////////
	// Constant data
	/////////////////////////

	// Is this a directory? If not, it is a file.
	dir bool

	/////////////////////////
	// Mutable state
	/////////////////////////

	mu syncutil.InvariantMutex

	// The current attributes of this inode.
	//
	// INVARIANT: No non-permission mode bits are set besides os.ModeDir
	// INVARIANT: If dir, then os.ModeDir is set
	// INVARIANT: If !dir, then os.ModeDir is not set
	attributes fuse.InodeAttributes // GUARDED_BY(mu)

	// For directories, entries describing the children of the directory.
	//
	// This array can never be shortened, nor can its elements be moved, because
	// we use its indices for Dirent.Offset, which is exposed to the user who
	// might be calling readdir in a loop while concurrently modifying the
	// directory. Unused entries can, however, be reused.
	//
	// TODO(jacobsa): Add good tests exercising concurrent modifications while
	// doing readdir, seekdir, etc. calls.
	//
	// INVARIANT: If dir is false, this is nil.
	// INVARIANT: For each i, entries[i].Offset == i+1
	// INVARIANT: Contains no duplicate names.
	entries []fuseutil.Dirent // GUARDED_BY(mu)

	// For files, the current contents of the file.
	//
	// INVARIANT: If dir is true, this is nil.
	contents []byte // GUARDED_BY(mu)
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func newInode(attrs fuse.InodeAttributes) (in *inode) {
	in = &inode{
		dir:        (attrs.Mode&os.ModeDir != 0),
		attributes: attrs,
	}

	in.mu = syncutil.NewInvariantMutex(in.checkInvariants)
	return
}

func (inode *inode) checkInvariants() {
	// No non-permission mode bits should be set besides os.ModeDir.
	if inode.attributes.Mode & ^(os.ModePerm|os.ModeDir) != 0 {
		panic(fmt.Sprintf("Unexpected mode: %v", inode.attributes.Mode))
	}

	// Check os.ModeDir.
	if inode.dir != (inode.attributes.Mode&os.ModeDir == os.ModeDir) {
		panic(
			fmt.Sprintf(
				"Unexpected mode: %v, dir: %v",
				inode.attributes.Mode,
				inode.dir))
	}

	// Check directory-specific stuff.
	if inode.dir {
		if inode.contents != nil {
			panic("Non-nil contents in a directory.")
		}

		childNames := make(map[string]struct{})
		for i, e := range inode.entries {
			if e.Offset != fuse.DirOffset(i+1) {
				panic(fmt.Sprintf("Unexpected offset: %v", e.Offset))
			}

			if _, ok := childNames[e.Name]; ok {
				panic(fmt.Sprintf("Duplicate name: %s", e.Name))
			}

			childNames[e.Name] = struct{}{}
		}
	}

	// Check file-specific stuff.
	if !inode.dir {
		if inode.entries != nil {
			panic("Non-nil entries in a file.")
		}
	}
}

// Return the index of the child within inode.entries, if it exists.
//
// REQUIRES: inode.dir
// SHARED_LOCKS_REQUIRED(inode.mu)
func (inode *inode) findChild(name string) (i int, ok bool) {
	if !inode.dir {
		panic("findChild called on non-directory.")
	}

	var e fuseutil.Dirent
	for i, e = range inode.entries {
		if e.Name == name {
			ok = true
			return
		}
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Public methods
////////////////////////////////////////////////////////////////////////

// Find an entry for the given child name and return its inode ID.
//
// REQUIRES: inode.dir
// SHARED_LOCKS_REQUIRED(inode.mu)
func (inode *inode) LookUpChild(name string) (id fuse.InodeID, ok bool) {
	index, ok := inode.findChild(name)
	if ok {
		id = inode.entries[index].Inode
	}

	return
}

// Add an entry for a child.
//
// REQUIRES: inode.dir
// EXCLUSIVE_LOCKS_REQUIRED(inode.mu)
func (inode *inode) AddChild(
	id fuse.InodeID,
	name string,
	dt fuseutil.DirentType) {
	e := fuseutil.Dirent{
		Offset: fuse.DirOffset(len(inode.entries) + 1),
		Inode:  id,
		Name:   name,
		Type:   dt,
	}

	inode.entries = append(inode.entries, e)
}

// Remove an entry for a child.
//
// REQUIRES: inode.dir
// REQUIRES: An entry for the given name exists.
// EXCLUSIVE_LOCKS_REQUIRED(inode.mu)
func (inode *inode) RemoveChild(name string)

// Serve a ReadDir request.
//
// REQUIRES: inode.dir
// SHARED_LOCKS_REQUIRED(inode.mu)
func (inode *inode) ReadDir(offset int, size int) (data []byte, err error) {
	if !inode.dir {
		panic("ReadDir called on non-directory.")
	}

	for i := offset; i < len(inode.entries); i++ {
		data = fuseutil.AppendDirent(data, inode.entries[i])

		// Trim and stop early if we've exceeded the requested size.
		if len(data) > size {
			data = data[:size]
			break
		}
	}

	return
}
<|endoftext|>


<commit_before>package keys

// ParsedInternalKey is a parsed or splited internal representation.
type ParsedInternalKey struct {
	UserKey  []byte
	Kind     Kind
	Sequence Sequence
}

// Parse parses input key as internal key and returns true for valid internal
// key. It is illegal to access other fields and methods after returning false.
func (k *ParsedInternalKey) Parse(key []byte) bool {
	i := len(key) - TagBytes
	if i < 0 {
		return false
	}
	k.UserKey = key[:i:i]
	k.Sequence, k.Kind = ExtractTag(key[i:])
	return k.Kind <= maxKind
}

// Tag returns tag of this internal key.
func (k *ParsedInternalKey) Tag() Tag {
	return PackTag(k.Sequence, k.Kind)
}

// Append appends this internal key to destination buffer.
func (k *ParsedInternalKey) Append(dst []byte) []byte {
	var buf [TagBytes]byte
	CombineTag(buf[:], k.Sequence, k.Kind)
	dst = append(dst, k.UserKey...)
	return append(dst, buf[:]...)
}
<commit_msg>Order keys.ParsedInternalKey fields as UserKey, Sequence, Kind<commit_after>package keys

// ParsedInternalKey is a parsed or splited internal representation.
type ParsedInternalKey struct {
	UserKey  []byte
	Sequence Sequence
	Kind     Kind
}

// Parse parses input key as internal key and returns true for valid internal
// key. It is illegal to access other fields and methods after returning false.
func (k *ParsedInternalKey) Parse(key []byte) bool {
	i := len(key) - TagBytes
	if i < 0 {
		return false
	}
	k.UserKey = key[:i:i]
	k.Sequence, k.Kind = ExtractTag(key[i:])
	return k.Kind <= maxKind
}

// Tag returns tag of this internal key.
func (k *ParsedInternalKey) Tag() Tag {
	return PackTag(k.Sequence, k.Kind)
}

// Append appends this internal key to destination buffer.
func (k *ParsedInternalKey) Append(dst []byte) []byte {
	var buf [TagBytes]byte
	CombineTag(buf[:], k.Sequence, k.Kind)
	dst = append(dst, k.UserKey...)
	return append(dst, buf[:]...)
}
<|endoftext|>


<commit_before>package refactoring

import (
	"fmt"

	"github.com/hashicorp/terraform/internal/addrs"
	"github.com/hashicorp/terraform/internal/dag"
	"github.com/hashicorp/terraform/internal/states"
)

type MoveResult struct {
	From, To addrs.AbsResourceInstance
}

// ApplyMoves modifies in-place the given state object so that any existing
// objects that are matched by a "from" argument of one of the move statements
// will be moved to instead appear at the "to" argument of that statement.
//
// The result is a map from the unique key of each absolute address that was
// either the source or destination of a move to a MoveResult describing
// what happened at that address.
//
// ApplyMoves does not have any error situations itself, and will instead just
// ignore any unresolvable move statements. Validation of a set of moves is
// a separate concern applied to the configuration, because validity of
// moves is always dependent only on the configuration, not on the state.
//
// ApplyMoves expects exclusive access to the given state while it's running.
// Don't read or write any part of the state structure until ApplyMoves returns.
func ApplyMoves(stmts []MoveStatement, state *states.State) map[addrs.UniqueKey]MoveResult {
	results := make(map[addrs.UniqueKey]MoveResult)

	// The methodology here is to construct a small graph of all of the move
	// statements where the edges represent where a particular statement
	// is either chained from or nested inside the effect of another statement.
	// That then means we can traverse the graph in topological sort order
	// to gradually move objects through potentially multiple moves each.

	g := buildMoveStatementGraph(stmts)

	// If there are any cycles in the graph then we'll not take any action
	// at all. The separate validation step should detect this and return
	// an error.
	if len(g.Cycles()) != 0 {
		return results
	}

	// The starting nodes are the ones that don't depend on any other nodes.
	startNodes := make(dag.Set, len(stmts))
	for _, v := range g.Vertices() {
		if len(g.UpEdges(v)) == 0 {
			startNodes.Add(v)
		}
	}

	g.DepthFirstWalk(startNodes, func(v dag.Vertex, depth int) error {
		stmt := v.(*MoveStatement)

		for _, ms := range state.Modules {
			modAddr := ms.Addr
			if !stmt.From.SelectsModule(modAddr) {
				continue
			}

			// We now know that the current module is relevant but what
			// we'll do with it depends on the object kind.
			switch kind := stmt.ObjectKind(); kind {
			case addrs.MoveEndpointModule:
				// For a module endpoint we just try the module address
				// directly.
				if newAddr, matches := modAddr.MoveDestination(stmt.From, stmt.To); matches {
					// We need to visit all of the resource instances in the
					// module and record them individually as results.
					for _, rs := range ms.Resources {
						relAddr := rs.Addr.Resource
						for key := range rs.Instances {
							oldInst := relAddr.Instance(key).Absolute(modAddr)
							newInst := relAddr.Instance(key).Absolute(newAddr)
							result := MoveResult{
								From: oldInst,
								To:   newInst,
							}
							results[oldInst.UniqueKey()] = result
							results[newInst.UniqueKey()] = result
						}
					}

					state.MoveModuleInstance(modAddr, newAddr)
					continue
				}
			case addrs.MoveEndpointResource:
				// For a resource endpoint we need to search each of the
				// resources and resource instances in the module.
				for _, rs := range ms.Resources {
					rAddr := rs.Addr
					if newAddr, matches := rAddr.MoveDestination(stmt.From, stmt.To); matches {
						for key := range rs.Instances {
							oldInst := rAddr.Instance(key)
							newInst := newAddr.Instance(key)
							result := MoveResult{
								From: oldInst,
								To:   newInst,
							}
							results[oldInst.UniqueKey()] = result
							results[newInst.UniqueKey()] = result
						}
						state.MoveAbsResource(rAddr, newAddr)
						continue
					}
					for key := range rs.Instances {
						iAddr := rAddr.Instance(key)
						if newAddr, matches := iAddr.MoveDestination(stmt.From, stmt.To); matches {
							result := MoveResult{From: iAddr, To: newAddr}
							results[iAddr.UniqueKey()] = result
							results[newAddr.UniqueKey()] = result

							state.MoveAbsResourceInstance(iAddr, newAddr)
							continue
						}
					}
				}
			default:
				panic(fmt.Sprintf("unhandled move object kind %s", kind))
			}
		}

		return nil
	})

	// FIXME: In the case of either chained or nested moves, "results" will
	// be left in a pretty interesting shape where the "old" address will
	// refer to a result that describes only the first step, while the "new"
	// address will refer to a result that describes only the last step.
	// To make that actually useful we'll need a different strategy where
	// the result describes the _effective_ source and destination, skipping
	// over any intermediate steps we took to get there, so that ultimately
	// we'll have enough information to annotate items in the plan with the
	// addresses the originally moved from.

	return results
}

// buildMoveStatementGraph constructs a dependency graph of the given move
// statements, where the nodes are all pointers to statements in the given
// slice and the edges represent either chaining or nesting relationships.
//
// buildMoveStatementGraph doesn't do any validation of the graph, so it
// may contain cycles and other sorts of invalidity.
func buildMoveStatementGraph(stmts []MoveStatement) *dag.AcyclicGraph {
	g := &dag.AcyclicGraph{}
	for i := range stmts {
		// The graph nodes are pointers to the actual statements directly.
		g.Add(&stmts[i])
	}

	// Now we'll add the edges representing chaining and nesting relationships.
	// We assume that a reasonable configuration will have at most tens of
	// move statements and thus this N*M algorithm is acceptable.
	for dependerI := range stmts {
		depender := &stmts[dependerI]
		for dependeeI := range stmts {
			dependee := &stmts[dependeeI]
			dependeeTo := dependee.To
			dependerFrom := depender.From
			if dependerFrom.CanChainFrom(dependeeTo) || dependerFrom.NestedWithin(dependeeTo) {
				g.Connect(dag.BasicEdge(depender, dependee))
			}
		}
	}

	return g
}
<commit_msg>correct the direction and walk order of the graph<commit_after>package refactoring

import (
	"fmt"

	"github.com/hashicorp/terraform/internal/addrs"
	"github.com/hashicorp/terraform/internal/dag"
	"github.com/hashicorp/terraform/internal/states"
)

type MoveResult struct {
	From, To addrs.AbsResourceInstance
}

// ApplyMoves modifies in-place the given state object so that any existing
// objects that are matched by a "from" argument of one of the move statements
// will be moved to instead appear at the "to" argument of that statement.
//
// The result is a map from the unique key of each absolute address that was
// either the source or destination of a move to a MoveResult describing
// what happened at that address.
//
// ApplyMoves does not have any error situations itself, and will instead just
// ignore any unresolvable move statements. Validation of a set of moves is
// a separate concern applied to the configuration, because validity of
// moves is always dependent only on the configuration, not on the state.
//
// ApplyMoves expects exclusive access to the given state while it's running.
// Don't read or write any part of the state structure until ApplyMoves returns.
func ApplyMoves(stmts []MoveStatement, state *states.State) map[addrs.UniqueKey]MoveResult {
	results := make(map[addrs.UniqueKey]MoveResult)

	// The methodology here is to construct a small graph of all of the move
	// statements where the edges represent where a particular statement
	// is either chained from or nested inside the effect of another statement.
	// That then means we can traverse the graph in topological sort order
	// to gradually move objects through potentially multiple moves each.

	g := buildMoveStatementGraph(stmts)

	// If there are any cycles in the graph then we'll not take any action
	// at all. The separate validation step should detect this and return
	// an error.
	if len(g.Cycles()) != 0 {
		return results
	}

	// The starting nodes are the ones that don't depend on any other nodes.
	startNodes := make(dag.Set, len(stmts))
	for _, v := range g.Vertices() {
		if len(g.DownEdges(v)) == 0 {
			startNodes.Add(v)
		}
	}

	g.ReverseDepthFirstWalk(startNodes, func(v dag.Vertex, depth int) error {
		stmt := v.(*MoveStatement)

		for _, ms := range state.Modules {
			modAddr := ms.Addr
			if !stmt.From.SelectsModule(modAddr) {
				continue
			}

			// We now know that the current module is relevant but what
			// we'll do with it depends on the object kind.
			switch kind := stmt.ObjectKind(); kind {
			case addrs.MoveEndpointModule:
				// For a module endpoint we just try the module address
				// directly.
				if newAddr, matches := modAddr.MoveDestination(stmt.From, stmt.To); matches {
					// We need to visit all of the resource instances in the
					// module and record them individually as results.
					for _, rs := range ms.Resources {
						relAddr := rs.Addr.Resource
						for key := range rs.Instances {
							oldInst := relAddr.Instance(key).Absolute(modAddr)
							newInst := relAddr.Instance(key).Absolute(newAddr)
							result := MoveResult{
								From: oldInst,
								To:   newInst,
							}
							results[oldInst.UniqueKey()] = result
							results[newInst.UniqueKey()] = result
						}
					}

					state.MoveModuleInstance(modAddr, newAddr)
					continue
				}
			case addrs.MoveEndpointResource:
				// For a resource endpoint we need to search each of the
				// resources and resource instances in the module.
				for _, rs := range ms.Resources {
					rAddr := rs.Addr
					if newAddr, matches := rAddr.MoveDestination(stmt.From, stmt.To); matches {
						for key := range rs.Instances {
							oldInst := rAddr.Instance(key)
							newInst := newAddr.Instance(key)
							result := MoveResult{
								From: oldInst,
								To:   newInst,
							}
							results[oldInst.UniqueKey()] = result
							results[newInst.UniqueKey()] = result
						}
						state.MoveAbsResource(rAddr, newAddr)
						continue
					}
					for key := range rs.Instances {
						iAddr := rAddr.Instance(key)
						if newAddr, matches := iAddr.MoveDestination(stmt.From, stmt.To); matches {
							result := MoveResult{From: iAddr, To: newAddr}
							results[iAddr.UniqueKey()] = result
							results[newAddr.UniqueKey()] = result

							state.MoveAbsResourceInstance(iAddr, newAddr)
							continue
						}
					}
				}
			default:
				panic(fmt.Sprintf("unhandled move object kind %s", kind))
			}
		}

		return nil
	})

	// FIXME: In the case of either chained or nested moves, "results" will
	// be left in a pretty interesting shape where the "old" address will
	// refer to a result that describes only the first step, while the "new"
	// address will refer to a result that describes only the last step.
	// To make that actually useful we'll need a different strategy where
	// the result describes the _effective_ source and destination, skipping
	// over any intermediate steps we took to get there, so that ultimately
	// we'll have enough information to annotate items in the plan with the
	// addresses the originally moved from.

	return results
}

// buildMoveStatementGraph constructs a dependency graph of the given move
// statements, where the nodes are all pointers to statements in the given
// slice and the edges represent either chaining or nesting relationships.
//
// buildMoveStatementGraph doesn't do any validation of the graph, so it
// may contain cycles and other sorts of invalidity.
func buildMoveStatementGraph(stmts []MoveStatement) *dag.AcyclicGraph {
	g := &dag.AcyclicGraph{}
	for i := range stmts {
		// The graph nodes are pointers to the actual statements directly.
		g.Add(&stmts[i])
	}

	// Now we'll add the edges representing chaining and nesting relationships.
	// We assume that a reasonable configuration will have at most tens of
	// move statements and thus this N*M algorithm is acceptable.
	for dependerI := range stmts {
		depender := &stmts[dependerI]
		for dependeeI := range stmts {
			dependee := &stmts[dependeeI]
			dependeeTo := dependee.To
			dependerFrom := depender.From
			if dependerFrom.CanChainFrom(dependeeTo) || dependerFrom.NestedWithin(dependeeTo) {
				g.Connect(dag.BasicEdge(depender, dependee))
			}
		}
	}

	return g
}
<|endoftext|>


<commit_before>package syscallcompat

import (
	"golang.org/x/sys/unix"
)

// Readlinkat exists both in Linux and in MacOS 10.10+. We may add an
// emulated version for users on older MacOS versions if there is
// demand.
// Buffer allocation is handled internally, unlike the bare unix.Readlinkat.
func Readlinkat(dirfd int, path string) (string, error) {
	// Allocate the buffer exponentially like os.Readlink does.
	for bufsz := 128; ; bufsz *= 2 {
		buf := make([]byte, bufsz)
		n, err := unix.Readlinkat(dirfd, path, buf)
		if err != nil {
			return "", err
		}
		if n < bufsz {
			return string(buf[0:n]), nil
		}
	}
}
<commit_msg>syscallcompat: add Faccessat<commit_after>package syscallcompat

import (
	"syscall"

	"golang.org/x/sys/unix"
)

// Readlinkat exists both in Linux and in MacOS 10.10+. We may add an
// emulated version for users on older MacOS versions if there is
// demand.
// Buffer allocation is handled internally, unlike the bare unix.Readlinkat.
func Readlinkat(dirfd int, path string) (string, error) {
	// Allocate the buffer exponentially like os.Readlink does.
	for bufsz := 128; ; bufsz *= 2 {
		buf := make([]byte, bufsz)
		n, err := unix.Readlinkat(dirfd, path, buf)
		if err != nil {
			return "", err
		}
		if n < bufsz {
			return string(buf[0:n]), nil
		}
	}
}

// Faccessat exists both in Linux and in MacOS 10.10+, but the Linux version
// DOES NOT support any flags. Emulate AT_SYMLINK_NOFOLLOW like glibc does.
func Faccessat(dirfd int, path string, mode uint32) error {
	var st unix.Stat_t
	err := Fstatat(dirfd, path, &st, unix.AT_SYMLINK_NOFOLLOW)
	if err != nil {
		return err
	}
	if st.Mode&syscall.S_IFMT == syscall.S_IFLNK {
		// Pretend that a symlink is always accessible
		return nil
	}
	return unix.Faccessat(dirfd, path, mode, 0)
}
<|endoftext|>


<commit_before>package scheduler

import (
	"github.com/zenoss/glog"
	coordclient "github.com/zenoss/serviced/coordinator/client"
	"github.com/zenoss/serviced/dao"
	"github.com/zenoss/serviced/datastore"
	"github.com/zenoss/serviced/facade"
	"github.com/zenoss/serviced/zzk"
)

type leaderFunc func(*facade.Facade, dao.ControlPlane, coordclient.Connection, <-chan coordclient.Event, string)

type scheduler struct {
	zkClient     *coordclient.Client // client from which connections can be created from
	cpDao        dao.ControlPlane    // ControlPlane interface
	cluster_path string              // path to the cluster node
	instance_id  string              // unique id for this node instance
	closing      chan chan error     // Sending a value on this channel notifies the schduler to shut down
	shutdown     chan error          // A error is placed on this channel when the scheduler shuts down
	started      bool                // is the loop running
	zkleaderFunc leaderFunc          // multiple implementations of leader function possible
	facade       *facade.Facade
}

func NewScheduler(cluster_path string, zkClient *coordclient.Client, instance_id string, cpDao dao.ControlPlane, facade *facade.Facade) (s *scheduler, shutdown <-chan error) {
	s = &scheduler{
		zkClient:     zkClient,
		cpDao:        cpDao,
		cluster_path: cluster_path,
		instance_id:  instance_id,
		closing:      make(chan chan error),
		shutdown:     make(chan error, 1),
		zkleaderFunc: Lead, // random scheduler implementation
		facade:       facade,
	}
	return s, s.shutdown
}

func (s *scheduler) Start() {
	if !s.started {
		s.started = true
		go s.loop()
	}
}

// Shut down node
func (s *scheduler) Stop() error {

	if !s.started {
		return nil
	}
	defer func() {
		s.started = false
	}()
	errc := make(chan error, 1)
	s.closing <- errc
	return <-errc
}

type hostNodeT struct {
	HostID  string
	version interface{}
}

func (h *hostNodeT) Version() interface{}           { return h.version }
func (h *hostNodeT) SetVersion(version interface{}) { h.version = version }

func (s *scheduler) loop() {
	glog.V(3).Infoln("entering scheduler")

	var err error
	//var this_node string
	defer func() {
		glog.V(3).Infoln("leaving scheduler")
		s.shutdown <- err
	}()

	allPools, err := s.facade.GetResourcePools(datastore.Get())
	if err != nil {
		glog.Error(err)
		return
	} else if allPools == nil || len(allPools) == 0 {
		glog.Error("no resource pools found")
		return
	}

	for _, aPool := range allPools {
		// TODO: Support non default pools
		// Currently, only the default pool gets a leader
		if aPool.ID != "default" {
			glog.Warningf("Non default pool: %v (not currently supported)", aPool.ID)
			continue
		}

		poolBasedConn, err := zzk.GetPoolBasedConnection(aPool.ID)
		if err != nil {
			glog.Error(err)
			return
		}

		hostNode := hostNodeT{HostID: s.instance_id}
		leader := poolBasedConn.NewLeader("/pools/"+aPool.ID+"/scheduler", &hostNode)
		events, err := leader.TakeLead()
		if err != nil {
			glog.Error("could not take lead: ", err)
			return
		}

		defer func() {
			leader.ReleaseLead()
		}()

		glog.Infof(" ********** Creating a leader for pool: %v --- %+v", aPool.ID, poolBasedConn)
		s.zkleaderFunc(s.facade, s.cpDao, poolBasedConn, events, aPool.ID)
	}
}
<commit_msg>updated comment<commit_after>package scheduler

import (
	"github.com/zenoss/glog"
	coordclient "github.com/zenoss/serviced/coordinator/client"
	"github.com/zenoss/serviced/dao"
	"github.com/zenoss/serviced/datastore"
	"github.com/zenoss/serviced/facade"
	"github.com/zenoss/serviced/zzk"
)

type leaderFunc func(*facade.Facade, dao.ControlPlane, coordclient.Connection, <-chan coordclient.Event, string)

type scheduler struct {
	zkClient     *coordclient.Client // client from which connections can be created from
	cpDao        dao.ControlPlane    // ControlPlane interface
	cluster_path string              // path to the cluster node
	instance_id  string              // unique id for this node instance
	closing      chan chan error     // Sending a value on this channel notifies the schduler to shut down
	shutdown     chan error          // A error is placed on this channel when the scheduler shuts down
	started      bool                // is the loop running
	zkleaderFunc leaderFunc          // multiple implementations of leader function possible
	facade       *facade.Facade
}

func NewScheduler(cluster_path string, zkClient *coordclient.Client, instance_id string, cpDao dao.ControlPlane, facade *facade.Facade) (s *scheduler, shutdown <-chan error) {
	s = &scheduler{
		zkClient:     zkClient,
		cpDao:        cpDao,
		cluster_path: cluster_path,
		instance_id:  instance_id,
		closing:      make(chan chan error),
		shutdown:     make(chan error, 1),
		zkleaderFunc: Lead, // random scheduler implementation
		facade:       facade,
	}
	return s, s.shutdown
}

func (s *scheduler) Start() {
	if !s.started {
		s.started = true
		go s.loop()
	}
}

// Shut down node
func (s *scheduler) Stop() error {

	if !s.started {
		return nil
	}
	defer func() {
		s.started = false
	}()
	errc := make(chan error, 1)
	s.closing <- errc
	return <-errc
}

type hostNodeT struct {
	HostID  string
	version interface{}
}

func (h *hostNodeT) Version() interface{}           { return h.version }
func (h *hostNodeT) SetVersion(version interface{}) { h.version = version }

func (s *scheduler) loop() {
	glog.V(3).Infoln("entering scheduler")

	var err error
	//var this_node string
	defer func() {
		glog.V(3).Infoln("leaving scheduler")
		s.shutdown <- err
	}()

	allPools, err := s.facade.GetResourcePools(datastore.Get())
	if err != nil {
		glog.Error(err)
		return
	} else if allPools == nil || len(allPools) == 0 {
		glog.Error("no resource pools found")
		return
	}

	// CLARK TODO
	// instead of looping through the pools, add a watch on /pools ... start/stop schedulers per pool
	for _, aPool := range allPools {
		// TODO: Support non default pools
		// Currently, only the default pool gets a leader
		if aPool.ID != "default" {
			glog.Warningf("Non default pool: %v (not currently supported)", aPool.ID)
			continue
		}

		poolBasedConn, err := zzk.GetPoolBasedConnection(aPool.ID)
		if err != nil {
			glog.Error(err)
			return
		}

		hostNode := hostNodeT{HostID: s.instance_id}
		leader := poolBasedConn.NewLeader("/pools/"+aPool.ID+"/scheduler", &hostNode)
		events, err := leader.TakeLead()
		if err != nil {
			glog.Error("could not take lead: ", err)
			return
		}

		defer func() {
			leader.ReleaseLead()
		}()

		glog.Infof(" ********** Creating a leader for pool: %v --- %+v", aPool.ID, poolBasedConn)
		s.zkleaderFunc(s.facade, s.cpDao, poolBasedConn, events, aPool.ID)
	}
}
<|endoftext|>


<commit_before>// affinity assigns issues based on team mentions and those team captains.
// The idea is to separate the work of triaging of issues and pull requests
// out to a larger pool of people to make it less of a burden to be involved.
package affinity

import (
	"fmt"
	"strings"

	"github.com/google/go-github/github"
	"github.com/parkr/auto-reply/ctx"
)

var explanation = `We are utilizing a new workflow in our issues and pull requests. Affinity teams have been setup to allow community members to hear about pull requests that may be interesting to them. When a new issue or pull request comes in, we are asking that the author mention the appropriate affinity team. I then assign a random "team captain" or two to the issue who is in charge of triaging it until it is closed or passing it off to another captain. In order to move forward with this new workflow, we need to know: which of the following teams best fits your issue or contribution?`

func assignTeamCaptains(context *ctx.Context, handler Handler, body string, assigneeCount int) error {
	if context.Issue.IsEmpty() {
		context.IncrStat("affinity.error.no_ref")
		return context.NewError("assignTeamCaptains: issue reference was not set; bailing")
	}

	team, err := findAffinityTeam(body, handler.teams)
	if err != nil {
		context.IncrStat("affinity.error.no_team")
		//return askForAffinityTeam(context, handler.teams)
		return context.NewError("%s: no team in the message body; unable to assign", context.Issue)
	}

	context.Log("team: %s", team)
	victims := team.RandomCaptainLoginsExcluding(context.Issue.Author, assigneeCount)
	if len(victims) == 0 {
		context.IncrStat("affinity.error.no_acceptable_captains")
		return context.NewError("%s: team captains other than issue author could not be found", context.Issue)
	}
	context.Log("selected affinity team captains for %s: %q", context.Issue, victims)
	_, _, err = context.GitHub.Issues.AddAssignees(
		context.Issue.Owner,
		context.Issue.Repo,
		context.Issue.Num,
		victims,
	)
	if err != nil {
		context.IncrStat("affinity.error.github_api")
		return context.NewError("assignTeamCaptains: problem assigning: %v", err)
	}

	context.IncrStat("affinity.success")
	context.Log("assignTeamCaptains: assigned %q to %s", victims, context.Issue)
	return nil
}

func findAffinityTeam(body string, allTeams []Team) (Team, error) {
	for _, team := range allTeams {
		if strings.Contains(body, team.Mention) {
			return team, nil
		}
	}
	return Team{}, fmt.Errorf("findAffinityTeam: no matching team")
}

func askForAffinityTeam(context *ctx.Context, allTeams []Team) error {
	_, _, err := context.GitHub.Issues.CreateComment(
		context.Issue.Owner,
		context.Issue.Repo,
		context.Issue.Num,
		&github.IssueComment{Body: github.String(buildAffinityTeamMessage(context, allTeams))},
	)
	if err != nil {
		return context.NewError("askForAffinityTeam: could not leave comment: %v", err)
	}
	return nil
}

func buildAffinityTeamMessage(context *ctx.Context, allTeams []Team) string {
	var prefix string
	if context.Issue.Author != "" {
		prefix = fmt.Sprintf("Hey, @%s!", context.Issue.Author)
	} else {
		prefix = "Hello!"
	}

	teams := []string{}
	for _, team := range allTeams {
		teams = append(teams, fmt.Sprintf(
			"- `%s` – %s",
			team.Mention, team.Description,
		))
	}

	return fmt.Sprintf(
		"%s %s\n\n%s\n\nMention one of these teams in a comment below and we'll get this sorted. Thanks!",
		prefix, explanation, strings.Join(teams, "\n"),
	)
}

func usersByLogin(users []*github.User) []string {
	logins := []string{}
	for _, user := range users {
		logins = append(logins, *user.Login)
	}
	return logins
}
<commit_msg>affinity: log issue author<commit_after>// affinity assigns issues based on team mentions and those team captains.
// The idea is to separate the work of triaging of issues and pull requests
// out to a larger pool of people to make it less of a burden to be involved.
package affinity

import (
	"fmt"
	"strings"

	"github.com/google/go-github/github"
	"github.com/parkr/auto-reply/ctx"
)

var explanation = `We are utilizing a new workflow in our issues and pull requests. Affinity teams have been setup to allow community members to hear about pull requests that may be interesting to them. When a new issue or pull request comes in, we are asking that the author mention the appropriate affinity team. I then assign a random "team captain" or two to the issue who is in charge of triaging it until it is closed or passing it off to another captain. In order to move forward with this new workflow, we need to know: which of the following teams best fits your issue or contribution?`

func assignTeamCaptains(context *ctx.Context, handler Handler, body string, assigneeCount int) error {
	if context.Issue.IsEmpty() {
		context.IncrStat("affinity.error.no_ref")
		return context.NewError("assignTeamCaptains: issue reference was not set; bailing")
	}

	team, err := findAffinityTeam(body, handler.teams)
	if err != nil {
		context.IncrStat("affinity.error.no_team")
		//return askForAffinityTeam(context, handler.teams)
		return context.NewError("%s: no team in the message body; unable to assign", context.Issue)
	}

	context.Log("team: %s, excluding: %s", team, context.Issue.Author)
	victims := team.RandomCaptainLoginsExcluding(context.Issue.Author, assigneeCount)
	if len(victims) == 0 {
		context.IncrStat("affinity.error.no_acceptable_captains")
		return context.NewError("%s: team captains other than issue author could not be found", context.Issue)
	}
	context.Log("selected affinity team captains for %s: %q", context.Issue, victims)
	_, _, err = context.GitHub.Issues.AddAssignees(
		context.Issue.Owner,
		context.Issue.Repo,
		context.Issue.Num,
		victims,
	)
	if err != nil {
		context.IncrStat("affinity.error.github_api")
		return context.NewError("assignTeamCaptains: problem assigning: %v", err)
	}

	context.IncrStat("affinity.success")
	context.Log("assignTeamCaptains: assigned %q to %s", victims, context.Issue)
	return nil
}

func findAffinityTeam(body string, allTeams []Team) (Team, error) {
	for _, team := range allTeams {
		if strings.Contains(body, team.Mention) {
			return team, nil
		}
	}
	return Team{}, fmt.Errorf("findAffinityTeam: no matching team")
}

func askForAffinityTeam(context *ctx.Context, allTeams []Team) error {
	_, _, err := context.GitHub.Issues.CreateComment(
		context.Issue.Owner,
		context.Issue.Repo,
		context.Issue.Num,
		&github.IssueComment{Body: github.String(buildAffinityTeamMessage(context, allTeams))},
	)
	if err != nil {
		return context.NewError("askForAffinityTeam: could not leave comment: %v", err)
	}
	return nil
}

func buildAffinityTeamMessage(context *ctx.Context, allTeams []Team) string {
	var prefix string
	if context.Issue.Author != "" {
		prefix = fmt.Sprintf("Hey, @%s!", context.Issue.Author)
	} else {
		prefix = "Hello!"
	}

	teams := []string{}
	for _, team := range allTeams {
		teams = append(teams, fmt.Sprintf(
			"- `%s` – %s",
			team.Mention, team.Description,
		))
	}

	return fmt.Sprintf(
		"%s %s\n\n%s\n\nMention one of these teams in a comment below and we'll get this sorted. Thanks!",
		prefix, explanation, strings.Join(teams, "\n"),
	)
}

func usersByLogin(users []*github.User) []string {
	logins := []string{}
	for _, user := range users {
		logins = append(logins, *user.Login)
	}
	return logins
}
<|endoftext|>