<commit_before>package deploy

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/remind101/deploy/Godeps/_workspace/src/github.com/codegangsta/cli"
	"github.com/remind101/deploy/Godeps/_workspace/src/github.com/github/hub/git"
	hub "github.com/remind101/deploy/Godeps/_workspace/src/github.com/github/hub/github"
	"github.com/remind101/deploy/Godeps/_workspace/src/github.com/google/go-github/github"
)

const (
	Name  = "deploy"
	Usage = "A command for creating GitHub deployments"
)

const (
	DefaultRef     = "master"
	DefaultTimeout = 20 * time.Second
)

var errTimeout = errors.New("Timed out waiting for build to start. Did you add a webhook to handle deployment events?")

func init() {
	cli.AppHelpTemplate = `USAGE:
   # Deploy the master branch of remind101/acme-inc to staging
   {{.Name}} --env=staging --ref=master remind101/acme-inc

   # Deploy HEAD of the current branch to staging
   {{.Name}} --env=staging remind101/acme-inc

   # Deploy the current GitHub repo to staging
   {{.Name}} --env=staging
{{if .Flags}}
OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}{{end}}
`
}

var flags = []cli.Flag{
	cli.StringFlag{
		Name:  "ref, branch, commit, tag",
		Value: "",
		Usage: "The git ref to deploy. Can be a git commit, branch or tag.",
	},
	cli.StringFlag{
		Name:  "env, e",
		Value: "",
		Usage: "The environment to deploy to.",
	},
	cli.BoolFlag{
		Name:  "force, f",
		Usage: "Ignore commit status checks.",
	},
	cli.BoolFlag{
		Name:  "detached, d",
		Usage: "Don't wait for the deployment to complete.",
	},
	cli.BoolFlag{
		Name:  "quiet, q",
		Usage: "Silence any output to STDOUT.",
	},
	cli.BoolFlag{
		Name:  "update, u",
		Usage: "Update the binary",
	},
}

var ProtectedEnvironments = map[string]bool{
	"production": true,
	"prod":       true,
}

// NewApp returns a new cli.App for the deploy command.
func NewApp() *cli.App {
	app := cli.NewApp()
	app.Version = Version
	app.Name = Name
	app.Usage = Usage
	app.Flags = flags
	app.Action = func(c *cli.Context) {
		if c.Bool("update") {
			updater := NewUpdater()
			if err := updater.Update(); err != nil {
				fmt.Println(err)
				os.Exit(-1)
			} else {
				os.Exit(0)
			}
		}

		if err := RunDeploy(c); err != nil {
			msg := err.Error()
			if err, ok := err.(*github.ErrorResponse); ok {
				if strings.HasPrefix(err.Message, "Conflict: Commit status checks failed for") {
					msg = "Commit status checks failed. You can bypass commit status checks with the --force flag."
				} else if strings.HasPrefix(err.Message, "No ref found for") {
					msg = fmt.Sprintf("%s. Did you push it to GitHub?", err.Message)
				} else {
					msg = err.Message
				}
			}

			fmt.Println(msg)
			os.Exit(-1)
		}
	}

	return app
}

// RunDeploy performs a deploy.
func RunDeploy(c *cli.Context) error {
	var w io.Writer
	if c.Bool("quiet") {
		w = ioutil.Discard
	} else {
		w = c.App.Writer
	}

	h, err := hub.CurrentConfig().PromptForHost("github.com")
	if err != nil {
		return err
	}

	client, err := newGitHubClient(h)
	if err != nil {
		return err
	}

	nwo, err := Repo(c.Args())
	if err != nil {
		return err
	}

	owner, repo, err := SplitRepo(nwo, os.Getenv("GITHUB_ORGANIZATION"))
	if err != nil {
		return fmt.Errorf("Invalid GitHub repo: %s", nwo)
	}

	err = displayNewCommits(owner, repo, c, client)
	if err != nil {
		return err
	}

	r, err := newDeploymentRequest(c)
	if err != nil {
		return err
	}

	fmt.Fprintf(w, "Deploying %s/%s@%s to %s...\n", owner, repo, *r.Ref, *r.Environment)

	if c.Bool("detached") {
		return nil
	}

	d, _, err := client.Repositories.CreateDeployment(owner, repo, r)
	if err != nil {
		return err
	}

	started := make(chan *github.DeploymentStatus)
	completed := make(chan *github.DeploymentStatus)

	go func() {
		started <- waitState(pendingStates, owner, repo, *d.ID, client)
	}()

	go func() {
		completed <- waitState(completedStates, owner, repo, *d.ID, client)
	}()

	select {
	case <-time.After(DefaultTimeout):
		return errTimeout
	case status := <-started:
		var url string
		if status.TargetURL != nil {
			url = *status.TargetURL
		}
		fmt.Fprintf(w, "%s\n", url)
	}

	status := <-completed

	if isFailed(*status.State) {
		return errors.New("Failed to deploy")
	}

	return nil
}

func displayNewCommits(owner string, repo string, c *cli.Context, client *github.Client) error {
	ref := Ref(c.String("ref"), git.Head)

	opt := &github.DeploymentsListOptions{
		Environment: c.String("env"),
	}

	deployments, _, err := client.Repositories.ListDeployments(owner, repo, opt)
	if err != nil || len(deployments) == 0 {
		return err
	}

	sha := *deployments[0].SHA
	compare, _, err := client.Repositories.CompareCommits(owner, repo, sha, ref)
	if err != nil || len(compare.Commits) == 0 {
		return err
	}

	fmt.Println("Deploying the following commits:\n")
	for _, commit := range compare.Commits {
		message := *commit.Commit.Message
		fmt.Printf("%-20s\t%s\n", *commit.Commit.Author.Name, strings.Split(message, "\n")[0])
	}
	fmt.Printf("\nSee entire diff here: https://github.com/%s/%s/compare/%s...%s\n\n", owner, repo, sha, ref)
	return nil
}

func newDeploymentRequest(c *cli.Context) (*github.DeploymentRequest, error) {
	ref := Ref(c.String("ref"), git.Head)

	env := c.String("env")
	if env == "" {
		return nil, fmt.Errorf("--env flag is required")
	}

	if ProtectedEnvironments[env] {
		yes := askYN(fmt.Sprintf("Are you sure you want to deploy %s to %s?", ref, env))
		if !yes {
			return nil, fmt.Errorf("Deployment aborted.")
		}
	}

	var contexts *[]string
	if c.Bool("force") {
		s := []string{}
		contexts = &s
	}

	return &github.DeploymentRequest{
		Ref:              github.String(ref),
		Task:             github.String("deploy"),
		AutoMerge:        github.Bool(false),
		Environment:      github.String(env),
		RequiredContexts: contexts,
		Payload: map[string]interface{}{
			"force": c.Bool("force"),
		},
		// TODO Description:
	}, nil
}

var (
	pendingStates   = []string{"pending"}
	completedStates = []string{"success", "error", "failure"}
)

func isFailed(state string) bool {
	return state == "error" || state == "failure"
}

// waitState waits for a deployment status that matches the given states, then
// sends on the returned channel.
func waitState(states []string, owner, repo string, deploymentID int, c *github.Client) *github.DeploymentStatus {
	for {
		<-time.After(1 * time.Second)

		statuses, _, err := c.Repositories.ListDeploymentStatuses(owner, repo, deploymentID, nil)
		if err != nil {
			continue
		}

		status := firstStatus(states, statuses)
		if status != nil {
			return status
		}
	}
}

// firstStatus takes a slice of github.DeploymentStatus and returns the
// first status that matches the provided slice of states.
func firstStatus(states []string, statuses []github.DeploymentStatus) *github.DeploymentStatus {
	for _, ds := range statuses {
		for _, s := range states {
			if ds.State != nil && *ds.State == s {
				return &ds
			}
		}
	}

	return nil
}

// refRegex is a regular expression that matches a full git HEAD ref.
var refRegex = regexp.MustCompile(`^refs/heads/(.*)$`)

// Ref attempts to return the proper git ref to deploy. If a ref is provided,
// that will be returned. If not, it will fallback to calling headFunc. If an
// error is returned (not in a git repo), then it will fallback to DefaultRef.
func Ref(ref string, headFunc func() (string, error)) string {
	if ref != "" {
		return ref
	}

	ref, err := headFunc()
	if err != nil {
		// An error means that we're either not in a GitRepo or we're
		// not on a branch. In this case, we just fallback to the
		// DefaultRef.
		return DefaultRef
	}

	// Convert `refs/heads/test-deploy` => `test-deploy`
	return refRegex.ReplaceAllString(ref, "$1")
}

// Repo will determine the correct GitHub repo to deploy to, based on a set of
// arguments.
func Repo(arguments []string) (string, error) {
	if len(arguments) != 0 {
		return arguments[0], nil
	}

	remotes, err := hub.Remotes()
	if err != nil {
		return "", err
	}

	repo := GitHubRepo(remotes)
	if repo == "" {
		return repo, errors.New("no GitHub repo found in .git/config")
	}

	return repo, nil
}

// A regular expression that can convert a URL.Path into a GitHub repo name.
var remoteRegex = regexp.MustCompile(`^/(.*)\.git$`)

// GitHubRepo, given a list of git remotes, will determine what the GitHub repo
// is.
func GitHubRepo(remotes []hub.Remote) string {
	// We only want to look at the `origin` remote.
	remote := findRemote("origin", remotes)
	if remote == nil {
		return ""
	}

	// Remotes that are not pointed at a GitHub repo are not valid.
	if remote.URL.Host != "github.com" {
		return ""
	}

	// Convert `/remind101/acme-inc.git` => `remind101/acme-inc`.
	return remoteRegex.ReplaceAllString(remote.URL.Path, "$1")
}

func findRemote(name string, remotes []hub.Remote) *hub.Remote {
	for _, r := range remotes {
		if r.Name == name {
			return &r
		}
	}

	return nil
}

var errInvalidRepo = errors.New("invalid repo")

// SplitRepo splits a repo string in the form remind101/acme-inc into it's owner
// and repo components.
func SplitRepo(nwo, defaultOrg string) (owner string, repo string, err error) {
	parts := strings.Split(nwo, "/")

	// If we were only given a repo name, and a default organization is set,
	// we'll use the defaultOrg as the owner.
	if len(parts) == 1 && defaultOrg != "" && parts[0] != "" {
		owner = defaultOrg
		repo = parts[0]
		return
	}

	if len(parts) != 2 {
		err = errInvalidRepo
		return
	}

	owner = parts[0]
	repo = parts[1]

	return
}

func askYN(prompt string) bool {
	r := bufio.NewReader(os.Stdin)
	fmt.Printf("%s (y/N)\n", prompt)
	a, _ := r.ReadString('\n')
	return strings.ToUpper(a) == "Y\n"
}
<commit_msg>clean up error handling code to be clearer<commit_after>package deploy

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/remind101/deploy/Godeps/_workspace/src/github.com/codegangsta/cli"
	"github.com/remind101/deploy/Godeps/_workspace/src/github.com/github/hub/git"
	hub "github.com/remind101/deploy/Godeps/_workspace/src/github.com/github/hub/github"
	"github.com/remind101/deploy/Godeps/_workspace/src/github.com/google/go-github/github"
)

const (
	Name  = "deploy"
	Usage = "A command for creating GitHub deployments"
)

const (
	DefaultRef     = "master"
	DefaultTimeout = 20 * time.Second
)

var errTimeout = errors.New("Timed out waiting for build to start. Did you add a webhook to handle deployment events?")

func init() {
	cli.AppHelpTemplate = `USAGE:
   # Deploy the master branch of remind101/acme-inc to staging
   {{.Name}} --env=staging --ref=master remind101/acme-inc

   # Deploy HEAD of the current branch to staging
   {{.Name}} --env=staging remind101/acme-inc

   # Deploy the current GitHub repo to staging
   {{.Name}} --env=staging
{{if .Flags}}
OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}{{end}}
`
}

var flags = []cli.Flag{
	cli.StringFlag{
		Name:  "ref, branch, commit, tag",
		Value: "",
		Usage: "The git ref to deploy. Can be a git commit, branch or tag.",
	},
	cli.StringFlag{
		Name:  "env, e",
		Value: "",
		Usage: "The environment to deploy to.",
	},
	cli.BoolFlag{
		Name:  "force, f",
		Usage: "Ignore commit status checks.",
	},
	cli.BoolFlag{
		Name:  "detached, d",
		Usage: "Don't wait for the deployment to complete.",
	},
	cli.BoolFlag{
		Name:  "quiet, q",
		Usage: "Silence any output to STDOUT.",
	},
	cli.BoolFlag{
		Name:  "update, u",
		Usage: "Update the binary",
	},
}

var ProtectedEnvironments = map[string]bool{
	"production": true,
	"prod":       true,
}

// NewApp returns a new cli.App for the deploy command.
func NewApp() *cli.App {
	app := cli.NewApp()
	app.Version = Version
	app.Name = Name
	app.Usage = Usage
	app.Flags = flags
	app.Action = func(c *cli.Context) {
		if c.Bool("update") {
			updater := NewUpdater()
			if err := updater.Update(); err != nil {
				fmt.Println(err)
				os.Exit(-1)
			} else {
				os.Exit(0)
			}
		}

		if err := RunDeploy(c); err != nil {
			msg := err.Error()
			if err, ok := err.(*github.ErrorResponse); ok {
				if strings.HasPrefix(err.Message, "Conflict: Commit status checks failed for") {
					msg = "Commit status checks failed. You can bypass commit status checks with the --force flag."
				} else if strings.HasPrefix(err.Message, "No ref found for") {
					msg = fmt.Sprintf("%s. Did you push it to GitHub?", err.Message)
				} else {
					msg = err.Message
				}
			}

			fmt.Println(msg)
			os.Exit(-1)
		}
	}

	return app
}

// RunDeploy performs a deploy.
func RunDeploy(c *cli.Context) error {
	var w io.Writer
	if c.Bool("quiet") {
		w = ioutil.Discard
	} else {
		w = c.App.Writer
	}

	h, err := hub.CurrentConfig().PromptForHost("github.com")
	if err != nil {
		return err
	}

	client, err := newGitHubClient(h)
	if err != nil {
		return err
	}

	nwo, err := Repo(c.Args())
	if err != nil {
		return err
	}

	owner, repo, err := SplitRepo(nwo, os.Getenv("GITHUB_ORGANIZATION"))
	if err != nil {
		return fmt.Errorf("Invalid GitHub repo: %s", nwo)
	}

	err = displayNewCommits(owner, repo, c, client)
	if err != nil {
		return err
	}

	r, err := newDeploymentRequest(c)
	if err != nil {
		return err
	}

	fmt.Fprintf(w, "Deploying %s/%s@%s to %s...\n", owner, repo, *r.Ref, *r.Environment)

	if c.Bool("detached") {
		return nil
	}

	d, _, err := client.Repositories.CreateDeployment(owner, repo, r)
	if err != nil {
		return err
	}

	started := make(chan *github.DeploymentStatus)
	completed := make(chan *github.DeploymentStatus)

	go func() {
		started <- waitState(pendingStates, owner, repo, *d.ID, client)
	}()

	go func() {
		completed <- waitState(completedStates, owner, repo, *d.ID, client)
	}()

	select {
	case <-time.After(DefaultTimeout):
		return errTimeout
	case status := <-started:
		var url string
		if status.TargetURL != nil {
			url = *status.TargetURL
		}
		fmt.Fprintf(w, "%s\n", url)
	}

	status := <-completed

	if isFailed(*status.State) {
		return errors.New("Failed to deploy")
	}

	return nil
}

func displayNewCommits(owner string, repo string, c *cli.Context, client *github.Client) error {
	ref := Ref(c.String("ref"), git.Head)

	opt := &github.DeploymentsListOptions{
		Environment: c.String("env"),
	}

	deployments, _, err := client.Repositories.ListDeployments(owner, repo, opt)
	if err != nil {
		return err
	} else if len(deployments) == 0 {
		return nil
	}

	sha := *deployments[0].SHA
	compare, _, err := client.Repositories.CompareCommits(owner, repo, sha, ref)
	if err != nil {
		return err
	} else if len(compare.Commits) == 0 {
		return nil
	}

	fmt.Println("Deploying the following commits:\n")
	for _, commit := range compare.Commits {
		message := *commit.Commit.Message
		fmt.Printf("%-20s\t%s\n", *commit.Commit.Author.Name, strings.Split(message, "\n")[0])
	}
	fmt.Printf("\nSee entire diff here: https://github.com/%s/%s/compare/%s...%s\n\n", owner, repo, sha, ref)
	return nil
}

func newDeploymentRequest(c *cli.Context) (*github.DeploymentRequest, error) {
	ref := Ref(c.String("ref"), git.Head)

	env := c.String("env")
	if env == "" {
		return nil, fmt.Errorf("--env flag is required")
	}

	if ProtectedEnvironments[env] {
		yes := askYN(fmt.Sprintf("Are you sure you want to deploy %s to %s?", ref, env))
		if !yes {
			return nil, fmt.Errorf("Deployment aborted.")
		}
	}

	var contexts *[]string
	if c.Bool("force") {
		s := []string{}
		contexts = &s
	}

	return &github.DeploymentRequest{
		Ref:              github.String(ref),
		Task:             github.String("deploy"),
		AutoMerge:        github.Bool(false),
		Environment:      github.String(env),
		RequiredContexts: contexts,
		Payload: map[string]interface{}{
			"force": c.Bool("force"),
		},
		// TODO Description:
	}, nil
}

var (
	pendingStates   = []string{"pending"}
	completedStates = []string{"success", "error", "failure"}
)

func isFailed(state string) bool {
	return state == "error" || state == "failure"
}

// waitState waits for a deployment status that matches the given states, then
// sends on the returned channel.
func waitState(states []string, owner, repo string, deploymentID int, c *github.Client) *github.DeploymentStatus {
	for {
		<-time.After(1 * time.Second)

		statuses, _, err := c.Repositories.ListDeploymentStatuses(owner, repo, deploymentID, nil)
		if err != nil {
			continue
		}

		status := firstStatus(states, statuses)
		if status != nil {
			return status
		}
	}
}

// firstStatus takes a slice of github.DeploymentStatus and returns the
// first status that matches the provided slice of states.
func firstStatus(states []string, statuses []github.DeploymentStatus) *github.DeploymentStatus {
	for _, ds := range statuses {
		for _, s := range states {
			if ds.State != nil && *ds.State == s {
				return &ds
			}
		}
	}

	return nil
}

// refRegex is a regular expression that matches a full git HEAD ref.
var refRegex = regexp.MustCompile(`^refs/heads/(.*)$`)

// Ref attempts to return the proper git ref to deploy. If a ref is provided,
// that will be returned. If not, it will fallback to calling headFunc. If an
// error is returned (not in a git repo), then it will fallback to DefaultRef.
func Ref(ref string, headFunc func() (string, error)) string {
	if ref != "" {
		return ref
	}

	ref, err := headFunc()
	if err != nil {
		// An error means that we're either not in a GitRepo or we're
		// not on a branch. In this case, we just fallback to the
		// DefaultRef.
		return DefaultRef
	}

	// Convert `refs/heads/test-deploy` => `test-deploy`
	return refRegex.ReplaceAllString(ref, "$1")
}

// Repo will determine the correct GitHub repo to deploy to, based on a set of
// arguments.
func Repo(arguments []string) (string, error) {
	if len(arguments) != 0 {
		return arguments[0], nil
	}

	remotes, err := hub.Remotes()
	if err != nil {
		return "", err
	}

	repo := GitHubRepo(remotes)
	if repo == "" {
		return repo, errors.New("no GitHub repo found in .git/config")
	}

	return repo, nil
}

// A regular expression that can convert a URL.Path into a GitHub repo name.
var remoteRegex = regexp.MustCompile(`^/(.*)\.git$`)

// GitHubRepo, given a list of git remotes, will determine what the GitHub repo
// is.
func GitHubRepo(remotes []hub.Remote) string {
	// We only want to look at the `origin` remote.
	remote := findRemote("origin", remotes)
	if remote == nil {
		return ""
	}

	// Remotes that are not pointed at a GitHub repo are not valid.
	if remote.URL.Host != "github.com" {
		return ""
	}

	// Convert `/remind101/acme-inc.git` => `remind101/acme-inc`.
	return remoteRegex.ReplaceAllString(remote.URL.Path, "$1")
}

func findRemote(name string, remotes []hub.Remote) *hub.Remote {
	for _, r := range remotes {
		if r.Name == name {
			return &r
		}
	}

	return nil
}

var errInvalidRepo = errors.New("invalid repo")

// SplitRepo splits a repo string in the form remind101/acme-inc into it's owner
// and repo components.
func SplitRepo(nwo, defaultOrg string) (owner string, repo string, err error) {
	parts := strings.Split(nwo, "/")

	// If we were only given a repo name, and a default organization is set,
	// we'll use the defaultOrg as the owner.
	if len(parts) == 1 && defaultOrg != "" && parts[0] != "" {
		owner = defaultOrg
		repo = parts[0]
		return
	}

	if len(parts) != 2 {
		err = errInvalidRepo
		return
	}

	owner = parts[0]
	repo = parts[1]

	return
}

func askYN(prompt string) bool {
	r := bufio.NewReader(os.Stdin)
	fmt.Printf("%s (y/N)\n", prompt)
	a, _ := r.ReadString('\n')
	return strings.ToUpper(a) == "Y\n"
}
<|endoftext|>


<commit_before>package cli

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/kopia/kopia/fs"
	"github.com/kopia/kopia/repo"
	"github.com/kopia/kopia/snapshot"

	kingpin "gopkg.in/alecthomas/kingpin.v2"
)

var (
	lsCommand = app.Command("list", "List a directory stored in repository object.").Alias("ls")

	lsCommandLong      = lsCommand.Flag("long", "Long output").Short('l').Bool()
	lsCommandRecursive = lsCommand.Flag("recursive", "Recursive output").Short('r').Bool()
	lsCommandShowOID   = lsCommand.Flag("show-object-id", "Show object IDs").Short('o').Bool()
	lsCommandPath      = lsCommand.Arg("path", "Path").Required().String()
)

func runLSCommand(context *kingpin.ParseContext) error {
	rep := mustOpenRepository(nil)
	defer rep.Close()

	mgr := snapshot.NewManager(rep)

	oid, err := parseObjectID(mgr, *lsCommandPath)
	if err != nil {
		return err
	}

	var prefix string
	if !*lsCommandLong {
		prefix = *lsCommandPath
		if !strings.HasSuffix(prefix, "/") {
			prefix += "/"
		}
	}

	return listDirectory(mgr, prefix, oid, "")
}

func init() {
	lsCommand.Action(runLSCommand)
}

func listDirectory(mgr *snapshot.Manager, prefix string, oid repo.ObjectID, indent string) error {
	d := mgr.DirectoryEntry(oid)

	entries, err := d.Readdir()
	if err != nil {
		return err
	}

	maxNameLen := 20
	for _, e := range entries {
		if l := len(nameToDisplay(prefix, e.Metadata())); l > maxNameLen {
			maxNameLen = l
		}
	}

	maxNameLenString := strconv.Itoa(maxNameLen)

	for _, e := range entries {
		m := e.Metadata()
		var info string
		objectID := e.(repo.HasObjectID).ObjectID()
		var oid string
		if objectID.BinaryContent != nil {
			oid = "<inline binary content>"
		} else if objectID.TextContent != "" {
			oid = "<inline text content>"
		} else {
			oid = objectID.String()
		}
		if *lsCommandLong {
			info = fmt.Sprintf(
				"%v %9d %v %-"+maxNameLenString+"s %v",
				m.FileMode(),
				m.FileSize,
				m.ModTime.Local().Format("02 Jan 06 15:04:05"),
				nameToDisplay(prefix, m),
				oid,
			)
		} else if *lsCommandShowOID {
			info = fmt.Sprintf(
				"%v %v",
				nameToDisplay(prefix, m),
				oid)
		} else {
			info = nameToDisplay(prefix, m)
		}
		fmt.Println(info)
		if *lsCommandRecursive && m.FileMode().IsDir() {
			listDirectory(mgr, prefix+m.Name+"/", objectID, indent+"  ")
		}
	}

	return nil
}

func nameToDisplay(prefix string, md *fs.EntryMetadata) string {
	suffix := ""
	if md.FileMode().IsDir() {
		suffix = "/"

	}
	if *lsCommandLong || *lsCommandRecursive {
		return prefix + md.Name + suffix
	}

	return md.Name
}
<commit_msg>extracted timeFormat constant for printing dates in CLI<commit_after>package cli

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/kopia/kopia/fs"
	"github.com/kopia/kopia/repo"
	"github.com/kopia/kopia/snapshot"

	kingpin "gopkg.in/alecthomas/kingpin.v2"
)

const timeFormat = "02 Jan 06 15:04:05"

var (
	lsCommand = app.Command("list", "List a directory stored in repository object.").Alias("ls")

	lsCommandLong      = lsCommand.Flag("long", "Long output").Short('l').Bool()
	lsCommandRecursive = lsCommand.Flag("recursive", "Recursive output").Short('r').Bool()
	lsCommandShowOID   = lsCommand.Flag("show-object-id", "Show object IDs").Short('o').Bool()
	lsCommandPath      = lsCommand.Arg("path", "Path").Required().String()
)

func runLSCommand(context *kingpin.ParseContext) error {
	rep := mustOpenRepository(nil)
	defer rep.Close()

	mgr := snapshot.NewManager(rep)

	oid, err := parseObjectID(mgr, *lsCommandPath)
	if err != nil {
		return err
	}

	var prefix string
	if !*lsCommandLong {
		prefix = *lsCommandPath
		if !strings.HasSuffix(prefix, "/") {
			prefix += "/"
		}
	}

	return listDirectory(mgr, prefix, oid, "")
}

func init() {
	lsCommand.Action(runLSCommand)
}

func listDirectory(mgr *snapshot.Manager, prefix string, oid repo.ObjectID, indent string) error {
	d := mgr.DirectoryEntry(oid)

	entries, err := d.Readdir()
	if err != nil {
		return err
	}

	maxNameLen := 20
	for _, e := range entries {
		if l := len(nameToDisplay(prefix, e.Metadata())); l > maxNameLen {
			maxNameLen = l
		}
	}

	maxNameLenString := strconv.Itoa(maxNameLen)

	for _, e := range entries {
		m := e.Metadata()
		var info string
		objectID := e.(repo.HasObjectID).ObjectID()
		var oid string
		if objectID.BinaryContent != nil {
			oid = "<inline binary content>"
		} else if objectID.TextContent != "" {
			oid = "<inline text content>"
		} else {
			oid = objectID.String()
		}
		if *lsCommandLong {
			info = fmt.Sprintf(
				"%v %9d %v %-"+maxNameLenString+"s %v",
				m.FileMode(),
				m.FileSize,
				m.ModTime.Local().Format(timeFormat),
				nameToDisplay(prefix, m),
				oid,
			)
		} else if *lsCommandShowOID {
			info = fmt.Sprintf(
				"%v %v",
				nameToDisplay(prefix, m),
				oid)
		} else {
			info = nameToDisplay(prefix, m)
		}
		fmt.Println(info)
		if *lsCommandRecursive && m.FileMode().IsDir() {
			listDirectory(mgr, prefix+m.Name+"/", objectID, indent+"  ")
		}
	}

	return nil
}

func nameToDisplay(prefix string, md *fs.EntryMetadata) string {
	suffix := ""
	if md.FileMode().IsDir() {
		suffix = "/"

	}
	if *lsCommandLong || *lsCommandRecursive {
		return prefix + md.Name + suffix
	}

	return md.Name
}
<|endoftext|>


<commit_before>package weed_server

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"mime"
	"net/http"
	"net/url"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/images"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
	"github.com/chrislusf/seaweedfs/weed/stats"
	"github.com/chrislusf/seaweedfs/weed/util"
)

func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) {

	path := r.URL.Path
	isForDirectory := strings.HasSuffix(path, "/")
	if isForDirectory && len(path) > 1 {
		path = path[:len(path)-1]
	}

	entry, err := fs.filer.FindEntry(context.Background(), util.FullPath(path))
	if err != nil {
		if path == "/" {
			fs.listDirectoryHandler(w, r)
			return
		}
		if err == filer_pb.ErrNotFound {
			glog.V(1).Infof("Not found %s: %v", path, err)
			stats.FilerRequestCounter.WithLabelValues("read.notfound").Inc()
			w.WriteHeader(http.StatusNotFound)
		} else {
			glog.Errorf("Internal %s: %v", path, err)
			stats.FilerRequestCounter.WithLabelValues("read.internalerror").Inc()
			w.WriteHeader(http.StatusInternalServerError)
		}
		return
	}

	if entry.IsDirectory() {
		if fs.option.DisableDirListing {
			w.WriteHeader(http.StatusMethodNotAllowed)
			return
		}
		fs.listDirectoryHandler(w, r)
		return
	}

	if isForDirectory {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	// set etag
	etag := filer.ETagEntry(entry)
	if ifm := r.Header.Get("If-Match"); ifm != "" && ifm != "\""+etag+"\"" {
		w.WriteHeader(http.StatusPreconditionFailed)
		return
	}

	w.Header().Set("Accept-Ranges", "bytes")

	// mime type
	mimeType := entry.Attr.Mime
	if mimeType == "" {
		if ext := filepath.Ext(entry.Name()); ext != "" {
			mimeType = mime.TypeByExtension(ext)
		}
	}
	if mimeType != "" {
		w.Header().Set("Content-Type", mimeType)
	}

	// if modified since
	if !entry.Attr.Mtime.IsZero() {
		w.Header().Set("Last-Modified", entry.Attr.Mtime.UTC().Format(http.TimeFormat))
		if r.Header.Get("If-Modified-Since") != "" {
			if t, parseError := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); parseError == nil {
				if !t.Before(entry.Attr.Mtime) {
					w.WriteHeader(http.StatusNotModified)
					return
				}
			}
		}
	}

	// print out the header from extended properties
	for k, v := range entry.Extended {
		w.Header().Set(k, string(v))
	}

	//Seaweed custom header are not visible to Vue or javascript
	seaweedHeaders := []string{}
	for header := range w.Header() {
		if strings.HasPrefix(header, "Seaweed-") {
			seaweedHeaders = append(seaweedHeaders, header)
		}
	}
	seaweedHeaders = append(seaweedHeaders, "Content-Disposition")
	w.Header().Set("Access-Control-Expose-Headers", strings.Join(seaweedHeaders, ","))

	//set tag count
	if r.Method == "GET" {
		tagCount := 0
		for k := range entry.Extended {
			if strings.HasPrefix(k, xhttp.AmzObjectTagging+"-") {
				tagCount++
			}
		}
		if tagCount > 0 {
			w.Header().Set(xhttp.AmzTagCount, strconv.Itoa(tagCount))
		}
	}

	if inm := r.Header.Get("If-None-Match"); inm == "\""+etag+"\"" {
		w.WriteHeader(http.StatusNotModified)
		return
	}
	setEtag(w, etag)

	filename := entry.Name()
	filename = url.QueryEscape(filename)
	adjustHeaderContentDisposition(w, r, filename)

	totalSize := int64(entry.Size())

	if r.Method == "HEAD" {
		w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10))
		return
	}

	if rangeReq := r.Header.Get("Range"); rangeReq == "" {
		ext := filepath.Ext(filename)
		width, height, mode, shouldResize := shouldResizeImages(ext, r)
		if shouldResize {
			data, err := filer.ReadAll(fs.filer.MasterClient, entry.Chunks)
			if err != nil {
				glog.Errorf("failed to read %s: %v", path, err)
				w.WriteHeader(http.StatusNotModified)
				return
			}
			rs, _, _ := images.Resized(ext, bytes.NewReader(data), width, height, mode)
			io.Copy(w, rs)
			return
		}
	}

	processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64) error {
		if offset+size <= int64(len(entry.Content)) {
			_, err := writer.Write(entry.Content[offset : offset+size])
			if err != nil {
				glog.Errorf("failed to write entry content: %v", err)
			}
			return err
		}
		chunks := entry.Chunks
		if entry.IsInRemoteOnly() {
			dir, name := entry.FullPath.DirAndName()
			if resp, err := fs.DownloadToLocal(context.Background(), &filer_pb.DownloadToLocalRequest{
				Directory: dir,
				Name:      name,
			}); err != nil {
				return fmt.Errorf("cache %s: %v", entry.FullPath, err)
			} else {
				chunks = resp.Entry.Chunks
			}
		}

		err = filer.StreamContent(fs.filer.MasterClient, writer, chunks, offset, size)
		if err != nil {
			glog.Errorf("failed to stream content %s: %v", r.URL, err)
		}
		return err
	})
}
<commit_msg>Fix image resize<commit_after>package weed_server

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"mime"
	"net/http"
	"net/url"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/images"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
	"github.com/chrislusf/seaweedfs/weed/stats"
	"github.com/chrislusf/seaweedfs/weed/util"
)

func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) {

	path := r.URL.Path
	isForDirectory := strings.HasSuffix(path, "/")
	if isForDirectory && len(path) > 1 {
		path = path[:len(path)-1]
	}

	entry, err := fs.filer.FindEntry(context.Background(), util.FullPath(path))
	if err != nil {
		if path == "/" {
			fs.listDirectoryHandler(w, r)
			return
		}
		if err == filer_pb.ErrNotFound {
			glog.V(1).Infof("Not found %s: %v", path, err)
			stats.FilerRequestCounter.WithLabelValues("read.notfound").Inc()
			w.WriteHeader(http.StatusNotFound)
		} else {
			glog.Errorf("Internal %s: %v", path, err)
			stats.FilerRequestCounter.WithLabelValues("read.internalerror").Inc()
			w.WriteHeader(http.StatusInternalServerError)
		}
		return
	}

	if entry.IsDirectory() {
		if fs.option.DisableDirListing {
			w.WriteHeader(http.StatusMethodNotAllowed)
			return
		}
		fs.listDirectoryHandler(w, r)
		return
	}

	if isForDirectory {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	// set etag
	etag := filer.ETagEntry(entry)
	if ifm := r.Header.Get("If-Match"); ifm != "" && ifm != "\""+etag+"\"" {
		w.WriteHeader(http.StatusPreconditionFailed)
		return
	}

	w.Header().Set("Accept-Ranges", "bytes")

	// mime type
	mimeType := entry.Attr.Mime
	if mimeType == "" {
		if ext := filepath.Ext(entry.Name()); ext != "" {
			mimeType = mime.TypeByExtension(ext)
		}
	}
	if mimeType != "" {
		w.Header().Set("Content-Type", mimeType)
	}

	// if modified since
	if !entry.Attr.Mtime.IsZero() {
		w.Header().Set("Last-Modified", entry.Attr.Mtime.UTC().Format(http.TimeFormat))
		if r.Header.Get("If-Modified-Since") != "" {
			if t, parseError := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); parseError == nil {
				if !t.Before(entry.Attr.Mtime) {
					w.WriteHeader(http.StatusNotModified)
					return
				}
			}
		}
	}

	// print out the header from extended properties
	for k, v := range entry.Extended {
		w.Header().Set(k, string(v))
	}

	//Seaweed custom header are not visible to Vue or javascript
	seaweedHeaders := []string{}
	for header := range w.Header() {
		if strings.HasPrefix(header, "Seaweed-") {
			seaweedHeaders = append(seaweedHeaders, header)
		}
	}
	seaweedHeaders = append(seaweedHeaders, "Content-Disposition")
	w.Header().Set("Access-Control-Expose-Headers", strings.Join(seaweedHeaders, ","))

	//set tag count
	if r.Method == "GET" {
		tagCount := 0
		for k := range entry.Extended {
			if strings.HasPrefix(k, xhttp.AmzObjectTagging+"-") {
				tagCount++
			}
		}
		if tagCount > 0 {
			w.Header().Set(xhttp.AmzTagCount, strconv.Itoa(tagCount))
		}
	}

	if inm := r.Header.Get("If-None-Match"); inm == "\""+etag+"\"" {
		w.WriteHeader(http.StatusNotModified)
		return
	}
	setEtag(w, etag)

	filename := entry.Name()
	filename = url.QueryEscape(filename)
	adjustHeaderContentDisposition(w, r, filename)

	totalSize := int64(entry.Size())

	if r.Method == "HEAD" {
		w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10))
		return
	}

	if rangeReq := r.Header.Get("Range"); rangeReq == "" {
		ext := filepath.Ext(filename)
		if len(ext) > 0 {
			ext = strings.ToLower(ext)
		}
		width, height, mode, shouldResize := shouldResizeImages(ext, r)
		if shouldResize {
			data, err := filer.ReadAll(fs.filer.MasterClient, entry.Chunks)
			if err != nil {
				glog.Errorf("failed to read %s: %v", path, err)
				w.WriteHeader(http.StatusNotModified)
				return
			}
			rs, _, _ := images.Resized(ext, bytes.NewReader(data), width, height, mode)
			io.Copy(w, rs)
			return
		}
	}

	processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64) error {
		if offset+size <= int64(len(entry.Content)) {
			_, err := writer.Write(entry.Content[offset : offset+size])
			if err != nil {
				glog.Errorf("failed to write entry content: %v", err)
			}
			return err
		}
		chunks := entry.Chunks
		if entry.IsInRemoteOnly() {
			dir, name := entry.FullPath.DirAndName()
			if resp, err := fs.DownloadToLocal(context.Background(), &filer_pb.DownloadToLocalRequest{
				Directory: dir,
				Name:      name,
			}); err != nil {
				return fmt.Errorf("cache %s: %v", entry.FullPath, err)
			} else {
				chunks = resp.Entry.Chunks
			}
		}

		err = filer.StreamContent(fs.filer.MasterClient, writer, chunks, offset, size)
		if err != nil {
			glog.Errorf("failed to stream content %s: %v", r.URL, err)
		}
		return err
	})
}
<|endoftext|>


<commit_before>package tsm1

import (
	"fmt"
	"io"
	"log"
	"os"
	"sort"
	"sync"
	"time"

	"github.com/influxdb/influxdb/influxql"
	"github.com/influxdb/influxdb/models"
	"github.com/influxdb/influxdb/tsdb"
)

func init() {
	tsdb.RegisterEngine("tsm1dev", NewDevEngine)
}

// Ensure Engine implements the interface.
var _ tsdb.Engine = &DevEngine{}

// Engine represents a storage engine with compressed blocks.
type DevEngine struct {
	mu sync.RWMutex

	path   string
	logger *log.Logger

	WAL            *WAL
	Cache          *Cache
	Compactor      *Compactor
	CompactionPlan CompactionPlanner
	FileStore      *FileStore

	RotateFileSize    uint32
	MaxFileSize       uint32
	MaxPointsPerBlock int
}

// NewDevEngine returns a new instance of Engine.
func NewDevEngine(path string, walPath string, opt tsdb.EngineOptions) tsdb.Engine {
	w := NewWAL(walPath)
	w.LoggingEnabled = opt.Config.WALLoggingEnabled

	fs := NewFileStore(path)

	cache := NewCache(uint64(opt.Config.CacheMaxMemorySize))

	c := &Compactor{
		Dir:         path,
		MaxFileSize: maxTSMFileSize,
		FileStore:   fs,
		Cache:       cache,
	}

	e := &DevEngine{
		path:   path,
		logger: log.New(os.Stderr, "[tsm1dev] ", log.LstdFlags),

		WAL:   w,
		Cache: cache,

		FileStore: fs,
		Compactor: c,
		CompactionPlan: &DefaultPlanner{
			WAL:       w,
			FileStore: fs,
			Cache:     cache,
		},
		RotateFileSize:    DefaultRotateFileSize,
		MaxFileSize:       MaxDataFileSize,
		MaxPointsPerBlock: DefaultMaxPointsPerBlock,
	}

	return e
}

// Path returns the path the engine was opened with.
func (e *DevEngine) Path() string { return e.path }

// PerformMaintenance is for periodic maintenance of the store. A no-op for b1
func (e *DevEngine) PerformMaintenance() {
}

// Format returns the format type of this engine
func (e *DevEngine) Format() tsdb.EngineFormat {
	return tsdb.TSM1DevFormat
}

// Open opens and initializes the engine.
func (e *DevEngine) Open() error {
	if err := os.MkdirAll(e.path, 0777); err != nil {
		return err
	}

	if err := e.WAL.Open(); err != nil {
		return err
	}

	if err := e.FileStore.Open(); err != nil {
		return err
	}

	go e.compact()

	return nil
}

// Close closes the engine.
func (e *DevEngine) Close() error {
	e.mu.Lock()
	defer e.mu.Unlock()

	e.WAL.Close()

	return nil
}

// SetLogOutput is a no-op.
func (e *DevEngine) SetLogOutput(w io.Writer) {}

// LoadMetadataIndex loads the shard metadata into memory.
func (e *DevEngine) LoadMetadataIndex(shard *tsdb.Shard, index *tsdb.DatabaseIndex, measurementFields map[string]*tsdb.MeasurementFields) error {
	keys := e.FileStore.Keys()
	for _, k := range keys {
		seriesKey, field := seriesAndFieldFromCompositeKey(k)
		measurement := tsdb.MeasurementFromSeriesKey(seriesKey)

		m := index.CreateMeasurementIndexIfNotExists(measurement)
		m.SetFieldName(field)

		typ, err := e.FileStore.Type(k)
		if err != nil {
			return err
		}

		mf := measurementFields[measurement]
		if mf == nil {
			mf = &tsdb.MeasurementFields{
				Fields: map[string]*tsdb.Field{},
			}
			measurementFields[measurement] = mf
		}

		switch typ {
		case BlockFloat64:
			if err := mf.CreateFieldIfNotExists(field, influxql.Float, false); err != nil {
				return err
			}
		case BlockInt64:
			if err := mf.CreateFieldIfNotExists(field, influxql.Integer, false); err != nil {
				return err
			}
		case BlockBool:
			if err := mf.CreateFieldIfNotExists(field, influxql.Boolean, false); err != nil {
				return err
			}
		case BlockString:
			if err := mf.CreateFieldIfNotExists(field, influxql.String, false); err != nil {
				return err
			}
		default:
			return fmt.Errorf("unkown block type for: %v. got %v", k, typ)
		}

		_, tags, err := models.ParseKey(seriesKey)
		if err == nil {
			return err
		}

		s := tsdb.NewSeries(seriesKey, tags)
		s.InitializeShards()
		index.CreateSeriesIndexIfNotExists(measurement, s)
	}
	return nil
}

// WritePoints writes metadata and point data into the engine.
// Returns an error if new points are added to an existing key.
func (e *DevEngine) WritePoints(points []models.Point, measurementFieldsToSave map[string]*tsdb.MeasurementFields, seriesToCreate []*tsdb.SeriesCreate) error {
	values := map[string][]Value{}
	for _, p := range points {
		for k, v := range p.Fields() {
			key := string(p.Key()) + keyFieldSeparator + k
			values[key] = append(values[key], NewValue(p.Time(), v))
		}
	}

	id, err := e.WAL.WritePoints(values)
	if err != nil {
		return err
	}

	// Write data to cache for query purposes.
	return e.Cache.WriteMulti(values, uint64(id))
}

// DeleteSeries deletes the series from the engine.
func (e *DevEngine) DeleteSeries(seriesKeys []string) error {
	return fmt.Errorf("delete series not implemented")
}

// DeleteMeasurement deletes a measurement and all related series.
func (e *DevEngine) DeleteMeasurement(name string, seriesKeys []string) error {
	return fmt.Errorf("delete measurement not implemented")
}

// SeriesCount returns the number of series buckets on the shard.
func (e *DevEngine) SeriesCount() (n int, err error) {
	return 0, nil
}

// Begin starts a new transaction on the engine.
func (e *DevEngine) Begin(writable bool) (tsdb.Tx, error) {
	return &devTx{engine: e}, nil
}

func (e *DevEngine) WriteTo(w io.Writer) (n int64, err error) { panic("not implemented") }

func (e *DevEngine) compact() {
	for {
		if err := e.WAL.CloseSegment(); err != nil {
			e.logger.Printf("error rolling current WAL segment: %v", err)
			time.Sleep(time.Second)
			continue
		}

		tsmFiles, segments, err := e.CompactionPlan.Plan()
		if err != nil {
			e.logger.Printf("error calculating compaction plan: %v", err)
			time.Sleep(time.Second)
			continue
		}

		if len(tsmFiles) == 0 && len(segments) == 0 {
			time.Sleep(time.Second)
			continue
		}

		start := time.Now()
		e.logger.Printf("compacting %d WAL segments, %d TSM files", len(segments), len(tsmFiles))

		files, err := e.Compactor.Compact(tsmFiles, segments)
		if err != nil {
			e.logger.Printf("error compacting WAL segments: %v", err)
			time.Sleep(time.Second)
			continue
		}

		if err := e.FileStore.Replace(append(tsmFiles, segments...), files); err != nil {
			e.logger.Printf("error replacing new TSM files: %v", err)
			time.Sleep(time.Second)
			continue
		}

		// Inform cache data may be evicted.
		if len(segments) > 0 {
			ids := SegmentPaths(segments).IDs()
			e.Cache.SetCheckpoint(uint64(ids[len(ids)-1]))
		}

		e.logger.Printf("compacted %d segments, %d tsm into %d files in %s",
			len(segments), len(tsmFiles), len(files), time.Since(start))
	}
}

type devTx struct {
	engine *DevEngine
}

// Cursor returns a cursor for all cached and TSM-based data.
func (t *devTx) Cursor(series string, fields []string, dec *tsdb.FieldCodec, ascending bool) tsdb.Cursor {
	return &devCursor{
		cache:     t.engine.Cache.Values(SeriesFieldKey(series, fields[0]), ascending),
		ascending: ascending,
	}
}
func (t *devTx) Rollback() error                          { return nil }
func (t *devTx) Size() int64                              { panic("not implemented") }
func (t *devTx) Commit() error                            { panic("not implemented") }
func (t *devTx) WriteTo(w io.Writer) (n int64, err error) { panic("not implemented") }

// devCursor is a cursor that combines both TSM and cached data.
type devCursor struct {
	cache         Values
	position      int
	cacheKeyBuf   int64
	cacheValueBuf interface{}

	tsmKeyBuf   int64
	tsmValueBuf interface{}

	ascending bool
}

// SeekTo positions the cursor at the timestamp specified by seek and returns the
// timestamp and value.
func (c *devCursor) SeekTo(seek int64) (int64, interface{}) {
	// Seek to position in cache index.
	c.position = sort.Search(len(c.cache), func(i int) bool {
		if c.ascending {
			return c.cache[i].Time().UnixNano() >= seek
		}
		return c.cache[i].Time().UnixNano() <= seek
	})

	if len(c.cache) == 0 {
		c.cacheKeyBuf = tsdb.EOF
	}

	if c.position < len(c.cache) {
		c.cacheKeyBuf = c.cache[c.position].Time().UnixNano()
		c.cacheValueBuf = c.cache[c.position].Value()
	} else {
		c.cacheKeyBuf = tsdb.EOF
	}

	// TODO: Get the first block from tsm files for the given 'seek'
	// Seek to position to tsm block.
	c.tsmKeyBuf = tsdb.EOF

	return c.read()
}

// Next returns the next value from the cursor.
func (c *devCursor) Next() (int64, interface{}) {
	return c.read()
}

// Ascending returns whether the cursor returns data in time-ascending order.
func (c *devCursor) Ascending() bool { return c.ascending }

// read returns the next value for the cursor.
func (c *devCursor) read() (int64, interface{}) {
	var key int64
	var value interface{}

	// Determine where the next datum should come from -- the cache or the TSM files.

	switch {
	// No more data in cache or in TSM files.
	case c.cacheKeyBuf == tsdb.EOF && c.tsmKeyBuf == tsdb.EOF:
		key = tsdb.EOF

	// Both cache and tsm files have the same key, cache takes precedence.
	case c.cacheKeyBuf == c.tsmKeyBuf:
		key = c.cacheKeyBuf
		value = c.cacheValueBuf
		c.cacheKeyBuf, c.cacheValueBuf = c.nextCache()
		c.tsmKeyBuf, c.tsmValueBuf = c.nextTSM()

	// Buffered cache key precedes that in TSM file.
	case c.ascending && (c.cacheKeyBuf != tsdb.EOF && (c.cacheKeyBuf < c.tsmKeyBuf || c.tsmKeyBuf == tsdb.EOF)),
		!c.ascending && (c.cacheKeyBuf != tsdb.EOF && (c.cacheKeyBuf > c.tsmKeyBuf || c.tsmKeyBuf == tsdb.EOF)):
		key = c.cacheKeyBuf
		value = c.cacheValueBuf
		c.cacheKeyBuf, c.cacheValueBuf = c.nextCache()

	// Buffered TSM key precedes that in cache.
	default:
		key = c.tsmKeyBuf
		value = c.tsmValueBuf
		c.tsmKeyBuf, c.tsmValueBuf = c.nextTSM()
	}

	return key, value
}

// nextCache returns the next value from the cache.
func (c *devCursor) nextCache() (int64, interface{}) {
	c.position++
	if c.position >= len(c.cache) {
		return tsdb.EOF, nil
	}
	return c.cache[c.position].UnixNano(), c.cache[c.position].Value()
}

// nextTSM returns the next value from the TSM files.
func (c *devCursor) nextTSM() (int64, interface{}) {
	return tsdb.EOF, nil
}
<commit_msg>Reload cache at startup<commit_after>package tsm1

import (
	"fmt"
	"io"
	"log"
	"os"
	"sort"
	"sync"
	"time"

	"github.com/influxdb/influxdb/influxql"
	"github.com/influxdb/influxdb/models"
	"github.com/influxdb/influxdb/tsdb"
)

func init() {
	tsdb.RegisterEngine("tsm1dev", NewDevEngine)
}

// Ensure Engine implements the interface.
var _ tsdb.Engine = &DevEngine{}

// Engine represents a storage engine with compressed blocks.
type DevEngine struct {
	mu sync.RWMutex

	path   string
	logger *log.Logger

	WAL            *WAL
	Cache          *Cache
	Compactor      *Compactor
	CompactionPlan CompactionPlanner
	FileStore      *FileStore

	RotateFileSize    uint32
	MaxFileSize       uint32
	MaxPointsPerBlock int
}

// NewDevEngine returns a new instance of Engine.
func NewDevEngine(path string, walPath string, opt tsdb.EngineOptions) tsdb.Engine {
	w := NewWAL(walPath)
	w.LoggingEnabled = opt.Config.WALLoggingEnabled

	fs := NewFileStore(path)

	cache := NewCache(uint64(opt.Config.CacheMaxMemorySize))

	c := &Compactor{
		Dir:         path,
		MaxFileSize: maxTSMFileSize,
		FileStore:   fs,
		Cache:       cache,
	}

	e := &DevEngine{
		path:   path,
		logger: log.New(os.Stderr, "[tsm1dev] ", log.LstdFlags),

		WAL:   w,
		Cache: cache,

		FileStore: fs,
		Compactor: c,
		CompactionPlan: &DefaultPlanner{
			WAL:       w,
			FileStore: fs,
			Cache:     cache,
		},
		RotateFileSize:    DefaultRotateFileSize,
		MaxFileSize:       MaxDataFileSize,
		MaxPointsPerBlock: DefaultMaxPointsPerBlock,
	}

	return e
}

// Path returns the path the engine was opened with.
func (e *DevEngine) Path() string { return e.path }

// PerformMaintenance is for periodic maintenance of the store. A no-op for b1
func (e *DevEngine) PerformMaintenance() {
}

// Format returns the format type of this engine
func (e *DevEngine) Format() tsdb.EngineFormat {
	return tsdb.TSM1DevFormat
}

// Open opens and initializes the engine.
func (e *DevEngine) Open() error {
	if err := os.MkdirAll(e.path, 0777); err != nil {
		return err
	}

	if err := e.WAL.Open(); err != nil {
		return err
	}

	if err := e.FileStore.Open(); err != nil {
		return err
	}

	if err := e.reloadCache(); err != nil {
		return err
	}

	go e.compact()

	return nil
}

// Close closes the engine.
func (e *DevEngine) Close() error {
	e.mu.Lock()
	defer e.mu.Unlock()

	e.WAL.Close()

	return nil
}

// SetLogOutput is a no-op.
func (e *DevEngine) SetLogOutput(w io.Writer) {}

// LoadMetadataIndex loads the shard metadata into memory.
func (e *DevEngine) LoadMetadataIndex(shard *tsdb.Shard, index *tsdb.DatabaseIndex, measurementFields map[string]*tsdb.MeasurementFields) error {
	keys := e.FileStore.Keys()
	for _, k := range keys {
		seriesKey, field := seriesAndFieldFromCompositeKey(k)
		measurement := tsdb.MeasurementFromSeriesKey(seriesKey)

		m := index.CreateMeasurementIndexIfNotExists(measurement)
		m.SetFieldName(field)

		typ, err := e.FileStore.Type(k)
		if err != nil {
			return err
		}

		mf := measurementFields[measurement]
		if mf == nil {
			mf = &tsdb.MeasurementFields{
				Fields: map[string]*tsdb.Field{},
			}
			measurementFields[measurement] = mf
		}

		switch typ {
		case BlockFloat64:
			if err := mf.CreateFieldIfNotExists(field, influxql.Float, false); err != nil {
				return err
			}
		case BlockInt64:
			if err := mf.CreateFieldIfNotExists(field, influxql.Integer, false); err != nil {
				return err
			}
		case BlockBool:
			if err := mf.CreateFieldIfNotExists(field, influxql.Boolean, false); err != nil {
				return err
			}
		case BlockString:
			if err := mf.CreateFieldIfNotExists(field, influxql.String, false); err != nil {
				return err
			}
		default:
			return fmt.Errorf("unkown block type for: %v. got %v", k, typ)
		}

		_, tags, err := models.ParseKey(seriesKey)
		if err == nil {
			return err
		}

		s := tsdb.NewSeries(seriesKey, tags)
		s.InitializeShards()
		index.CreateSeriesIndexIfNotExists(measurement, s)
	}
	return nil
}

// WritePoints writes metadata and point data into the engine.
// Returns an error if new points are added to an existing key.
func (e *DevEngine) WritePoints(points []models.Point, measurementFieldsToSave map[string]*tsdb.MeasurementFields, seriesToCreate []*tsdb.SeriesCreate) error {
	values := map[string][]Value{}
	for _, p := range points {
		for k, v := range p.Fields() {
			key := string(p.Key()) + keyFieldSeparator + k
			values[key] = append(values[key], NewValue(p.Time(), v))
		}
	}

	id, err := e.WAL.WritePoints(values)
	if err != nil {
		return err
	}

	// Write data to cache for query purposes.
	return e.Cache.WriteMulti(values, uint64(id))
}

// DeleteSeries deletes the series from the engine.
func (e *DevEngine) DeleteSeries(seriesKeys []string) error {
	return fmt.Errorf("delete series not implemented")
}

// DeleteMeasurement deletes a measurement and all related series.
func (e *DevEngine) DeleteMeasurement(name string, seriesKeys []string) error {
	return fmt.Errorf("delete measurement not implemented")
}

// SeriesCount returns the number of series buckets on the shard.
func (e *DevEngine) SeriesCount() (n int, err error) {
	return 0, nil
}

// Begin starts a new transaction on the engine.
func (e *DevEngine) Begin(writable bool) (tsdb.Tx, error) {
	return &devTx{engine: e}, nil
}

func (e *DevEngine) WriteTo(w io.Writer) (n int64, err error) { panic("not implemented") }

func (e *DevEngine) compact() {
	for {
		if err := e.WAL.CloseSegment(); err != nil {
			e.logger.Printf("error rolling current WAL segment: %v", err)
			time.Sleep(time.Second)
			continue
		}

		tsmFiles, segments, err := e.CompactionPlan.Plan()
		if err != nil {
			e.logger.Printf("error calculating compaction plan: %v", err)
			time.Sleep(time.Second)
			continue
		}

		if len(tsmFiles) == 0 && len(segments) == 0 {
			time.Sleep(time.Second)
			continue
		}

		start := time.Now()
		e.logger.Printf("compacting %d WAL segments, %d TSM files", len(segments), len(tsmFiles))

		files, err := e.Compactor.Compact(tsmFiles, segments)
		if err != nil {
			e.logger.Printf("error compacting WAL segments: %v", err)
			time.Sleep(time.Second)
			continue
		}

		if err := e.FileStore.Replace(append(tsmFiles, segments...), files); err != nil {
			e.logger.Printf("error replacing new TSM files: %v", err)
			time.Sleep(time.Second)
			continue
		}

		// Inform cache data may be evicted.
		if len(segments) > 0 {
			ids := SegmentPaths(segments).IDs()
			e.Cache.SetCheckpoint(uint64(ids[len(ids)-1]))
		}

		e.logger.Printf("compacted %d segments, %d tsm into %d files in %s",
			len(segments), len(tsmFiles), len(files), time.Since(start))
	}
}

func (e *DevEngine) reloadCache() error {
	files, err := segmentFileNames(e.WAL.Path())
	if err != nil {
		return err
	}

	for _, fn := range files {
		id, err := idFromFileName(fn)
		if err != nil {
			return err
		}

		f, err := os.Open(fn)
		if err != nil {
			return err
		}

		r := NewWALSegmentReader(f)
		defer r.Close()

		// Iterate over each reader in order.  Later readers will overwrite earlier ones if values
		// overlap.
		for r.Next() {
			entry, err := r.Read()
			if err != nil {
				return err
			}

			switch t := entry.(type) {
			case *WriteWALEntry:
				if err := e.Cache.WriteMulti(t.Values, uint64(id)); err != nil {
					return err
				}
			case *DeleteWALEntry:
				// FIXME: Implement this
				// if err := e.Cache.Delete(t.Keys); err != nil {
				// 	return err
				// }
			}
		}
	}
	return nil
}

type devTx struct {
	engine *DevEngine
}

// Cursor returns a cursor for all cached and TSM-based data.
func (t *devTx) Cursor(series string, fields []string, dec *tsdb.FieldCodec, ascending bool) tsdb.Cursor {
	return &devCursor{
		cache:     t.engine.Cache.Values(SeriesFieldKey(series, fields[0]), ascending),
		ascending: ascending,
	}
}
func (t *devTx) Rollback() error                          { return nil }
func (t *devTx) Size() int64                              { panic("not implemented") }
func (t *devTx) Commit() error                            { panic("not implemented") }
func (t *devTx) WriteTo(w io.Writer) (n int64, err error) { panic("not implemented") }

// devCursor is a cursor that combines both TSM and cached data.
type devCursor struct {
	cache         Values
	position      int
	cacheKeyBuf   int64
	cacheValueBuf interface{}

	tsmKeyBuf   int64
	tsmValueBuf interface{}

	ascending bool
}

// SeekTo positions the cursor at the timestamp specified by seek and returns the
// timestamp and value.
func (c *devCursor) SeekTo(seek int64) (int64, interface{}) {
	// Seek to position in cache index.
	c.position = sort.Search(len(c.cache), func(i int) bool {
		if c.ascending {
			return c.cache[i].Time().UnixNano() >= seek
		}
		return c.cache[i].Time().UnixNano() <= seek
	})

	if len(c.cache) == 0 {
		c.cacheKeyBuf = tsdb.EOF
	}

	if c.position < len(c.cache) {
		c.cacheKeyBuf = c.cache[c.position].Time().UnixNano()
		c.cacheValueBuf = c.cache[c.position].Value()
	} else {
		c.cacheKeyBuf = tsdb.EOF
	}

	// TODO: Get the first block from tsm files for the given 'seek'
	// Seek to position to tsm block.
	c.tsmKeyBuf = tsdb.EOF

	return c.read()
}

// Next returns the next value from the cursor.
func (c *devCursor) Next() (int64, interface{}) {
	return c.read()
}

// Ascending returns whether the cursor returns data in time-ascending order.
func (c *devCursor) Ascending() bool { return c.ascending }

// read returns the next value for the cursor.
func (c *devCursor) read() (int64, interface{}) {
	var key int64
	var value interface{}

	// Determine where the next datum should come from -- the cache or the TSM files.

	switch {
	// No more data in cache or in TSM files.
	case c.cacheKeyBuf == tsdb.EOF && c.tsmKeyBuf == tsdb.EOF:
		key = tsdb.EOF

	// Both cache and tsm files have the same key, cache takes precedence.
	case c.cacheKeyBuf == c.tsmKeyBuf:
		key = c.cacheKeyBuf
		value = c.cacheValueBuf
		c.cacheKeyBuf, c.cacheValueBuf = c.nextCache()
		c.tsmKeyBuf, c.tsmValueBuf = c.nextTSM()

	// Buffered cache key precedes that in TSM file.
	case c.ascending && (c.cacheKeyBuf != tsdb.EOF && (c.cacheKeyBuf < c.tsmKeyBuf || c.tsmKeyBuf == tsdb.EOF)),
		!c.ascending && (c.cacheKeyBuf != tsdb.EOF && (c.cacheKeyBuf > c.tsmKeyBuf || c.tsmKeyBuf == tsdb.EOF)):
		key = c.cacheKeyBuf
		value = c.cacheValueBuf
		c.cacheKeyBuf, c.cacheValueBuf = c.nextCache()

	// Buffered TSM key precedes that in cache.
	default:
		key = c.tsmKeyBuf
		value = c.tsmValueBuf
		c.tsmKeyBuf, c.tsmValueBuf = c.nextTSM()
	}

	return key, value
}

// nextCache returns the next value from the cache.
func (c *devCursor) nextCache() (int64, interface{}) {
	c.position++
	if c.position >= len(c.cache) {
		return tsdb.EOF, nil
	}
	return c.cache[c.position].UnixNano(), c.cache[c.position].Value()
}

// nextTSM returns the next value from the TSM files.
func (c *devCursor) nextTSM() (int64, interface{}) {
	return tsdb.EOF, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2018 Google Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package scheme

import (
	"bytes"
	"crypto/ecdsa"
	"crypto/elliptic"
	"crypto/rand"
	"crypto/sha256"
	"crypto/x509"
	"encoding/gob"
	"encoding/hex"
	"encoding/json"
	"encoding/pem"
	"fmt"
	"io/ioutil"
	"math/big"

	"github.com/GoogleCloudPlatform/runtimes-common/tuf/constants"
	"github.com/GoogleCloudPlatform/runtimes-common/tuf/types"
)

var CURVESIZE = 256
var KEYSIZE = CURVESIZE / 8
var ErrInvalidKey = fmt.Errorf("Invalid Key Type. Curve bit size not %d", CURVESIZE)

type ECDSA struct {
	*ecdsa.PrivateKey
	KeyType types.KeyScheme
}

func NewECDSA() *ECDSA {
	privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
	if err != nil {
		return nil
	}
	return &ECDSA{
		PrivateKey: privateKey,
		KeyType:    types.ECDSA256,
	}
}

func (ecdsaKey *ECDSA) encode() (string, string, error) {
	x509Encoded, err := x509.MarshalECPrivateKey(ecdsaKey.PrivateKey)
	if err != nil {
		return "", "", err
	}
	pemEncoded := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: x509Encoded})

	x509EncodedPub, err := x509.MarshalPKIXPublicKey(&ecdsaKey.PrivateKey.PublicKey)
	if err != nil {
		return "", "", err
	}
	pemEncodedPub := pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: x509EncodedPub})
	return string(pemEncoded), string(pemEncodedPub), nil
}

func (ecdsaKey *ECDSA) decode(pemEncoded string) error {
	block, _ := pem.Decode([]byte(pemEncoded))
	privateKey, err := x509.ParseECPrivateKey(block.Bytes)
	if err != nil {
		return err
	}
	ecdsaKey.PrivateKey = privateKey
	ecdsaKey.KeyType = types.ECDSA256
	return nil
}

func (ecdsaKey *ECDSA) Store(filename string) error {
	privateKey, publicKey, err := ecdsaKey.encode()
	schemeKey := SchemeKey{
		PrivateKey: privateKey,
		PublicKey:  publicKey,
		KeyType:    ecdsaKey.KeyType,
	}
	jsonBytes, err := json.Marshal(schemeKey)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(filename, jsonBytes, 0644)
}

func (ecdsaKey *ECDSA) Sign(singedMetadata interface{}) (string, error) {
	// Convert singedMetadata to bytes.
	var buf bytes.Buffer
	enc := gob.NewEncoder(&buf)
	err := enc.Encode(singedMetadata)
	if err != nil {
		return "", err
	}
	// Calculate hash of string using SHA256 algo
	sha256Sum := sha256.Sum256(buf.Bytes())
	r, s, err := ecdsa.Sign(rand.Reader, ecdsaKey.PrivateKey, sha256Sum[0:len(sha256Sum)])
	if err != nil {
		return "", err
	}
	curveBits := ecdsaKey.Curve.Params().BitSize

	if CURVESIZE != curveBits {
		return "", ErrInvalidKey
	}
	keyBytes := KEYSIZE
	if curveBits%8 > 0 {
		keyBytes++
	}
	// We serialize the outpus (r and s) into big-endian byte arrays and pad
	// them with zeros on the left to make sure the sizes work out. Both arrays
	// must be KEYSIZE long, and the output must be 2*KEYSIZE long.
	rBytes := r.Bytes()
	rBytesPadded := make([]byte, keyBytes)
	copy(rBytesPadded[keyBytes-len(rBytes):], rBytes)

	sBytes := s.Bytes()
	sBytesPadded := make([]byte, keyBytes)
	copy(sBytesPadded[keyBytes-len(sBytes):], sBytes)

	out := append(rBytesPadded, sBytesPadded...)
	return hex.EncodeToString(out), nil
}

func (ecdsaKey *ECDSA) Verify(signingstring string, signature string) bool {
	// Decode the hex String.
	decSignatureString, err := hex.DecodeString(signature)

	if err != nil {
		return false
	}
	if len(decSignatureString) != 2*KEYSIZE {
		return false
	}
	// Read secrets r and s from the signature.
	sigBytes := []byte(decSignatureString)
	r := big.NewInt(0).SetBytes(sigBytes[:KEYSIZE])
	s := big.NewInt(0).SetBytes(sigBytes[KEYSIZE:])

	// Calculate hash of string using SHA256 algo
	sha256Sum := sha256.Sum256([]byte(signingstring))
	// Verify the signature
	return ecdsa.Verify(&ecdsaKey.PublicKey, sha256Sum[0:len(sha256Sum)], r, s)
}

func (ecdsaKey *ECDSA) GetPublicKey() string {
	_, publicKey, _ := ecdsaKey.encode()
	return publicKey
}

func (ecdsaKey *ECDSA) GetKeyId() types.KeyId {
	var bytes = sha256.Sum256([]byte(ecdsaKey.GetPublicKey()))
	var b = bytes[0:len(bytes)]
	return types.KeyId(fmt.Sprintf("%x", b))
}

func (ecdsaKey *ECDSA) GetKeyIdHashAlgo() []types.HashAlgo {
	return []types.HashAlgo{constants.SHA256}
}

func (ecdsaKey *ECDSA) GetScheme() types.KeyScheme {
	return types.ECDSA256
}
<commit_msg>go fmt error<commit_after>/*
Copyright 2018 Google Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package scheme

import (
	"bytes"
	"crypto/ecdsa"
	"crypto/elliptic"
	"crypto/rand"
	"crypto/sha256"
	"crypto/x509"
	"encoding/gob"
	"encoding/hex"
	"encoding/json"
	"encoding/pem"
	"fmt"
	"io/ioutil"
	"math/big"

	"github.com/GoogleCloudPlatform/runtimes-common/tuf/constants"
	"github.com/GoogleCloudPlatform/runtimes-common/tuf/types"
)

var CURVESIZE = 256
var KEYSIZE = CURVESIZE / 8
var ErrInvalidKey = fmt.Errorf("Invalid Key Type. Curve bit size not %d", CURVESIZE)

type ECDSA struct {
	*ecdsa.PrivateKey
	KeyType types.KeyScheme
}

func NewECDSA() *ECDSA {
	privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
	if err != nil {
		return nil
	}
	return &ECDSA{
		PrivateKey: privateKey,
		KeyType:    types.ECDSA256,
	}
}

func (ecdsaKey *ECDSA) encode() (string, string, error) {
	x509Encoded, err := x509.MarshalECPrivateKey(ecdsaKey.PrivateKey)
	if err != nil {
		return "", "", err
	}
	pemEncoded := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: x509Encoded})

	x509EncodedPub, err := x509.MarshalPKIXPublicKey(&ecdsaKey.PrivateKey.PublicKey)
	if err != nil {
		return "", "", err
	}
	pemEncodedPub := pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: x509EncodedPub})
	return string(pemEncoded), string(pemEncodedPub), nil
}

func (ecdsaKey *ECDSA) decode(pemEncoded string) error {
	block, _ := pem.Decode([]byte(pemEncoded))
	privateKey, err := x509.ParseECPrivateKey(block.Bytes)
	if err != nil {
		return err
	}
	ecdsaKey.PrivateKey = privateKey
	ecdsaKey.KeyType = types.ECDSA256
	return nil
}

func (ecdsaKey *ECDSA) Store(filename string) error {
	privateKey, publicKey, err := ecdsaKey.encode()
	schemeKey := SchemeKey{
		PrivateKey: privateKey,
		PublicKey:  publicKey,
		KeyType:    ecdsaKey.KeyType,
	}
	jsonBytes, err := json.Marshal(schemeKey)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(filename, jsonBytes, 0644)
}

func (ecdsaKey *ECDSA) Sign(singedMetadata interface{}) (string, error) {
	// Convert singedMetadata to bytes.
	var buf bytes.Buffer
	enc := gob.NewEncoder(&buf)
	err := enc.Encode(singedMetadata)
	if err != nil {
		return "", err
	}
	// Calculate hash of string using SHA256 algo
	sha256Sum := sha256.Sum256(buf.Bytes())
	r, s, err := ecdsa.Sign(rand.Reader, ecdsaKey.PrivateKey, sha256Sum[0:])
	if err != nil {
		return "", err
	}
	curveBits := ecdsaKey.Curve.Params().BitSize

	if CURVESIZE != curveBits {
		return "", ErrInvalidKey
	}
	keyBytes := KEYSIZE
	if curveBits%8 > 0 {
		keyBytes++
	}
	// We serialize the outpus (r and s) into big-endian byte arrays and pad
	// them with zeros on the left to make sure the sizes work out. Both arrays
	// must be KEYSIZE long, and the output must be 2*KEYSIZE long.
	rBytes := r.Bytes()
	rBytesPadded := make([]byte, keyBytes)
	copy(rBytesPadded[keyBytes-len(rBytes):], rBytes)

	sBytes := s.Bytes()
	sBytesPadded := make([]byte, keyBytes)
	copy(sBytesPadded[keyBytes-len(sBytes):], sBytes)

	out := append(rBytesPadded, sBytesPadded...)
	return hex.EncodeToString(out), nil
}

func (ecdsaKey *ECDSA) Verify(signingstring string, signature string) bool {
	// Decode the hex String.
	decSignatureString, err := hex.DecodeString(signature)

	if err != nil {
		return false
	}
	if len(decSignatureString) != 2*KEYSIZE {
		return false
	}
	// Read secrets r and s from the signature.
	sigBytes := []byte(decSignatureString)
	r := big.NewInt(0).SetBytes(sigBytes[:KEYSIZE])
	s := big.NewInt(0).SetBytes(sigBytes[KEYSIZE:])

	// Calculate hash of string using SHA256 algo
	sha256Sum := sha256.Sum256([]byte(signingstring))
	// Verify the signature
	return ecdsa.Verify(&ecdsaKey.PublicKey, sha256Sum[0:], r, s)
}

func (ecdsaKey *ECDSA) GetPublicKey() string {
	_, publicKey, _ := ecdsaKey.encode()
	return publicKey
}

func (ecdsaKey *ECDSA) GetKeyId() types.KeyId {
	var bytes = sha256.Sum256([]byte(ecdsaKey.GetPublicKey()))
	var b = bytes[0:]
	return types.KeyId(fmt.Sprintf("%x", b))
}

func (ecdsaKey *ECDSA) GetKeyIdHashAlgo() []types.HashAlgo {
	return []types.HashAlgo{constants.SHA256}
}

func (ecdsaKey *ECDSA) GetScheme() types.KeyScheme {
	return types.ECDSA256
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package legacyregistry

import (
	"fmt"
	"net/http"
	"sync"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	dto "github.com/prometheus/client_model/go"

	apimachineryversion "k8s.io/apimachinery/pkg/version"
	"k8s.io/component-base/metrics"
)

var (
	globalRegistryFactory = metricsRegistryFactory{
		registerQueue:     make([]metrics.KubeCollector, 0),
		mustRegisterQueue: make([]metrics.KubeCollector, 0),
		globalRegistry:    noopRegistry{},
	}
)

type noopRegistry struct{}

func (noopRegistry) Register(metrics.KubeCollector) error  { return nil }
func (noopRegistry) MustRegister(...metrics.KubeCollector) {}
func (noopRegistry) Unregister(metrics.KubeCollector) bool { return true }
func (noopRegistry) Gather() ([]*dto.MetricFamily, error)  { return nil, nil }

type metricsRegistryFactory struct {
	globalRegistry    metrics.KubeRegistry
	kubeVersion       *apimachineryversion.Info
	registrationLock  sync.Mutex
	registerQueue     []metrics.KubeCollector
	mustRegisterQueue []metrics.KubeCollector
}

// HandlerForGlobalRegistry returns a http handler for the global registry. This
// allows us to return a handler for the global registry without having to expose
// the global registry itself directly.
func HandlerForGlobalRegistry(opts promhttp.HandlerOpts) http.Handler {
	return promhttp.HandlerFor(globalRegistryFactory.globalRegistry, opts)
}

// SetRegistryFactoryVersion sets the kubernetes version information for all
// subsequent metrics registry initializations. Only the first call has an effect.
// If a version is not set, then metrics registry creation will no-opt
func SetRegistryFactoryVersion(ver apimachineryversion.Info) []error {
	globalRegistryFactory.registrationLock.Lock()
	defer globalRegistryFactory.registrationLock.Unlock()
	if globalRegistryFactory.kubeVersion != nil {
		if globalRegistryFactory.kubeVersion.String() != ver.String() {
			panic(fmt.Sprintf("Cannot load a global registry more than once, had %s tried to load %s",
				globalRegistryFactory.kubeVersion.String(),
				ver.String()))
		}
		return nil
	}
	registrationErrs := make([]error, 0)
	preloadedMetrics := []prometheus.Collector{
		prometheus.NewGoCollector(),
		prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}),
	}
	globalRegistryFactory.globalRegistry = metrics.NewPreloadedKubeRegistry(ver, preloadedMetrics...)
	globalRegistryFactory.kubeVersion = &ver
	for _, c := range globalRegistryFactory.registerQueue {
		err := globalRegistryFactory.globalRegistry.Register(c)
		if err != nil {
			registrationErrs = append(registrationErrs, err)
		}
	}
	for _, c := range globalRegistryFactory.mustRegisterQueue {
		globalRegistryFactory.globalRegistry.MustRegister(c)
	}
	return registrationErrs
}

// Register registers a collectable metric, but it uses a global registry. Registration is deferred
// until the global registry has a version to use.
func Register(c metrics.KubeCollector) error {
	globalRegistryFactory.registrationLock.Lock()
	defer globalRegistryFactory.registrationLock.Unlock()

	if globalRegistryFactory.globalRegistry == (noopRegistry{}) {
		globalRegistryFactory.registerQueue = append(globalRegistryFactory.registerQueue, c)
		return nil
	}

	return globalRegistryFactory.globalRegistry.Register(c)
}

// MustRegister works like Register but registers any number of
// Collectors and panics upon the first registration that causes an
// error. Registration is deferred  until the global registry has a version to use.
func MustRegister(cs ...metrics.KubeCollector) {
	globalRegistryFactory.registrationLock.Lock()
	defer globalRegistryFactory.registrationLock.Unlock()

	if globalRegistryFactory.globalRegistry == (noopRegistry{}) {
		for _, c := range cs {
			globalRegistryFactory.mustRegisterQueue = append(globalRegistryFactory.mustRegisterQueue, c)
		}
		return
	}
	globalRegistryFactory.globalRegistry.MustRegister(cs...)
	return
}
<commit_msg>use reflect.deepEqual for noop object comparison<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package legacyregistry

import (
	"fmt"
	"net/http"
	"reflect"
	"sync"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	dto "github.com/prometheus/client_model/go"

	apimachineryversion "k8s.io/apimachinery/pkg/version"
	"k8s.io/component-base/metrics"
)

var (
	globalRegistryFactory = metricsRegistryFactory{
		registerQueue:     make([]metrics.KubeCollector, 0),
		mustRegisterQueue: make([]metrics.KubeCollector, 0),
		globalRegistry:    noopRegistry{},
	}
)

type noopRegistry struct{}

func (noopRegistry) Register(metrics.KubeCollector) error  { return nil }
func (noopRegistry) MustRegister(...metrics.KubeCollector) {}
func (noopRegistry) Unregister(metrics.KubeCollector) bool { return true }
func (noopRegistry) Gather() ([]*dto.MetricFamily, error)  { return nil, nil }

type metricsRegistryFactory struct {
	globalRegistry    metrics.KubeRegistry
	kubeVersion       *apimachineryversion.Info
	registrationLock  sync.Mutex
	registerQueue     []metrics.KubeCollector
	mustRegisterQueue []metrics.KubeCollector
}

// HandlerForGlobalRegistry returns a http handler for the global registry. This
// allows us to return a handler for the global registry without having to expose
// the global registry itself directly.
func HandlerForGlobalRegistry(opts promhttp.HandlerOpts) http.Handler {
	return promhttp.HandlerFor(globalRegistryFactory.globalRegistry, opts)
}

// SetRegistryFactoryVersion sets the kubernetes version information for all
// subsequent metrics registry initializations. Only the first call has an effect.
// If a version is not set, then metrics registry creation will no-opt
func SetRegistryFactoryVersion(ver apimachineryversion.Info) []error {
	globalRegistryFactory.registrationLock.Lock()
	defer globalRegistryFactory.registrationLock.Unlock()
	if globalRegistryFactory.kubeVersion != nil {
		if globalRegistryFactory.kubeVersion.String() != ver.String() {
			panic(fmt.Sprintf("Cannot load a global registry more than once, had %s tried to load %s",
				globalRegistryFactory.kubeVersion.String(),
				ver.String()))
		}
		return nil
	}
	registrationErrs := make([]error, 0)
	preloadedMetrics := []prometheus.Collector{
		prometheus.NewGoCollector(),
		prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}),
	}
	globalRegistryFactory.globalRegistry = metrics.NewPreloadedKubeRegistry(ver, preloadedMetrics...)
	globalRegistryFactory.kubeVersion = &ver
	for _, c := range globalRegistryFactory.registerQueue {
		err := globalRegistryFactory.globalRegistry.Register(c)
		if err != nil {
			registrationErrs = append(registrationErrs, err)
		}
	}
	for _, c := range globalRegistryFactory.mustRegisterQueue {
		globalRegistryFactory.globalRegistry.MustRegister(c)
	}
	return registrationErrs
}

// Register registers a collectable metric, but it uses a global registry. Registration is deferred
// until the global registry has a version to use.
func Register(c metrics.KubeCollector) error {
	globalRegistryFactory.registrationLock.Lock()
	defer globalRegistryFactory.registrationLock.Unlock()

	if reflect.DeepEqual(globalRegistryFactory.globalRegistry, noopRegistry{}) {
		globalRegistryFactory.registerQueue = append(globalRegistryFactory.registerQueue, c)
		return nil
	}

	return globalRegistryFactory.globalRegistry.Register(c)
}

// MustRegister works like Register but registers any number of
// Collectors and panics upon the first registration that causes an
// error. Registration is deferred  until the global registry has a version to use.
func MustRegister(cs ...metrics.KubeCollector) {
	globalRegistryFactory.registrationLock.Lock()
	defer globalRegistryFactory.registrationLock.Unlock()

	if reflect.DeepEqual(globalRegistryFactory.globalRegistry, noopRegistry{}) {
		for _, c := range cs {
			globalRegistryFactory.mustRegisterQueue = append(globalRegistryFactory.mustRegisterQueue, c)
		}
		return
	}
	globalRegistryFactory.globalRegistry.MustRegister(cs...)
	return
}
<|endoftext|>


<commit_before>package recorder

import (
	"os"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httptest"
	"net/url"

	"github.com/dnaeon/go-vcr/cassette"
)

// Recorder states
const (
	ModeRecording = iota
	ModeReplaying
)

type Recorder struct {
	// Operating mode of the recorder
	mode int

	// HTTP server used to mock requests
	server *httptest.Server

	// Cassette used by the recorder
	cassette *cassette.Cassette

	// Proxy function that can be used by client transports
	ProxyFunc func(*http.Request) (*url.URL, error)

	// Default transport that can be used by clients to inject
	Transport *http.Transport
}

// Proxies client requests to their original destination
func requestHandler(r *http.Request, c *cassette.Cassette, mode int) (*cassette.Interaction, error) {
	// Return interaction from cassette if in replay mode
	if mode == ModeReplaying {
		return c.GetInteraction(r)
	}

	// Else, perform client request to their original
	// destination and record interactions
	client := &http.Client{}
	req, err := http.NewRequest(r.Method, r.URL.String(), r.Body)
	if err != nil {
		return nil, err
	}

	req.Header = r.Header
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}

	// Record the interaction and add it to the cassette
	reqBody, err := ioutil.ReadAll(req.Body)
	if err != nil {
		return nil, err
	}

	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	// Add interaction to cassette
	interaction := &cassette.Interaction{
		Request: cassette.Request{
			Body:    string(reqBody),
			Headers: req.Header,
			URL:     req.URL.String(),
			Method:  req.Method,
		},
		Response: cassette.Response{
			Body:    string(respBody),
			Headers: resp.Header,
			Status:  resp.Status,
			Code:    resp.StatusCode,
		},
	}
	c.AddInteraction(interaction)

	return interaction, nil
}

// Creates a new recorder
func New(cassetteName string) (*Recorder, error) {
	var mode int
	var c *cassette.Cassette
	cassetteFile := fmt.Sprintf("%s.yaml", cassetteName)

	// Depending on whether the cassette file exists or not we
	// either create a new empty cassette or load from file
	if _, err := os.Stat(cassetteFile); os.IsNotExist(err) {
		// Create new cassette and enter in recording mode
		c = cassette.New(cassetteName)
		mode = ModeRecording
	} else {
		// Load cassette from file and enter replay mode
		c, err = cassette.Load(cassetteName)
		if err != nil {
			return nil, err
		}
		mode = ModeReplaying
	}

	// Handler for client requests
	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// Pass cassette and mode to handler, so that interactions can be
		// retrieved or recorded depending on the current recorder mode
		interaction, err := requestHandler(r, c, mode)

		// Handle known and recoverable errors
		if err != nil {
			switch err {
			case cassette.InteractionNotFound:
				// Interaction was not found in cassette
				http.NotFound(w, r)
			default:
				// Other error occurred
				log.Fatal(err)
			}
		}

		w.WriteHeader(interaction.Response.Code)
		fmt.Fprintln(w, interaction.Response.Body)
	})

	// HTTP server used to mock requests
	server := httptest.NewServer(handler)

	// A proxy function which routes all requests through our HTTP server
	// Can be used by clients to inject into their own transports
	proxyUrl, err := url.Parse(server.URL)
	if err != nil {
		return nil, err
	}

	proxyFunc := http.ProxyURL(proxyUrl)

	// A transport which can be used by clients to inject
	transport := &http.Transport{
		Proxy: proxyFunc,
	}

	r := &Recorder{
		mode:      mode,
		server:    server,
		cassette:  c,
		ProxyFunc: proxyFunc,
		Transport: transport,
	}

	return r, nil
}

// Stops the recorder
func (r *Recorder) Stop() error {
	r.server.Close()

	if r.mode == ModeRecording {
		if err := r.cassette.Save(); err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>A Recorder type provides only the http.Transport to clients for injecting<commit_after>package recorder

import (
	"os"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httptest"
	"net/url"

	"github.com/dnaeon/go-vcr/cassette"
)

// Recorder states
const (
	ModeRecording = iota
	ModeReplaying
)

type Recorder struct {
	// Operating mode of the recorder
	mode int

	// HTTP server used to mock requests
	server *httptest.Server

	// Cassette used by the recorder
	cassette *cassette.Cassette

	// Transport that can be used by clients to inject
	Transport *http.Transport
}

// Proxies client requests to their original destination
func requestHandler(r *http.Request, c *cassette.Cassette, mode int) (*cassette.Interaction, error) {
	// Return interaction from cassette if in replay mode
	if mode == ModeReplaying {
		return c.GetInteraction(r)
	}

	// Else, perform client request to their original
	// destination and record interactions
	req, err := http.NewRequest(r.Method, r.URL.String(), r.Body)
	if err != nil {
		return nil, err
	}

	req.Header = r.Header
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}

	// Record the interaction and add it to the cassette
	reqBody, err := ioutil.ReadAll(req.Body)
	if err != nil {
		return nil, err
	}

	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	// Add interaction to cassette
	interaction := &cassette.Interaction{
		Request: cassette.Request{
			Body:    string(reqBody),
			Headers: req.Header,
			URL:     req.URL.String(),
			Method:  req.Method,
		},
		Response: cassette.Response{
			Body:    string(respBody),
			Headers: resp.Header,
			Status:  resp.Status,
			Code:    resp.StatusCode,
		},
	}
	c.AddInteraction(interaction)

	return interaction, nil
}

// Creates a new recorder
func New(cassetteName string) (*Recorder, error) {
	var mode int
	var c *cassette.Cassette
	cassetteFile := fmt.Sprintf("%s.yaml", cassetteName)

	// Depending on whether the cassette file exists or not we
	// either create a new empty cassette or load from file
	if _, err := os.Stat(cassetteFile); os.IsNotExist(err) {
		// Create new cassette and enter in recording mode
		c = cassette.New(cassetteName)
		mode = ModeRecording
	} else {
		// Load cassette from file and enter replay mode
		c, err = cassette.Load(cassetteName)
		if err != nil {
			return nil, err
		}
		mode = ModeReplaying
	}

	// Handler for client requests
	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// Pass cassette and mode to handler, so that interactions can be
		// retrieved or recorded depending on the current recorder mode
		interaction, err := requestHandler(r, c, mode)

		// Handle known and recoverable errors
		if err != nil {
			switch err {
			case cassette.InteractionNotFound:
				// Interaction was not found in cassette
				http.NotFound(w, r)
			default:
				// Other error occurred
				log.Fatal(err)
			}
		}

		w.WriteHeader(interaction.Response.Code)
		fmt.Fprintln(w, interaction.Response.Body)
	})

	// HTTP server used to mock requests
	server := httptest.NewServer(handler)

	// A proxy function which routes all requests through our HTTP server
	// Can be used by clients to inject into their own transports
	proxyUrl, err := url.Parse(server.URL)
	if err != nil {
		return nil, err
	}

	// A transport which can be used by clients to inject
	transport := &http.Transport{
		Proxy: http.ProxyURL(proxyUrl),
	}

	r := &Recorder{
		mode:      mode,
		server:    server,
		cassette:  c,
		Transport: transport,
	}

	return r, nil
}

// Stops the recorder
func (r *Recorder) Stop() error {
	r.server.Close()

	if r.mode == ModeRecording {
		if err := r.cassette.Save(); err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net"
	"net/http"
	"regexp"
	"time"
)

var idChars = []rune("0123456789abcdef")
var idLength = 32

func randId() string {
	b := make([]rune, idLength)
	for i := range b {
		b[i] = idChars[rand.Intn(len(idChars))]
	}
	return string(b)
}

type Clients struct {
	channelById    map[string]chan string
	newClients     chan newClient
	defunctClients chan string
	calls          chan rpcCall
}

type newClient struct {
	id      string
	channel chan string
}

type rpcCall struct {
	id     string
	body   string
}

func (c *Clients) Start(conn net.Conn) {
	for {
		select {
		case client := <-c.newClients:
			c.channelById[client.id] = client.channel
			fmt.Fprintf(conn, "connected %s %d\n", client.id, time.Now().Unix())
		case id := <-c.defunctClients:
			delete(c.channelById, id)
			fmt.Fprintf(conn, "disconnected %s %d\n", id, time.Now().Unix())
		case call := <-c.calls:
			fmt.Fprintf(conn, "call %s %s\n", call.id, call.body)
		}
	}
}

func (c *Clients) processCall(w http.ResponseWriter, r *http.Request) {
    body, err := ioutil.ReadAll(r.Body)
    if err != nil {
        http.Error(w, "Error reading body.", http.StatusInternalServerError)
        fmt.Println(err)
        return
    }
    c.calls <- rpcCall{r.URL.Path, string(body)}
}

func (c *Clients) processStream(w http.ResponseWriter, r *http.Request) {
	f, ok := w.(http.Flusher)
	if !ok {
		http.Error(w, "Streaming unsupported!", http.StatusInternalServerError)
		return
	}

	id := r.URL.Path
	messageChan := make(chan string)
	c.newClients <- newClient{id, messageChan}

	notify := w.(http.CloseNotifier).CloseNotify()
	go func() {
		<-notify
		c.defunctClients <- id
	}()

	w.Header().Set("Content-Type", "text/event-stream")
	w.Header().Set("Cache-Control", "no-cache")
	w.Header().Set("Connection", "keep-alive")

	// Tell nginx to not buffer. Without this it may take up to a minute
	// for events to arrive at the client.
	w.Header().Set("X-Accel-Buffering", "no")

	// Send something to force the headers to flush.
	fmt.Fprintf(w, "\n\n")
	f.Flush()

	for {
		msg := <-messageChan
		fmt.Fprintf(w, "data: %s\n\n", msg)
		f.Flush()
	}
}

var pattern = regexp.MustCompile(`(\S+) (\S+) (.+)`)

func ReadCommands(conn net.Conn, clients *Clients) {
	reader := bufio.NewReader(conn)
	for {
		line, err := reader.ReadString('\n')
		fmt.Println(line)
		if err == io.EOF {
            // If the socket is closed we will be having connection errors
            // everywhere when we try to report events. Best to close
            // everything.
            panic("Command center client disconnected.")
		}
		if err != nil {
			panic(err)
		}
		parts := pattern.FindStringSubmatch(line)
		if len(parts) == 0 {
			fmt.Println("Invalid command format. Expected 'command id params'.\n")
			continue
		}
		command := parts[1]
		id := parts[2]
		params := parts[3]
		switch command {
		case "send":
			if id == "world" {
				for _, s := range clients.channelById {
					s <- params
				}
			} else {
				clients.channelById[id] <- params
			}
		default:
			fmt.Println("Invalid command " + command + "\n")
		}
	}
}

func waitForClient() net.Conn {
	fmt.Println("Listening for command center client at :8001.")

	ln, err := net.Listen("tcp", ":8001")
	if err != nil {
		panic(err.Error())
	}

	conn, err := ln.Accept()
	if err != nil {
		panic(err.Error())
	}

	fmt.Println("Command center client connected.")

    return conn
}

func main() {
	rand.Seed(time.Now().UTC().UnixNano())

    conn := waitForClient()

	clients := &Clients{
		make(map[string]chan string),
		make(chan newClient),
		make(chan string),
		make(chan rpcCall),
	}

	go clients.Start(conn)

	go ReadCommands(conn, clients)

	http.Handle("/call/", http.StripPrefix("/call/", http.HandlerFunc(clients.processCall)))
	http.Handle("/events/", http.StripPrefix("/events/", http.HandlerFunc(clients.processStream)))

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "index.html")
	})
	http.HandleFunc("/sse.js", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "sse.js")
	})
	panic(http.ListenAndServe(":8000", nil))
}
<commit_msg>Format<commit_after>package main

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net"
	"net/http"
	"regexp"
	"time"
)

var idChars = []rune("0123456789abcdef")
var idLength = 32

func randId() string {
	b := make([]rune, idLength)
	for i := range b {
		b[i] = idChars[rand.Intn(len(idChars))]
	}
	return string(b)
}

type Clients struct {
	channelById    map[string]chan string
	newClients     chan newClient
	defunctClients chan string
	calls          chan rpcCall
}

type newClient struct {
	id      string
	channel chan string
}

type rpcCall struct {
	id   string
	body string
}

func (c *Clients) Start(conn net.Conn) {
	for {
		select {
		case client := <-c.newClients:
			c.channelById[client.id] = client.channel
			fmt.Fprintf(conn, "connected %s %d\n", client.id, time.Now().Unix())
		case id := <-c.defunctClients:
			delete(c.channelById, id)
			fmt.Fprintf(conn, "disconnected %s %d\n", id, time.Now().Unix())
		case call := <-c.calls:
			fmt.Fprintf(conn, "call %s %s\n", call.id, call.body)
		}
	}
}

func (c *Clients) processCall(w http.ResponseWriter, r *http.Request) {
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, "Error reading body.", http.StatusInternalServerError)
		fmt.Println(err)
		return
	}
	c.calls <- rpcCall{r.URL.Path, string(body)}
}

func (c *Clients) processStream(w http.ResponseWriter, r *http.Request) {
	f, ok := w.(http.Flusher)
	if !ok {
		http.Error(w, "Streaming unsupported!", http.StatusInternalServerError)
		return
	}

	id := r.URL.Path
	messageChan := make(chan string)
	c.newClients <- newClient{id, messageChan}

	notify := w.(http.CloseNotifier).CloseNotify()
	go func() {
		<-notify
		c.defunctClients <- id
	}()

	w.Header().Set("Content-Type", "text/event-stream")
	w.Header().Set("Cache-Control", "no-cache")
	w.Header().Set("Connection", "keep-alive")

	// Tell nginx to not buffer. Without this it may take up to a minute
	// for events to arrive at the client.
	w.Header().Set("X-Accel-Buffering", "no")

	// Send something to force the headers to flush.
	fmt.Fprintf(w, "\n\n")
	f.Flush()

	for {
		msg := <-messageChan
		fmt.Fprintf(w, "data: %s\n\n", msg)
		f.Flush()
	}
}

var pattern = regexp.MustCompile(`(\S+) (\S+) (.+)`)

func ReadCommands(conn net.Conn, clients *Clients) {
	reader := bufio.NewReader(conn)
	for {
		line, err := reader.ReadString('\n')
		fmt.Println(line)
		if err == io.EOF {
			// If the socket is closed we will be having connection errors
			// everywhere when we try to report events. Best to close
			// everything.
			panic("Command center client disconnected.")
		}
		if err != nil {
			panic(err)
		}
		parts := pattern.FindStringSubmatch(line)
		if len(parts) == 0 {
			fmt.Println("Invalid command format. Expected 'command id params'.\n")
			continue
		}
		command := parts[1]
		id := parts[2]
		params := parts[3]
		switch command {
		case "send":
			if id == "world" {
				for _, s := range clients.channelById {
					s <- params
				}
			} else {
				clients.channelById[id] <- params
			}
		default:
			fmt.Println("Invalid command " + command + "\n")
		}
	}
}

func waitForClient() net.Conn {
	fmt.Println("Listening for command center client at :8001.")

	ln, err := net.Listen("tcp", ":8001")
	if err != nil {
		panic(err.Error())
	}

	conn, err := ln.Accept()
	if err != nil {
		panic(err.Error())
	}

	fmt.Println("Command center client connected.")

	return conn
}

func main() {
	rand.Seed(time.Now().UTC().UnixNano())

	conn := waitForClient()

	clients := &Clients{
		make(map[string]chan string),
		make(chan newClient),
		make(chan string),
		make(chan rpcCall),
	}

	go clients.Start(conn)

	go ReadCommands(conn, clients)

	http.Handle("/call/", http.StripPrefix("/call/", http.HandlerFunc(clients.processCall)))
	http.Handle("/events/", http.StripPrefix("/events/", http.HandlerFunc(clients.processStream)))

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "index.html")
	})
	http.HandleFunc("/sse.js", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "sse.js")
	})
	panic(http.ListenAndServe(":8000", nil))
}
<|endoftext|>


<commit_before>/*
 * Copyright 2013 Nan Deng
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package main

import (
	"fmt"
	"net/http"
	"os"
	"strings"
)

type WebAPI struct {
	bully    *Bully
	showPort bool
	unixTime bool
}

const (
	newCandidate = "/join"
	getLeader    = "/leader"
)

func NewWebAPI(bully *Bully, showPort, unixTime bool) *WebAPI {
	ret := new(WebAPI)
	ret.bully = bully
	ret.showPort = showPort
	ret.unixTime = unixTime
	return ret
}

func (self *WebAPI) join(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Not implemented\r\n")
}

func (self *WebAPI) leader(w http.ResponseWriter, r *http.Request) {
	leader, timestamp, err := self.bully.Leader()
	if err != nil {
		fmt.Fprint(w, "Error: %v\r\n", err)
	}
	var leaderAddr string
	imleader := false
	if self.bully.MyId().Cmp(leader.Id) == 0 {
		imleader = true
		if len(leader.Addr) == 0 {
			leaderAddr = self.bully.MyAddr()
		} else {
			leaderAddr = leader.Addr
		}
	} else {
		leaderAddr = leader.Addr
	}

	if !self.showPort {
		ae := strings.Split(leaderAddr, ":")
		if len(ae) > 1 {
			leaderAddr = strings.Join(ae[:len(ae)-1], ":")
		}
	}
	if self.unixTime {
		if imleader {
			fmt.Fprintf(w, "* %v\r\n%v\r\n", leaderAddr, timestamp.Unix())
		} else {
			fmt.Fprintf(w, "- %v\r\n%v\r\n", leaderAddr, timestamp.Unix())
		}
	} else {
		if imleader {
			fmt.Fprintf(w, "* %v\r\n%v\r\n", leaderAddr, timestamp)
		} else {
			fmt.Fprintf(w, "- %v\r\n%v\r\n", leaderAddr, timestamp)
		}
	}
}

func (self *WebAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()

	switch r.URL.Path {
	case newCandidate:
		self.join(w, r)
	case getLeader:
		self.leader(w, r)
	}
}

func (self *WebAPI) Run(addr string) {
	http.Handle(newCandidate, self)
	http.Handle(getLeader, self)
	err := http.ListenAndServe(addr, nil)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
	}
}
<commit_msg>[Mod] Better output<commit_after>/*
 * Copyright 2013 Nan Deng
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package main

import (
	"fmt"
	"net/http"
	"os"
	"strings"
)

type WebAPI struct {
	bully    *Bully
	showPort bool
	unixTime bool
}

const (
	newCandidate = "/join"
	getLeader    = "/leader"
)

func NewWebAPI(bully *Bully, showPort, unixTime bool) *WebAPI {
	ret := new(WebAPI)
	ret.bully = bully
	ret.showPort = showPort
	ret.unixTime = unixTime
	return ret
}

func (self *WebAPI) join(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Not implemented\r\n")
}

func (self *WebAPI) leader(w http.ResponseWriter, r *http.Request) {
	leader, timestamp, err := self.bully.Leader()
	if err != nil {
		fmt.Fprint(w, "Error: %v\r\n", err)
	}
	var leaderAddr string
	imleader := "remote"
	if self.bully.MyId().Cmp(leader.Id) == 0 {
		imleader = "local"
		if len(leader.Addr) == 0 {
			leaderAddr = self.bully.MyAddr()
		} else {
			leaderAddr = leader.Addr
		}
	} else {
		leaderAddr = leader.Addr
	}

	if !self.showPort {
		ae := strings.Split(leaderAddr, ":")
		if len(ae) > 1 {
			leaderAddr = strings.Join(ae[:len(ae)-1], ":")
		}
	}
	if self.unixTime {
		fmt.Fprintf(w, "%v\t%v\r\n%v\r\n", imleader, leaderAddr, timestamp.Unix())
	} else {
		fmt.Fprintf(w, "%v\t%v\r\n%v\r\n", imleader, leaderAddr, timestamp)
	}
}

func (self *WebAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()

	switch r.URL.Path {
	case newCandidate:
		self.join(w, r)
	case getLeader:
		self.leader(w, r)
	}
}

func (self *WebAPI) Run(addr string) {
	http.Handle(newCandidate, self)
	http.Handle(getLeader, self)
	err := http.ListenAndServe(addr, nil)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
	}
}
<|endoftext|>