<commit_before>// Copyright 2020 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package commonci

import (
	"context"
	"errors"
	"fmt"
	"log"
	"os"
	"strings"
	"time"

	"github.com/google/go-github/github"
	"golang.org/x/oauth2"
)

// GithubRequestHandler carries information relating to the GitHub session that
// is being used for the continuous integration.
type GithubRequestHandler struct {
	// hashSecret is the GitHub secret that is specified with the hook, it is
	// used to validate whether the response that is received is from GitHub.
	hashSecret string
	// Client is the connection to GitHub that should be utilised.
	client *github.Client
	// accessToken is the OAuth token that should be used for interactions with
	// the GitHub API and to retrieve repo contents.
	accessToken string
	labels      map[string]bool
}

// GithubPRUpdate is used to specify how an update to the status of a PR should
// be made with the UpdatePRStatus method.
type GithubPRUpdate struct {
	Owner       string
	Repo        string
	Ref         string
	NewStatus   string
	URL         string
	Description string
	Context     string
}

// Retry retries a function maxN times or when it returns true.
// In between each retry there is a small delay.
// This is intended to be used for posting results to GitHub from GCB, which
// frequently experiences errors likely due to connection issues.
func Retry(maxN uint, name string, f func() error) error {
	var err error
	for i := uint(0); i <= maxN; i++ {
		if err = f(); err == nil {
			return nil
		}
		log.Printf("Retry %d of %s, error: %v", i, name, err)
		time.Sleep(250 * time.Millisecond)
	}
	return err
}

// CreateCIOutputGist creates a GitHub Gist, and appends a comment with the
// result of the validator into it.  The function returns the URL and ID of the
// Gist that was created, and an error if experienced during processing.
func (g *GithubRequestHandler) CreateCIOutputGist(description, content string) (string, string, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second)
	defer cancel() // cancel context if the function returns before the timeout

	public := false
	// Create a new Gist struct - the description is used as the tag-line of
	// the created content, and the GistFilename (within the Files map) as
	// the input filename in the GitHub UI.
	gist := &github.Gist{
		Description: &description,
		Public:      &public,
		Files: map[github.GistFilename]github.GistFile{
			"oc-ci-run": {Content: &content},
		},
	}

	if err := Retry(5, fmt.Sprintf("gist creation for %s with content\n%s\n", description, content), func() error {
		var err error
		gist, _, err = g.client.Gists.Create(ctx, gist)
		return err
	}); err != nil {
		return "", "", fmt.Errorf("could not create gist: %s", err)
	}
	return *gist.HTMLURL, *gist.ID, nil
}

// AddGistComment adds a comment to a gist and returns its ID.
func (g *GithubRequestHandler) AddGistComment(gistID, title, output string) (int64, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
	defer cancel() // cancel context if the function returns before the timeout

	gistComment := fmt.Sprintf("# %s\n%s", title, output)

	var id int64
	if err := Retry(5, "gist comment creation", func() error {
		c, _, err := g.client.Gists.CreateComment(ctx, gistID, &github.GistComment{Body: &gistComment})
		if err != nil {
			return err
		}
		id = c.GetID()
		return nil
	}); err != nil {
		return 0, err
	}
	return id, nil
}

// UpdatePRStatus takes an input githubPRUpdate struct and updates a GitHub
// pull request's status with the relevant details. It returns an error if
// the update was not successful.
func (g *GithubRequestHandler) UpdatePRStatus(update *GithubPRUpdate) error {
	if !validStatuses[update.NewStatus] {
		return fmt.Errorf("invalid status %s", update.NewStatus)
	}

	if update.NewStatus == "" || update.Repo == "" || update.Ref == "" || update.Owner == "" {
		return fmt.Errorf("must specify required fields (status (%s), repo (%s), reference (%s) and owner (%s)) for update", update.NewStatus, update.Repo, update.Ref, update.Owner)
	}

	// The go-github library takes string pointers within the struct, and hence
	// we have to provide everything as a pointer.
	status := &github.RepoStatus{
		State:       &update.NewStatus,
		TargetURL:   &update.URL,
		Description: &update.Description,
	}
	ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
	defer cancel() // cancel context if the function returns before the timeout

	// Context is an optional argument.
	if update.Context != "" {
		status.Context = &update.Context
	}

	if update.Description != "" {
		status.Description = &update.Description
	}

	return Retry(5, "PR status update", func() error {
		_, _, err := g.client.Repositories.CreateStatus(ctx, update.Owner, update.Repo, update.Ref, status)
		return err
	})
}

// IsPRApproved checks whether a PR is approved or not.
// TODO: If this function is actually used, it should undergo testing due to having some logic.
// unit tests can be created based onon actual models-ci repo data that's sent back for a particular PR.
func (g *GithubRequestHandler) IsPRApproved(owner, repo string, prNumber int) (bool, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
	defer cancel() // cancel context if the function returns before the timeout
	var reviews []*github.PullRequestReview
	if err := Retry(5, "get PR reviews list", func() error {
		var err error
		reviews, _, err = g.client.PullRequests.ListReviews(ctx, owner, repo, prNumber, nil)
		return err
	}); err != nil {
		return false, err
	}

	for i := len(reviews) - 1; i != -1; i-- {
		review := reviews[i]
		switch strings.ToLower(review.GetState()) {
		case "approved":
			return true, nil
		case "changes_requested":
			return false, nil
		}
	}
	return false, nil
}

// PostLabel posts the given label to the PR. It is idempotent.
// unit tests can be created based onon actual models-ci repo data that's sent back.
func (g *GithubRequestHandler) PostLabel(labelName, labelColor, owner, repo string, prNumber int) error {
	if g.labels[labelName] {
		// Label already exists.
		return nil
	}

	label := &github.Label{Name: &labelName, Color: &labelColor}
	ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
	defer cancel()

	// Label may very well already exist within the repo, so skip creation if we see it.
	_, _, err := g.client.Issues.GetLabel(ctx, owner, repo, labelName)
	if err != nil {
		if err := Retry(5, "creating label", func() error {
			_, _, err = g.client.Issues.CreateLabel(ctx, owner, repo, label)
			return err
		}); err != nil {
			return err
		}
	}

	err = Retry(5, "adding label to PR", func() error {
		_, _, err = g.client.Issues.AddLabelsToIssue(ctx, owner, repo, prNumber, []string{labelName})
		return err
	})
	if err == nil {
		g.labels[labelName] = true
	}

	return err
}

// DeleteLabel removes the given label from the PR. It does not remove the
// label from the repo.
func (g *GithubRequestHandler) DeleteLabel(labelName, owner, repo string, prNumber int) error {
	ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
	defer cancel()
	if err := Retry(5, "removing label from PR", func() error {
		_, err := g.client.Issues.RemoveLabelForIssue(ctx, owner, repo, prNumber, labelName)
		return err
	}); err != nil {
		return err
	}

	// Do not take the second step to delete the label from the repo as
	// we're only interested in deleting the label from the PR.

	delete(g.labels, labelName)
	return nil
}

// AddPRComment posts a comment to the PR.
func (g *GithubRequestHandler) AddPRComment(body *string, owner, repo string, prNumber int) error {
	ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
	defer cancel()
	if err := Retry(5, "posting issue comment to PR", func() error {
		_, _, err := g.client.Issues.CreateComment(ctx, owner, repo, prNumber, &github.IssueComment{Body: body})
		return err
	}); err != nil {
		return err
	}
	return nil
}

// AddOrEditPRComment posts or edits a comment to a PR.
// If signature is empty, it's equivalent to AddPRComment.
// Otherwise, all comments are searched first, and if any matches the
// signature, then the comment is edited. If none matches, then a new comment
// is posted.
func (g *GithubRequestHandler) AddOrEditPRComment(signature string, body *string, owner, repo string, prNumber int) error {
	if signature == "" {
		return g.AddPRComment(body, owner, repo, prNumber)
	}

	ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
	defer cancel()

	var comments []*github.IssueComment
	if err := Retry(5, "get PR comments list", func() error {
		var err error
		comments, _, err = g.client.Issues.ListComments(ctx, owner, repo, prNumber, nil)
		return err
	}); err != nil {
		// If somehow this fails, we should be resilient and just post another comment.
		return g.AddPRComment(body, owner, repo, prNumber)
	}

	for _, pc := range comments {
		if strings.Contains(*pc.Body, signature) {
			if err := Retry(5, "edit PR comment", func() error {
				_, _, err := g.client.Issues.EditComment(ctx, owner, repo, *pc.ID, &github.IssueComment{Body: body})
				return err
			}); err != nil {
				return err
			}
		}
	}
	return g.AddPRComment(body, owner, repo, prNumber)
}

// NewGitHubRequestHandler sets up a new GithubRequestHandler struct which
// creates an oauth2 client with a GitHub access token (as specified by the
// GITHUB_ACCESS_TOKEN environment variable), and a connection to the GitHub
// API through the github.com/google/go-github/github library. It returns the
// initialised GithubRequestHandler struct, or an error as to why the
// initialisation failed.
func NewGitHubRequestHandler() (*GithubRequestHandler, error) {
	accesstk := os.Getenv("GITHUB_ACCESS_TOKEN")
	if accesstk == "" {
		return nil, errors.New("newGitHubRequestHandler: invalid access token environment variable set")
	}

	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: accesstk},
	)
	tc := oauth2.NewClient(oauth2.NoContext, ts)

	// Set the timeout for the oauth client such that we do not hang around
	// waiting for the client to complete.
	tc.Timeout = 2 * time.Second

	// Create a new GitHub client using the go-github library.
	client := github.NewClient(tc)
	return &GithubRequestHandler{
		// If the environment variable GITHUB_SECRET was set then we store it in
		// the struct, this is a secret that is used to calculate a hash of the
		// message so that we can validate it.
		client:      client,
		accessToken: accesstk,
		labels:      map[string]bool{},
	}, nil
}
<commit_msg>fix-up<commit_after>// Copyright 2020 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package commonci

import (
	"context"
	"errors"
	"fmt"
	"log"
	"os"
	"strings"
	"time"

	"github.com/google/go-github/github"
	"golang.org/x/oauth2"
)

// GithubRequestHandler carries information relating to the GitHub session that
// is being used for the continuous integration.
type GithubRequestHandler struct {
	// hashSecret is the GitHub secret that is specified with the hook, it is
	// used to validate whether the response that is received is from GitHub.
	hashSecret string
	// Client is the connection to GitHub that should be utilised.
	client *github.Client
	// accessToken is the OAuth token that should be used for interactions with
	// the GitHub API and to retrieve repo contents.
	accessToken string
	labels      map[string]bool
}

// GithubPRUpdate is used to specify how an update to the status of a PR should
// be made with the UpdatePRStatus method.
type GithubPRUpdate struct {
	Owner       string
	Repo        string
	Ref         string
	NewStatus   string
	URL         string
	Description string
	Context     string
}

// Retry retries a function maxN times or when it returns true.
// In between each retry there is a small delay.
// This is intended to be used for posting results to GitHub from GCB, which
// frequently experiences errors likely due to connection issues.
func Retry(maxN uint, name string, f func() error) error {
	var err error
	for i := uint(0); i <= maxN; i++ {
		if err = f(); err == nil {
			return nil
		}
		log.Printf("Retry %d of %s, error: %v", i, name, err)
		time.Sleep(250 * time.Millisecond)
	}
	return err
}

// CreateCIOutputGist creates a GitHub Gist, and appends a comment with the
// result of the validator into it.  The function returns the URL and ID of the
// Gist that was created, and an error if experienced during processing.
func (g *GithubRequestHandler) CreateCIOutputGist(description, content string) (string, string, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second)
	defer cancel() // cancel context if the function returns before the timeout

	public := false
	// Create a new Gist struct - the description is used as the tag-line of
	// the created content, and the GistFilename (within the Files map) as
	// the input filename in the GitHub UI.
	gist := &github.Gist{
		Description: &description,
		Public:      &public,
		Files: map[github.GistFilename]github.GistFile{
			"oc-ci-run": {Content: &content},
		},
	}

	if err := Retry(5, fmt.Sprintf("gist creation for %s with content\n%s\n", description, content), func() error {
		var err error
		gist, _, err = g.client.Gists.Create(ctx, gist)
		return err
	}); err != nil {
		return "", "", fmt.Errorf("could not create gist: %s", err)
	}
	return *gist.HTMLURL, *gist.ID, nil
}

// AddGistComment adds a comment to a gist and returns its ID.
func (g *GithubRequestHandler) AddGistComment(gistID, title, output string) (int64, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
	defer cancel() // cancel context if the function returns before the timeout

	gistComment := fmt.Sprintf("# %s\n%s", title, output)

	var id int64
	if err := Retry(5, "gist comment creation", func() error {
		c, _, err := g.client.Gists.CreateComment(ctx, gistID, &github.GistComment{Body: &gistComment})
		if err != nil {
			return err
		}
		id = c.GetID()
		return nil
	}); err != nil {
		return 0, err
	}
	return id, nil
}

// UpdatePRStatus takes an input githubPRUpdate struct and updates a GitHub
// pull request's status with the relevant details. It returns an error if
// the update was not successful.
func (g *GithubRequestHandler) UpdatePRStatus(update *GithubPRUpdate) error {
	if !validStatuses[update.NewStatus] {
		return fmt.Errorf("invalid status %s", update.NewStatus)
	}

	if update.NewStatus == "" || update.Repo == "" || update.Ref == "" || update.Owner == "" {
		return fmt.Errorf("must specify required fields (status (%s), repo (%s), reference (%s) and owner (%s)) for update", update.NewStatus, update.Repo, update.Ref, update.Owner)
	}

	// The go-github library takes string pointers within the struct, and hence
	// we have to provide everything as a pointer.
	status := &github.RepoStatus{
		State:       &update.NewStatus,
		TargetURL:   &update.URL,
		Description: &update.Description,
	}
	ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
	defer cancel() // cancel context if the function returns before the timeout

	// Context is an optional argument.
	if update.Context != "" {
		status.Context = &update.Context
	}

	if update.Description != "" {
		status.Description = &update.Description
	}

	return Retry(5, "PR status update", func() error {
		_, _, err := g.client.Repositories.CreateStatus(ctx, update.Owner, update.Repo, update.Ref, status)
		return err
	})
}

// IsPRApproved checks whether a PR is approved or not.
// TODO: If this function is actually used, it should undergo testing due to having some logic.
// unit tests can be created based onon actual models-ci repo data that's sent back for a particular PR.
func (g *GithubRequestHandler) IsPRApproved(owner, repo string, prNumber int) (bool, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
	defer cancel() // cancel context if the function returns before the timeout
	var reviews []*github.PullRequestReview
	if err := Retry(5, "get PR reviews list", func() error {
		var err error
		reviews, _, err = g.client.PullRequests.ListReviews(ctx, owner, repo, prNumber, nil)
		return err
	}); err != nil {
		return false, err
	}

	for i := len(reviews) - 1; i != -1; i-- {
		review := reviews[i]
		switch strings.ToLower(review.GetState()) {
		case "approved":
			return true, nil
		case "changes_requested":
			return false, nil
		}
	}
	return false, nil
}

// PostLabel posts the given label to the PR. It is idempotent.
// unit tests can be created based onon actual models-ci repo data that's sent back.
func (g *GithubRequestHandler) PostLabel(labelName, labelColor, owner, repo string, prNumber int) error {
	if g.labels[labelName] {
		// Label already exists.
		return nil
	}

	label := &github.Label{Name: &labelName, Color: &labelColor}
	ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
	defer cancel()

	// Label may very well already exist within the repo, so skip creation if we see it.
	_, _, err := g.client.Issues.GetLabel(ctx, owner, repo, labelName)
	if err != nil {
		if err := Retry(5, "creating label", func() error {
			_, _, err = g.client.Issues.CreateLabel(ctx, owner, repo, label)
			return err
		}); err != nil {
			return err
		}
	}

	err = Retry(5, "adding label to PR", func() error {
		_, _, err = g.client.Issues.AddLabelsToIssue(ctx, owner, repo, prNumber, []string{labelName})
		return err
	})
	if err == nil {
		g.labels[labelName] = true
	}

	return err
}

// DeleteLabel removes the given label from the PR. It does not remove the
// label from the repo.
func (g *GithubRequestHandler) DeleteLabel(labelName, owner, repo string, prNumber int) error {
	ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
	defer cancel()
	if err := Retry(5, "removing label from PR", func() error {
		_, err := g.client.Issues.RemoveLabelForIssue(ctx, owner, repo, prNumber, labelName)
		return err
	}); err != nil {
		return err
	}

	// Do not take the second step to delete the label from the repo as
	// we're only interested in deleting the label from the PR.

	delete(g.labels, labelName)
	return nil
}

// AddPRComment posts a comment to the PR.
func (g *GithubRequestHandler) AddPRComment(body *string, owner, repo string, prNumber int) error {
	ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
	defer cancel()
	if err := Retry(5, "posting issue comment to PR", func() error {
		_, _, err := g.client.Issues.CreateComment(ctx, owner, repo, prNumber, &github.IssueComment{Body: body})
		return err
	}); err != nil {
		return err
	}
	return nil
}

// AddOrEditPRComment posts or edits a comment to a PR.
// If signature is empty, it's equivalent to AddPRComment.
// Otherwise, all comments are searched first, and if any matches the
// signature, then the comment is edited. If none matches, then a new comment
// is posted.
func (g *GithubRequestHandler) AddOrEditPRComment(signature string, body *string, owner, repo string, prNumber int) error {
	if signature == "" {
		return g.AddPRComment(body, owner, repo, prNumber)
	}

	ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
	defer cancel()

	var comments []*github.IssueComment
	if err := Retry(5, "get PR comments list", func() error {
		var err error
		comments, _, err = g.client.Issues.ListComments(ctx, owner, repo, prNumber, nil)
		return err
	}); err != nil {
		// If somehow this fails, we should be resilient and just post another comment.
		return g.AddPRComment(body, owner, repo, prNumber)
	}

	for _, pc := range comments {
		if strings.Contains(*pc.Body, signature) {
			if err := Retry(5, "edit PR comment", func() error {
				_, _, err := g.client.Issues.EditComment(ctx, owner, repo, *pc.ID, &github.IssueComment{Body: body})
				return err
			}); err != nil {
				return g.AddPRComment(body, owner, repo, prNumber)
			}
			return nil
		}
	}
	return g.AddPRComment(body, owner, repo, prNumber)
}

// NewGitHubRequestHandler sets up a new GithubRequestHandler struct which
// creates an oauth2 client with a GitHub access token (as specified by the
// GITHUB_ACCESS_TOKEN environment variable), and a connection to the GitHub
// API through the github.com/google/go-github/github library. It returns the
// initialised GithubRequestHandler struct, or an error as to why the
// initialisation failed.
func NewGitHubRequestHandler() (*GithubRequestHandler, error) {
	accesstk := os.Getenv("GITHUB_ACCESS_TOKEN")
	if accesstk == "" {
		return nil, errors.New("newGitHubRequestHandler: invalid access token environment variable set")
	}

	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: accesstk},
	)
	tc := oauth2.NewClient(oauth2.NoContext, ts)

	// Set the timeout for the oauth client such that we do not hang around
	// waiting for the client to complete.
	tc.Timeout = 2 * time.Second

	// Create a new GitHub client using the go-github library.
	client := github.NewClient(tc)
	return &GithubRequestHandler{
		// If the environment variable GITHUB_SECRET was set then we store it in
		// the struct, this is a secret that is used to calculate a hash of the
		// message so that we can validate it.
		client:      client,
		accessToken: accesstk,
		labels:      map[string]bool{},
	}, nil
}
<|endoftext|>


<commit_before>package common

import (
	"errors"
	"fmt"
	"net"
	"os"
	"regexp"
	"time"

	"github.com/hashicorp/packer/common/uuid"
	"github.com/hashicorp/packer/helper/communicator"
	"github.com/hashicorp/packer/template/interpolate"
)

var reShutdownBehavior = regexp.MustCompile("^(stop|terminate)$")

type AmiFilterOptions struct {
	Filters    map[*string]*string
	Owners     []*string
	MostRecent bool `mapstructure:"most_recent"`
}

func (d *AmiFilterOptions) Empty() bool {
	return len(d.Owners) == 0 && len(d.Filters) == 0
}

// RunConfig contains configuration for running an instance from a source
// AMI and details on how to access that launched image.
type RunConfig struct {
	AssociatePublicIpAddress          bool              `mapstructure:"associate_public_ip_address"`
	AvailabilityZone                  string            `mapstructure:"availability_zone"`
	EbsOptimized                      bool              `mapstructure:"ebs_optimized"`
	IamInstanceProfile                string            `mapstructure:"iam_instance_profile"`
	InstanceType                      string            `mapstructure:"instance_type"`
	RunTags                           map[string]string `mapstructure:"run_tags"`
	SourceAmi                         string            `mapstructure:"source_ami"`
	SourceAmiFilter                   AmiFilterOptions  `mapstructure:"source_ami_filter"`
	SpotPrice                         string            `mapstructure:"spot_price"`
	SpotPriceAutoProduct              string            `mapstructure:"spot_price_auto_product"`
	DisableStopInstance               bool              `mapstructure:"disable_stop_instance"`
	SecurityGroupId                   string            `mapstructure:"security_group_id"`
	SecurityGroupIds                  []string          `mapstructure:"security_group_ids"`
	TemporarySGSourceCidr             string            `mapstructure:"temporary_security_group_source_cidr"`
	SubnetId                          string            `mapstructure:"subnet_id"`
	TemporaryKeyPairName              string            `mapstructure:"temporary_key_pair_name"`
	UserData                          string            `mapstructure:"user_data"`
	UserDataFile                      string            `mapstructure:"user_data_file"`
	WindowsPasswordTimeout            time.Duration     `mapstructure:"windows_password_timeout"`
	VpcId                             string            `mapstructure:"vpc_id"`
	InstanceInitiatedShutdownBehavior string            `mapstructure:"shutdown_behavior"`

	// Communicator settings
	Comm           communicator.Config `mapstructure:",squash"`
	SSHKeyPairName string              `mapstructure:"ssh_keypair_name"`
	SSHInterface   string              `mapstructure:"ssh_interface"`
}

func (c *RunConfig) Prepare(ctx *interpolate.Context) []error {
	// If we are not given an explicit ssh_keypair_name or
	// ssh_private_key_file, then create a temporary one, but only if the
	// temporary_key_pair_name has not been provided and we are not using
	// ssh_password.
	if c.SSHKeyPairName == "" && c.TemporaryKeyPairName == "" &&
		c.Comm.SSHPrivateKey == "" && c.Comm.SSHPassword == "" {

		c.TemporaryKeyPairName = fmt.Sprintf("packer_%s", uuid.TimeOrderedUUID())
	}

	if c.WindowsPasswordTimeout == 0 {
		c.WindowsPasswordTimeout = 20 * time.Minute
	}

	if c.RunTags == nil {
		c.RunTags = make(map[string]string)
	}

	// Validation
	errs := c.Comm.Prepare(ctx)

	// Valadating ssh_interface
	if c.SSHInterface != "public_ip" &&
		c.SSHInterface != "private_ip" &&
		c.SSHInterface != "public_dns" &&
		c.SSHInterface != "private_dns" &&
		c.SSHInterface != "" {
		errs = append(errs, errors.New(fmt.Sprintf("Unknown interface type: %s", c.SSHInterface)))
	}

	if c.SSHKeyPairName != "" {
		if c.Comm.Type == "winrm" && c.Comm.WinRMPassword == "" && c.Comm.SSHPrivateKey == "" {
			errs = append(errs, errors.New("ssh_private_key_file must be provided to retrieve the winrm password when using ssh_keypair_name."))
		} else if c.Comm.SSHPrivateKey == "" && !c.Comm.SSHAgentAuth {
			errs = append(errs, errors.New("ssh_private_key_file must be provided or ssh_agent_auth enabled when ssh_keypair_name is specified."))
		}
	}

	if c.SourceAmi == "" && c.SourceAmiFilter.Empty() {
		errs = append(errs, errors.New("A source_ami or source_ami_filter must be specified"))
	}

	if c.InstanceType == "" {
		errs = append(errs, errors.New("An instance_type must be specified"))
	}

	if c.SpotPrice == "auto" {
		if c.SpotPriceAutoProduct == "" {
			errs = append(errs, errors.New(
				"spot_price_auto_product must be specified when spot_price is auto"))
		}
	}

	if c.UserData != "" && c.UserDataFile != "" {
		errs = append(errs, fmt.Errorf("Only one of user_data or user_data_file can be specified."))
	} else if c.UserDataFile != "" {
		if _, err := os.Stat(c.UserDataFile); err != nil {
			errs = append(errs, fmt.Errorf("user_data_file not found: %s", c.UserDataFile))
		}
	}

	if c.SecurityGroupId != "" {
		if len(c.SecurityGroupIds) > 0 {
			errs = append(errs, fmt.Errorf("Only one of security_group_id or security_group_ids can be specified."))
		} else {
			c.SecurityGroupIds = []string{c.SecurityGroupId}
			c.SecurityGroupId = ""
		}
	}

	if c.TemporarySGSourceCidr == "" {
		c.TemporarySGSourceCidr = "0.0.0.0/0"
	} else {
		if _, _, err := net.ParseCIDR(c.TemporarySGSourceCidr); err != nil {
			errs = append(errs, fmt.Errorf("Error parsing temporary_security_group_source_cidr: %s", err.Error()))
		}
	}

	if c.InstanceInitiatedShutdownBehavior == "" {
		c.InstanceInitiatedShutdownBehavior = "stop"
	} else if !reShutdownBehavior.MatchString(c.InstanceInitiatedShutdownBehavior) {
		errs = append(errs, fmt.Errorf("shutdown_behavior only accepts 'stop' or 'terminate' values."))
	}

	return errs
}

func (c *RunConfig) IsSpotInstance() bool {
	return c.SpotPrice != "" && c.SpotPrice != "0"
}
<commit_msg>spelling: validating<commit_after>package common

import (
	"errors"
	"fmt"
	"net"
	"os"
	"regexp"
	"time"

	"github.com/hashicorp/packer/common/uuid"
	"github.com/hashicorp/packer/helper/communicator"
	"github.com/hashicorp/packer/template/interpolate"
)

var reShutdownBehavior = regexp.MustCompile("^(stop|terminate)$")

type AmiFilterOptions struct {
	Filters    map[*string]*string
	Owners     []*string
	MostRecent bool `mapstructure:"most_recent"`
}

func (d *AmiFilterOptions) Empty() bool {
	return len(d.Owners) == 0 && len(d.Filters) == 0
}

// RunConfig contains configuration for running an instance from a source
// AMI and details on how to access that launched image.
type RunConfig struct {
	AssociatePublicIpAddress          bool              `mapstructure:"associate_public_ip_address"`
	AvailabilityZone                  string            `mapstructure:"availability_zone"`
	EbsOptimized                      bool              `mapstructure:"ebs_optimized"`
	IamInstanceProfile                string            `mapstructure:"iam_instance_profile"`
	InstanceType                      string            `mapstructure:"instance_type"`
	RunTags                           map[string]string `mapstructure:"run_tags"`
	SourceAmi                         string            `mapstructure:"source_ami"`
	SourceAmiFilter                   AmiFilterOptions  `mapstructure:"source_ami_filter"`
	SpotPrice                         string            `mapstructure:"spot_price"`
	SpotPriceAutoProduct              string            `mapstructure:"spot_price_auto_product"`
	DisableStopInstance               bool              `mapstructure:"disable_stop_instance"`
	SecurityGroupId                   string            `mapstructure:"security_group_id"`
	SecurityGroupIds                  []string          `mapstructure:"security_group_ids"`
	TemporarySGSourceCidr             string            `mapstructure:"temporary_security_group_source_cidr"`
	SubnetId                          string            `mapstructure:"subnet_id"`
	TemporaryKeyPairName              string            `mapstructure:"temporary_key_pair_name"`
	UserData                          string            `mapstructure:"user_data"`
	UserDataFile                      string            `mapstructure:"user_data_file"`
	WindowsPasswordTimeout            time.Duration     `mapstructure:"windows_password_timeout"`
	VpcId                             string            `mapstructure:"vpc_id"`
	InstanceInitiatedShutdownBehavior string            `mapstructure:"shutdown_behavior"`

	// Communicator settings
	Comm           communicator.Config `mapstructure:",squash"`
	SSHKeyPairName string              `mapstructure:"ssh_keypair_name"`
	SSHInterface   string              `mapstructure:"ssh_interface"`
}

func (c *RunConfig) Prepare(ctx *interpolate.Context) []error {
	// If we are not given an explicit ssh_keypair_name or
	// ssh_private_key_file, then create a temporary one, but only if the
	// temporary_key_pair_name has not been provided and we are not using
	// ssh_password.
	if c.SSHKeyPairName == "" && c.TemporaryKeyPairName == "" &&
		c.Comm.SSHPrivateKey == "" && c.Comm.SSHPassword == "" {

		c.TemporaryKeyPairName = fmt.Sprintf("packer_%s", uuid.TimeOrderedUUID())
	}

	if c.WindowsPasswordTimeout == 0 {
		c.WindowsPasswordTimeout = 20 * time.Minute
	}

	if c.RunTags == nil {
		c.RunTags = make(map[string]string)
	}

	// Validation
	errs := c.Comm.Prepare(ctx)

	// Validating ssh_interface
	if c.SSHInterface != "public_ip" &&
		c.SSHInterface != "private_ip" &&
		c.SSHInterface != "public_dns" &&
		c.SSHInterface != "private_dns" &&
		c.SSHInterface != "" {
		errs = append(errs, errors.New(fmt.Sprintf("Unknown interface type: %s", c.SSHInterface)))
	}

	if c.SSHKeyPairName != "" {
		if c.Comm.Type == "winrm" && c.Comm.WinRMPassword == "" && c.Comm.SSHPrivateKey == "" {
			errs = append(errs, errors.New("ssh_private_key_file must be provided to retrieve the winrm password when using ssh_keypair_name."))
		} else if c.Comm.SSHPrivateKey == "" && !c.Comm.SSHAgentAuth {
			errs = append(errs, errors.New("ssh_private_key_file must be provided or ssh_agent_auth enabled when ssh_keypair_name is specified."))
		}
	}

	if c.SourceAmi == "" && c.SourceAmiFilter.Empty() {
		errs = append(errs, errors.New("A source_ami or source_ami_filter must be specified"))
	}

	if c.InstanceType == "" {
		errs = append(errs, errors.New("An instance_type must be specified"))
	}

	if c.SpotPrice == "auto" {
		if c.SpotPriceAutoProduct == "" {
			errs = append(errs, errors.New(
				"spot_price_auto_product must be specified when spot_price is auto"))
		}
	}

	if c.UserData != "" && c.UserDataFile != "" {
		errs = append(errs, fmt.Errorf("Only one of user_data or user_data_file can be specified."))
	} else if c.UserDataFile != "" {
		if _, err := os.Stat(c.UserDataFile); err != nil {
			errs = append(errs, fmt.Errorf("user_data_file not found: %s", c.UserDataFile))
		}
	}

	if c.SecurityGroupId != "" {
		if len(c.SecurityGroupIds) > 0 {
			errs = append(errs, fmt.Errorf("Only one of security_group_id or security_group_ids can be specified."))
		} else {
			c.SecurityGroupIds = []string{c.SecurityGroupId}
			c.SecurityGroupId = ""
		}
	}

	if c.TemporarySGSourceCidr == "" {
		c.TemporarySGSourceCidr = "0.0.0.0/0"
	} else {
		if _, _, err := net.ParseCIDR(c.TemporarySGSourceCidr); err != nil {
			errs = append(errs, fmt.Errorf("Error parsing temporary_security_group_source_cidr: %s", err.Error()))
		}
	}

	if c.InstanceInitiatedShutdownBehavior == "" {
		c.InstanceInitiatedShutdownBehavior = "stop"
	} else if !reShutdownBehavior.MatchString(c.InstanceInitiatedShutdownBehavior) {
		errs = append(errs, fmt.Errorf("shutdown_behavior only accepts 'stop' or 'terminate' values."))
	}

	return errs
}

func (c *RunConfig) IsSpotInstance() bool {
	return c.SpotPrice != "" && c.SpotPrice != "0"
}
<|endoftext|>


<commit_before>package googlecompute

import (
	"fmt"
	"log"
	"net/http"
	"runtime"
	"time"

	"github.com/mitchellh/packer/packer"

	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"golang.org/x/oauth2/jwt"
	"google.golang.org/api/compute/v1"
)

// driverGCE is a Driver implementation that actually talks to GCE.
// Create an instance using NewDriverGCE.
type driverGCE struct {
	projectId string
	service   *compute.Service
	ui        packer.Ui
}

var DriverScopes = []string{"https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/devstorage.full_control"}

func NewDriverGCE(ui packer.Ui, p string, a *accountFile) (Driver, error) {
	var err error

	var client *http.Client

	// Auth with AccountFile first if provided
	if a.PrivateKey != "" {
		log.Printf("[INFO] Requesting Google token via AccountFile...")
		log.Printf("[INFO]   -- Email: %s", a.ClientEmail)
		log.Printf("[INFO]   -- Scopes: %s", DriverScopes)
		log.Printf("[INFO]   -- Private Key Length: %d", len(a.PrivateKey))

		conf := jwt.Config{
			Email:      a.ClientEmail,
			PrivateKey: []byte(a.PrivateKey),
			Scopes:     DriverScopes,
			TokenURL:   "https://accounts.google.com/o/oauth2/token",
		}

		// Initiate an http.Client. The following GET request will be
		// authorized and authenticated on the behalf of
		// your service account.
		client = conf.Client(oauth2.NoContext)
	} else {
		log.Printf("[INFO] Requesting Google token via GCE Service Role...")
		client = &http.Client{
			Transport: &oauth2.Transport{
				// Fetch from Google Compute Engine's metadata server to retrieve
				// an access token for the provided account.
				// If no account is specified, "default" is used.
				Source: google.ComputeTokenSource(""),
			},
		}
	}

	if err != nil {
		return nil, err
	}

	log.Printf("[INFO] Instantiating GCE client...")
	service, err := compute.New(client)
	// Set UserAgent
	versionString := "0.0.0"
	// TODO(dcunnin): Use Packer's version code from version.go
	// versionString := main.Version
	// if main.VersionPrerelease != "" {
	//      versionString = fmt.Sprintf("%s-%s", versionString, main.VersionPrerelease)
	// }
	service.UserAgent = fmt.Sprintf(
		"(%s %s) Packer/%s", runtime.GOOS, runtime.GOARCH, versionString)

	if err != nil {
		return nil, err
	}

	return &driverGCE{
		projectId: p,
		service:   service,
		ui:        ui,
	}, nil
}

func (d *driverGCE) ImageExists(name string) bool {
	_, err := d.service.Images.Get(d.projectId, name).Do()
	// The API may return an error for reasons other than the image not
	// existing, but this heuristic is sufficient for now.
	return err == nil
}

func (d *driverGCE) CreateImage(name, description, zone, disk string) <-chan error {
	image := &compute.Image{
		Description: description,
		Name:        name,
		SourceDisk:  fmt.Sprintf("%s%s/zones/%s/disks/%s", d.service.BasePath, d.projectId, zone, disk),
		SourceType:  "RAW",
	}

	errCh := make(chan error, 1)
	op, err := d.service.Images.Insert(d.projectId, image).Do()
	if err != nil {
		errCh <- err
	} else {
		go waitForState(errCh, "DONE", d.refreshGlobalOp(op))
	}

	return errCh
}

func (d *driverGCE) DeleteImage(name string) <-chan error {
	errCh := make(chan error, 1)
	op, err := d.service.Images.Delete(d.projectId, name).Do()
	if err != nil {
		errCh <- err
	} else {
		go waitForState(errCh, "DONE", d.refreshGlobalOp(op))
	}

	return errCh
}

func (d *driverGCE) DeleteInstance(zone, name string) (<-chan error, error) {
	op, err := d.service.Instances.Delete(d.projectId, zone, name).Do()
	if err != nil {
		return nil, err
	}

	errCh := make(chan error, 1)
	go waitForState(errCh, "DONE", d.refreshZoneOp(zone, op))
	return errCh, nil
}

func (d *driverGCE) DeleteDisk(zone, name string) (<-chan error, error) {
	op, err := d.service.Disks.Delete(d.projectId, zone, name).Do()
	if err != nil {
		return nil, err
	}

	errCh := make(chan error, 1)
	go waitForState(errCh, "DONE", d.refreshZoneOp(zone, op))
	return errCh, nil
}

func (d *driverGCE) GetNatIP(zone, name string) (string, error) {
	instance, err := d.service.Instances.Get(d.projectId, zone, name).Do()
	if err != nil {
		return "", err
	}

	for _, ni := range instance.NetworkInterfaces {
		if ni.AccessConfigs == nil {
			continue
		}
		for _, ac := range ni.AccessConfigs {
			if ac.NatIP != "" {
				return ac.NatIP, nil
			}
		}
	}

	return "", nil
}

func (d *driverGCE) GetInternalIP(zone, name string) (string, error) {
	instance, err := d.service.Instances.Get(d.projectId, zone, name).Do()
	if err != nil {
		return "", err
	}

	for _, ni := range instance.NetworkInterfaces {
		if ni.NetworkIP == "" {
			continue
		}
		return ni.NetworkIP, nil
	}

	return "", nil
}

func (d *driverGCE) RunInstance(c *InstanceConfig) (<-chan error, error) {
	// Get the zone
	d.ui.Message(fmt.Sprintf("Loading zone: %s", c.Zone))
	zone, err := d.service.Zones.Get(d.projectId, c.Zone).Do()
	if err != nil {
		return nil, err
	}

	// Get the image
	d.ui.Message(fmt.Sprintf("Loading image: %s in project %s", c.Image.Name, c.Image.ProjectId))
	image, err := d.getImage(c.Image)
	if err != nil {
		return nil, err
	}

	// Get the machine type
	d.ui.Message(fmt.Sprintf("Loading machine type: %s", c.MachineType))
	machineType, err := d.service.MachineTypes.Get(
		d.projectId, zone.Name, c.MachineType).Do()
	if err != nil {
		return nil, err
	}
	// TODO(mitchellh): deprecation warnings

	// Get the network
	d.ui.Message(fmt.Sprintf("Loading network: %s", c.Network))
	network, err := d.service.Networks.Get(d.projectId, c.Network).Do()
	if err != nil {
		return nil, err
	}

	// Build up the metadata
	metadata := make([]*compute.MetadataItems, len(c.Metadata))
	for k, v := range c.Metadata {
		metadata = append(metadata, &compute.MetadataItems{
			Key:   k,
			Value: v,
		})
	}

	// Create the instance information
	instance := compute.Instance{
		Description: c.Description,
		Disks: []*compute.AttachedDisk{
			&compute.AttachedDisk{
				Type:       "PERSISTENT",
				Mode:       "READ_WRITE",
				Kind:       "compute#attachedDisk",
				Boot:       true,
				AutoDelete: false,
				InitializeParams: &compute.AttachedDiskInitializeParams{
					SourceImage: image.SelfLink,
					DiskSizeGb:  c.DiskSizeGb,
				},
			},
		},
		MachineType: machineType.SelfLink,
		Metadata: &compute.Metadata{
			Items: metadata,
		},
		Name: c.Name,
		NetworkInterfaces: []*compute.NetworkInterface{
			&compute.NetworkInterface{
				AccessConfigs: []*compute.AccessConfig{
					&compute.AccessConfig{
						Name: "AccessConfig created by Packer",
						Type: "ONE_TO_ONE_NAT",
					},
				},
				Network: network.SelfLink,
			},
		},
		ServiceAccounts: []*compute.ServiceAccount{
			&compute.ServiceAccount{
				Email: "default",
				Scopes: []string{
					"https://www.googleapis.com/auth/userinfo.email",
					"https://www.googleapis.com/auth/compute",
					"https://www.googleapis.com/auth/devstorage.full_control",
				},
			},
		},
		Tags: &compute.Tags{
			Items: c.Tags,
		},
	}

	d.ui.Message("Requesting instance creation...")
	op, err := d.service.Instances.Insert(d.projectId, zone.Name, &instance).Do()
	if err != nil {
		return nil, err
	}

	errCh := make(chan error, 1)
	go waitForState(errCh, "DONE", d.refreshZoneOp(zone.Name, op))
	return errCh, nil
}

func (d *driverGCE) WaitForInstance(state, zone, name string) <-chan error {
	errCh := make(chan error, 1)
	go waitForState(errCh, state, d.refreshInstanceState(zone, name))
	return errCh
}

func (d *driverGCE) getImage(img Image) (image *compute.Image, err error) {
	projects := []string{img.ProjectId, "centos-cloud", "coreos-cloud", "debian-cloud", "google-containers", "opensuse-cloud", "rhel-cloud", "suse-cloud", "ubuntu-os-cloud", "windows-cloud"}
	for _, project := range projects {
		image, err = d.service.Images.Get(project, img.Name).Do()
		if err == nil && image != nil && image.SelfLink != "" {
			return
		}
		image = nil
	}

	err = fmt.Errorf("Image %s could not be found in any of these projects: %s", img.Name, projects)
	return
}

func (d *driverGCE) refreshInstanceState(zone, name string) stateRefreshFunc {
	return func() (string, error) {
		instance, err := d.service.Instances.Get(d.projectId, zone, name).Do()
		if err != nil {
			return "", err
		}
		return instance.Status, nil
	}
}

func (d *driverGCE) refreshGlobalOp(op *compute.Operation) stateRefreshFunc {
	return func() (string, error) {
		newOp, err := d.service.GlobalOperations.Get(d.projectId, op.Name).Do()
		if err != nil {
			return "", err
		}

		// If the op is done, check for errors
		err = nil
		if newOp.Status == "DONE" {
			if newOp.Error != nil {
				for _, e := range newOp.Error.Errors {
					err = packer.MultiErrorAppend(err, fmt.Errorf(e.Message))
				}
			}
		}

		return newOp.Status, err
	}
}

func (d *driverGCE) refreshZoneOp(zone string, op *compute.Operation) stateRefreshFunc {
	return func() (string, error) {
		newOp, err := d.service.ZoneOperations.Get(d.projectId, zone, op.Name).Do()
		if err != nil {
			return "", err
		}

		// If the op is done, check for errors
		err = nil
		if newOp.Status == "DONE" {
			if newOp.Error != nil {
				for _, e := range newOp.Error.Errors {
					err = packer.MultiErrorAppend(err, fmt.Errorf(e.Message))
				}
			}
		}

		return newOp.Status, err
	}
}

// stateRefreshFunc is used to refresh the state of a thing and is
// used in conjunction with waitForState.
type stateRefreshFunc func() (string, error)

// waitForState will spin in a loop forever waiting for state to
// reach a certain target.
func waitForState(errCh chan<- error, target string, refresh stateRefreshFunc) {
	for {
		state, err := refresh()
		if err != nil {
			errCh <- err
			return
		}
		if state == target {
			errCh <- nil
			return
		}

		time.Sleep(2 * time.Second)
	}
}
<commit_msg>Fixed GCE builder after dependency change.<commit_after>package googlecompute

import (
	"fmt"
	"log"
	"net/http"
	"runtime"
	"time"

	"github.com/mitchellh/packer/packer"

	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"golang.org/x/oauth2/jwt"
	"google.golang.org/api/compute/v1"
)

// driverGCE is a Driver implementation that actually talks to GCE.
// Create an instance using NewDriverGCE.
type driverGCE struct {
	projectId string
	service   *compute.Service
	ui        packer.Ui
}

var DriverScopes = []string{"https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/devstorage.full_control"}

func NewDriverGCE(ui packer.Ui, p string, a *accountFile) (Driver, error) {
	var err error

	var client *http.Client

	// Auth with AccountFile first if provided
	if a.PrivateKey != "" {
		log.Printf("[INFO] Requesting Google token via AccountFile...")
		log.Printf("[INFO]   -- Email: %s", a.ClientEmail)
		log.Printf("[INFO]   -- Scopes: %s", DriverScopes)
		log.Printf("[INFO]   -- Private Key Length: %d", len(a.PrivateKey))

		conf := jwt.Config{
			Email:      a.ClientEmail,
			PrivateKey: []byte(a.PrivateKey),
			Scopes:     DriverScopes,
			TokenURL:   "https://accounts.google.com/o/oauth2/token",
		}

		// Initiate an http.Client. The following GET request will be
		// authorized and authenticated on the behalf of
		// your service account.
		client = conf.Client(oauth2.NoContext)
	} else {
		log.Printf("[INFO] Requesting Google token via GCE Service Role...")
		client = &http.Client{
			Transport: &oauth2.Transport{
				// Fetch from Google Compute Engine's metadata server to retrieve
				// an access token for the provided account.
				// If no account is specified, "default" is used.
				Source: google.ComputeTokenSource(""),
			},
		}
	}

	if err != nil {
		return nil, err
	}

	log.Printf("[INFO] Instantiating GCE client...")
	service, err := compute.New(client)
	// Set UserAgent
	versionString := "0.0.0"
	// TODO(dcunnin): Use Packer's version code from version.go
	// versionString := main.Version
	// if main.VersionPrerelease != "" {
	//      versionString = fmt.Sprintf("%s-%s", versionString, main.VersionPrerelease)
	// }
	service.UserAgent = fmt.Sprintf(
		"(%s %s) Packer/%s", runtime.GOOS, runtime.GOARCH, versionString)

	if err != nil {
		return nil, err
	}

	return &driverGCE{
		projectId: p,
		service:   service,
		ui:        ui,
	}, nil
}

func (d *driverGCE) ImageExists(name string) bool {
	_, err := d.service.Images.Get(d.projectId, name).Do()
	// The API may return an error for reasons other than the image not
	// existing, but this heuristic is sufficient for now.
	return err == nil
}

func (d *driverGCE) CreateImage(name, description, zone, disk string) <-chan error {
	image := &compute.Image{
		Description: description,
		Name:        name,
		SourceDisk:  fmt.Sprintf("%s%s/zones/%s/disks/%s", d.service.BasePath, d.projectId, zone, disk),
		SourceType:  "RAW",
	}

	errCh := make(chan error, 1)
	op, err := d.service.Images.Insert(d.projectId, image).Do()
	if err != nil {
		errCh <- err
	} else {
		go waitForState(errCh, "DONE", d.refreshGlobalOp(op))
	}

	return errCh
}

func (d *driverGCE) DeleteImage(name string) <-chan error {
	errCh := make(chan error, 1)
	op, err := d.service.Images.Delete(d.projectId, name).Do()
	if err != nil {
		errCh <- err
	} else {
		go waitForState(errCh, "DONE", d.refreshGlobalOp(op))
	}

	return errCh
}

func (d *driverGCE) DeleteInstance(zone, name string) (<-chan error, error) {
	op, err := d.service.Instances.Delete(d.projectId, zone, name).Do()
	if err != nil {
		return nil, err
	}

	errCh := make(chan error, 1)
	go waitForState(errCh, "DONE", d.refreshZoneOp(zone, op))
	return errCh, nil
}

func (d *driverGCE) DeleteDisk(zone, name string) (<-chan error, error) {
	op, err := d.service.Disks.Delete(d.projectId, zone, name).Do()
	if err != nil {
		return nil, err
	}

	errCh := make(chan error, 1)
	go waitForState(errCh, "DONE", d.refreshZoneOp(zone, op))
	return errCh, nil
}

func (d *driverGCE) GetNatIP(zone, name string) (string, error) {
	instance, err := d.service.Instances.Get(d.projectId, zone, name).Do()
	if err != nil {
		return "", err
	}

	for _, ni := range instance.NetworkInterfaces {
		if ni.AccessConfigs == nil {
			continue
		}
		for _, ac := range ni.AccessConfigs {
			if ac.NatIP != "" {
				return ac.NatIP, nil
			}
		}
	}

	return "", nil
}

func (d *driverGCE) GetInternalIP(zone, name string) (string, error) {
	instance, err := d.service.Instances.Get(d.projectId, zone, name).Do()
	if err != nil {
		return "", err
	}

	for _, ni := range instance.NetworkInterfaces {
		if ni.NetworkIP == "" {
			continue
		}
		return ni.NetworkIP, nil
	}

	return "", nil
}

func (d *driverGCE) RunInstance(c *InstanceConfig) (<-chan error, error) {
	// Get the zone
	d.ui.Message(fmt.Sprintf("Loading zone: %s", c.Zone))
	zone, err := d.service.Zones.Get(d.projectId, c.Zone).Do()
	if err != nil {
		return nil, err
	}

	// Get the image
	d.ui.Message(fmt.Sprintf("Loading image: %s in project %s", c.Image.Name, c.Image.ProjectId))
	image, err := d.getImage(c.Image)
	if err != nil {
		return nil, err
	}

	// Get the machine type
	d.ui.Message(fmt.Sprintf("Loading machine type: %s", c.MachineType))
	machineType, err := d.service.MachineTypes.Get(
		d.projectId, zone.Name, c.MachineType).Do()
	if err != nil {
		return nil, err
	}
	// TODO(mitchellh): deprecation warnings

	// Get the network
	d.ui.Message(fmt.Sprintf("Loading network: %s", c.Network))
	network, err := d.service.Networks.Get(d.projectId, c.Network).Do()
	if err != nil {
		return nil, err
	}

	// Build up the metadata
	metadata := make([]*compute.MetadataItems, len(c.Metadata))
	for k, v := range c.Metadata {
		metadata = append(metadata, &compute.MetadataItems{
			Key:   k,
			Value: &v,
		})
	}

	// Create the instance information
	instance := compute.Instance{
		Description: c.Description,
		Disks: []*compute.AttachedDisk{
			&compute.AttachedDisk{
				Type:       "PERSISTENT",
				Mode:       "READ_WRITE",
				Kind:       "compute#attachedDisk",
				Boot:       true,
				AutoDelete: false,
				InitializeParams: &compute.AttachedDiskInitializeParams{
					SourceImage: image.SelfLink,
					DiskSizeGb:  c.DiskSizeGb,
				},
			},
		},
		MachineType: machineType.SelfLink,
		Metadata: &compute.Metadata{
			Items: metadata,
		},
		Name: c.Name,
		NetworkInterfaces: []*compute.NetworkInterface{
			&compute.NetworkInterface{
				AccessConfigs: []*compute.AccessConfig{
					&compute.AccessConfig{
						Name: "AccessConfig created by Packer",
						Type: "ONE_TO_ONE_NAT",
					},
				},
				Network: network.SelfLink,
			},
		},
		ServiceAccounts: []*compute.ServiceAccount{
			&compute.ServiceAccount{
				Email: "default",
				Scopes: []string{
					"https://www.googleapis.com/auth/userinfo.email",
					"https://www.googleapis.com/auth/compute",
					"https://www.googleapis.com/auth/devstorage.full_control",
				},
			},
		},
		Tags: &compute.Tags{
			Items: c.Tags,
		},
	}

	d.ui.Message("Requesting instance creation...")
	op, err := d.service.Instances.Insert(d.projectId, zone.Name, &instance).Do()
	if err != nil {
		return nil, err
	}

	errCh := make(chan error, 1)
	go waitForState(errCh, "DONE", d.refreshZoneOp(zone.Name, op))
	return errCh, nil
}

func (d *driverGCE) WaitForInstance(state, zone, name string) <-chan error {
	errCh := make(chan error, 1)
	go waitForState(errCh, state, d.refreshInstanceState(zone, name))
	return errCh
}

func (d *driverGCE) getImage(img Image) (image *compute.Image, err error) {
	projects := []string{img.ProjectId, "centos-cloud", "coreos-cloud", "debian-cloud", "google-containers", "opensuse-cloud", "rhel-cloud", "suse-cloud", "ubuntu-os-cloud", "windows-cloud"}
	for _, project := range projects {
		image, err = d.service.Images.Get(project, img.Name).Do()
		if err == nil && image != nil && image.SelfLink != "" {
			return
		}
		image = nil
	}

	err = fmt.Errorf("Image %s could not be found in any of these projects: %s", img.Name, projects)
	return
}

func (d *driverGCE) refreshInstanceState(zone, name string) stateRefreshFunc {
	return func() (string, error) {
		instance, err := d.service.Instances.Get(d.projectId, zone, name).Do()
		if err != nil {
			return "", err
		}
		return instance.Status, nil
	}
}

func (d *driverGCE) refreshGlobalOp(op *compute.Operation) stateRefreshFunc {
	return func() (string, error) {
		newOp, err := d.service.GlobalOperations.Get(d.projectId, op.Name).Do()
		if err != nil {
			return "", err
		}

		// If the op is done, check for errors
		err = nil
		if newOp.Status == "DONE" {
			if newOp.Error != nil {
				for _, e := range newOp.Error.Errors {
					err = packer.MultiErrorAppend(err, fmt.Errorf(e.Message))
				}
			}
		}

		return newOp.Status, err
	}
}

func (d *driverGCE) refreshZoneOp(zone string, op *compute.Operation) stateRefreshFunc {
	return func() (string, error) {
		newOp, err := d.service.ZoneOperations.Get(d.projectId, zone, op.Name).Do()
		if err != nil {
			return "", err
		}

		// If the op is done, check for errors
		err = nil
		if newOp.Status == "DONE" {
			if newOp.Error != nil {
				for _, e := range newOp.Error.Errors {
					err = packer.MultiErrorAppend(err, fmt.Errorf(e.Message))
				}
			}
		}

		return newOp.Status, err
	}
}

// stateRefreshFunc is used to refresh the state of a thing and is
// used in conjunction with waitForState.
type stateRefreshFunc func() (string, error)

// waitForState will spin in a loop forever waiting for state to
// reach a certain target.
func waitForState(errCh chan<- error, target string, refresh stateRefreshFunc) {
	for {
		state, err := refresh()
		if err != nil {
			errCh <- err
			return
		}
		if state == target {
			errCh <- nil
			return
		}

		time.Sleep(2 * time.Second)
	}
}
<|endoftext|>


<commit_before>package machine

import "golang.org/x/net/context"

type State func(context.Context, Transitioner)

type Joiner interface {
	Wait(timeout int64)
}

type Transitioner interface {
	Next(State)
	Fork(context.Context, ...State) Joiner
	Done()
}

type Machine interface {
	Run(context.Context, State) Joiner
}
<commit_msg>added comment into state interfaces<commit_after>package machine

import "golang.org/x/net/context"

//State this is the state fucntion signiture.
type State func(context.Context, Transitioner)

//Joiner I needed an interface so I can work with async operation.
type Joiner interface {
	//Wait sometimes you need to block and wait unitl the Joiner operation
	//completes before you move on. So this method should be blocking operation.
	Wait(timeout int64)
}

//Transitioner is an interface which can abstarcted common oeprations which
//state machine needed to operate.
type Transitioner interface {
	//Next gets a next state. Once you call it you should not do anything on
	//caller. becuase state machine has moved on to next state.
	Next(State)
	//Fork is a special method which I found it useful because it runs each pass
	//states as a initial state of a brand new state machine. this makes it useful
	//when a state contains sub states which needs to be called in parallel.
	//it returns a Joiner which helps called to wait until all those state machine
	//complete their operations.
	//context.Context is being used to have a share context between thoses state
	//machine.
	Fork(context.Context, ...State) Joiner
	//Done is method that tells state machien that we are Done and no more states
	//are going to be processed.
	Done()
}

//Machine is base blocking start point.
type Machine interface {
	//Run is the start point of state machine. context.Context is a share context
	//between all those states inside state machine. the second argument is an
	//initial state of our state machine.
	Run(context.Context, State) Joiner
}
<|endoftext|>


<commit_before>package garbage4

import (
	"os"
	"syscall"
	"unsafe"
)

const MAX_STATE_SIZE = 32768

type State struct {
	ref      []byte
	file     *os.File
	offset   int
	data     *[MAX_STATE_SIZE]byte
	channels map[string]int
}

type Position struct {
	offset    uint32
	segmentId uint64
}

func loadState(t *Topic) (*State, error) {
	root := PATH + t.name
	if err := os.MkdirAll(root, 0700); err != nil {
		return nil, err
	}
	path := root + "/state.q"
	file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0600)
	if err != nil {
		return nil, err
	}

	info, err := file.Stat()
	if err != nil {
		return nil, err
	}

	if size := info.Size(); size == 0 {
		file.Truncate(MAX_STATE_SIZE)
	} else if size != MAX_STATE_SIZE {
		panic("invalid state file size")
	}

	ref, err := syscall.Mmap(int(file.Fd()), 0, MAX_STATE_SIZE, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED)
	if err != nil {
		file.Close()
		return nil, err
	}

	state := &State{
		ref:      ref,
		file:     file,
		channels: make(map[string]int),
		data:     (*[MAX_STATE_SIZE]byte)(unsafe.Pointer(&ref[0])),
	}

	start, end := 16, 16
	for {
		if state.data[end] == 0 {
			if end == start {
				break
			}
			offset := end + 1
			state.channels[string(state.data[start:end])] = offset
			start = offset + 16
			end = start
		} else {
			end++
		}
	}
	state.offset = end
	return state, nil
}

func (s *State) loadPosition(offset int) *Position {
	return (*Position)(unsafe.Pointer(&s.data[offset]))
}

func (s *State) loadOrCreatePosition(name string) *Position {
	offset, exists := s.channels[name]
	if exists == false {
		//todo check for overflow
		//todo compact
		s.offset += copy(s.data[s.offset:], name)
		s.data[s.offset] = 0
		offset = s.offset + 1
		s.offset += 17
	}
	return s.loadPosition(offset)
}

func (s *State) Close() {
	syscall.Munmap(s.ref)
	s.file.Close()
	s.data, s.ref = nil, nil
}
<commit_msg>Even though we'll never call the stat's loadOrCreate more than once per channel per topic (since the topic has its own channel cache), we should update the state's channel lookup so that, if we ever need it, it behaves like we'd expect (currently only called from the topic's worker, so no need to synchronize, for now)<commit_after>package garbage4

import (
	"os"
	"syscall"
	"unsafe"
)

const MAX_STATE_SIZE = 32768

type State struct {
	ref      []byte
	file     *os.File
	offset   int
	data     *[MAX_STATE_SIZE]byte
	channels map[string]int
}

type Position struct {
	offset    uint32
	segmentId uint64
}

func loadState(t *Topic) (*State, error) {
	root := PATH + t.name
	if err := os.MkdirAll(root, 0700); err != nil {
		return nil, err
	}
	path := root + "/state.q"
	file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0600)
	if err != nil {
		return nil, err
	}

	info, err := file.Stat()
	if err != nil {
		return nil, err
	}

	if size := info.Size(); size == 0 {
		file.Truncate(MAX_STATE_SIZE)
	} else if size != MAX_STATE_SIZE {
		panic("invalid state file size")
	}

	ref, err := syscall.Mmap(int(file.Fd()), 0, MAX_STATE_SIZE, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED)
	if err != nil {
		file.Close()
		return nil, err
	}

	state := &State{
		ref:      ref,
		file:     file,
		channels: make(map[string]int),
		data:     (*[MAX_STATE_SIZE]byte)(unsafe.Pointer(&ref[0])),
	}

	start, end := 16, 16
	for {
		if state.data[end] == 0 {
			if end == start {
				break
			}
			offset := end + 1
			state.channels[string(state.data[start:end])] = offset
			start = offset + 16
			end = start
		} else {
			end++
		}
	}
	state.offset = end
	return state, nil
}

func (s *State) loadPosition(offset int) *Position {
	return (*Position)(unsafe.Pointer(&s.data[offset]))
}

func (s *State) loadOrCreatePosition(name string) *Position {
	offset, exists := s.channels[name]
	if exists == false {
		//todo check for overflow
		//todo compact
		s.offset += copy(s.data[s.offset:], name)
		s.data[s.offset] = 0
		offset = s.offset + 1
		s.offset += 17 //segmentId + offset + 1 for name null
		s.channels[name] = offset
	}
	return s.loadPosition(offset)
}

func (s *State) Close() {
	syscall.Munmap(s.ref)
	s.file.Close()
	s.data, s.ref = nil, nil
}
<|endoftext|>


<commit_before>package templar

import (
	"fmt"
	"github.com/amir/raidman"
	"net/http"
	"strings"
	"time"
)

type DebugStats struct{}

func (d *DebugStats) StartRequest(req *http.Request) {
	fmt.Printf("[%s] S %s %s\n", time.Now().Format(time.RFC3339Nano), req.Method, req.URL)
}

func (d *DebugStats) Emit(req *http.Request, dur time.Duration) {
	fmt.Printf("[%s] E %s %s (%s)\n", time.Now().Format(time.RFC3339Nano), req.Method, req.URL, dur)
}

func (d *DebugStats) RequestTimeout(req *http.Request, timeout time.Duration) {
	fmt.Printf("[%s] T %s %s (%s)\n", time.Now().Format(time.RFC3339Nano), req.Method, req.URL, timeout)
}

var _ = Stats(&DebugStats{})

type StatsdOutput struct {
	client StatsdClient
}

var _ = Stats(&StatsdOutput{})

func NewStatsdOutput(client StatsdClient) *StatsdOutput {
	return &StatsdOutput{client}
}

func (s *StatsdOutput) url(req *http.Request) string {
	return req.Host + strings.Replace(req.URL.Path, "/", "-", -1)
}

func (s *StatsdOutput) StartRequest(req *http.Request) {
	s.client.Incr("templar.request.method."+req.Method, 1)
	s.client.Incr("templar.request.host."+req.Host, 1)
	s.client.Incr("templar.request.url."+s.url(req), 1)
	s.client.GaugeDelta("templar.requests.active", 1)
}

func (s *StatsdOutput) Emit(req *http.Request, delta time.Duration) {
	s.client.GaugeDelta("templar.requests.active", -1)
	s.client.PrecisionTiming("templar.request.url."+s.url(req), delta)
}

func (s *StatsdOutput) RequestTimeout(req *http.Request, timeout time.Duration) {
	s.client.Incr("templar.timeout.host."+req.Host, 1)
	s.client.Incr("templar.timeout.url."+s.url(req), 1)
}

type RiemannOutput struct {
	client RiemannClient
}

func NewRiemannOutput(client RiemannClient) *RiemannOutput {
	return &RiemannOutput{client}
}

func (r *RiemannOutput) StartRequest(req *http.Request) {
	attributes := make(map[string]string)
	attributes["method"] = req.Method
	attributes["host"] = req.Host
	attributes["path"] = req.URL.Path
	var event = &raidman.Event{
		State:      "ok",
		Service:    "templar request",
		Metric:     1,
		Attributes: attributes,
	}
	r.client.Send(event)
}

func (r *RiemannOutput) Emit(req *http.Request, delta time.Duration) {
	attributes := make(map[string]string)
	attributes["method"] = req.Method
	attributes["host"] = req.Host
	attributes["path"] = req.URL.Path
	var event = &raidman.Event{
		State:      "ok",
		Service:    "templar response",
		Metric:     1000.0 * delta.Seconds(),
		Attributes: attributes,
	}
	r.client.Send(event)
}

func (r *RiemannOutput) RequestTimeout(req *http.Request, timeout time.Duration) {
	attributes := make(map[string]string)
	attributes["method"] = req.Method
	attributes["host"] = req.Host
	attributes["path"] = req.URL.Path
	var event = &raidman.Event{
		State:      "warning",
		Service:    "templar timeout",
		Metric:     timeout.Seconds() * 1000.0,
		Attributes: attributes,
	}
	r.client.Send(event)
}

type MultiStats []Stats

var _ = Stats(MultiStats{})

func (m MultiStats) StartRequest(req *http.Request) {
	for _, s := range m {
		s.StartRequest(req)
	}
}

func (m MultiStats) Emit(req *http.Request, t time.Duration) {
	for _, s := range m {
		s.Emit(req, t)
	}
}

func (m MultiStats) RequestTimeout(req *http.Request, timeout time.Duration) {
	for _, s := range m {
		s.RequestTimeout(req, timeout)
	}
}
<commit_msg>change host attribute on riemann hosts<commit_after>package templar

import (
	"fmt"
	"github.com/amir/raidman"
	"net/http"
	"strings"
	"time"
)

type DebugStats struct{}

func (d *DebugStats) StartRequest(req *http.Request) {
	fmt.Printf("[%s] S %s %s\n", time.Now().Format(time.RFC3339Nano), req.Method, req.URL)
}

func (d *DebugStats) Emit(req *http.Request, dur time.Duration) {
	fmt.Printf("[%s] E %s %s (%s)\n", time.Now().Format(time.RFC3339Nano), req.Method, req.URL, dur)
}

func (d *DebugStats) RequestTimeout(req *http.Request, timeout time.Duration) {
	fmt.Printf("[%s] T %s %s (%s)\n", time.Now().Format(time.RFC3339Nano), req.Method, req.URL, timeout)
}

var _ = Stats(&DebugStats{})

type StatsdOutput struct {
	client StatsdClient
}

var _ = Stats(&StatsdOutput{})

func NewStatsdOutput(client StatsdClient) *StatsdOutput {
	return &StatsdOutput{client}
}

func (s *StatsdOutput) url(req *http.Request) string {
	return req.Host + strings.Replace(req.URL.Path, "/", "-", -1)
}

func (s *StatsdOutput) StartRequest(req *http.Request) {
	s.client.Incr("templar.request.method."+req.Method, 1)
	s.client.Incr("templar.request.host."+req.Host, 1)
	s.client.Incr("templar.request.url."+s.url(req), 1)
	s.client.GaugeDelta("templar.requests.active", 1)
}

func (s *StatsdOutput) Emit(req *http.Request, delta time.Duration) {
	s.client.GaugeDelta("templar.requests.active", -1)
	s.client.PrecisionTiming("templar.request.url."+s.url(req), delta)
}

func (s *StatsdOutput) RequestTimeout(req *http.Request, timeout time.Duration) {
	s.client.Incr("templar.timeout.host."+req.Host, 1)
	s.client.Incr("templar.timeout.url."+s.url(req), 1)
}

type RiemannOutput struct {
	client RiemannClient
}

func NewRiemannOutput(client RiemannClient) *RiemannOutput {
	return &RiemannOutput{client}
}

func (r *RiemannOutput) StartRequest(req *http.Request) {
	attributes := make(map[string]string)
	attributes["method"] = req.Method
	attributes["http-host"] = req.Host
	attributes["path"] = req.URL.Path
	var event = &raidman.Event{
		State:      "ok",
		Service:    "templar request",
		Metric:     1,
		Attributes: attributes,
	}
	r.client.Send(event)
}

func (r *RiemannOutput) Emit(req *http.Request, delta time.Duration) {
	attributes := make(map[string]string)
	attributes["method"] = req.Method
	attributes["http-host"] = req.Host
	attributes["path"] = req.URL.Path
	var event = &raidman.Event{
		State:      "ok",
		Service:    "templar response",
		Metric:     1000.0 * delta.Seconds(),
		Attributes: attributes,
	}
	r.client.Send(event)
}

func (r *RiemannOutput) RequestTimeout(req *http.Request, timeout time.Duration) {
	attributes := make(map[string]string)
	attributes["method"] = req.Method
	attributes["host"] = req.Host
	attributes["path"] = req.URL.Path
	var event = &raidman.Event{
		State:      "warning",
		Service:    "templar timeout",
		Metric:     timeout.Seconds() * 1000.0,
		Attributes: attributes,
	}
	r.client.Send(event)
}

type MultiStats []Stats

var _ = Stats(MultiStats{})

func (m MultiStats) StartRequest(req *http.Request) {
	for _, s := range m {
		s.StartRequest(req)
	}
}

func (m MultiStats) Emit(req *http.Request, t time.Duration) {
	for _, s := range m {
		s.Emit(req, t)
	}
}

func (m MultiStats) RequestTimeout(req *http.Request, timeout time.Duration) {
	for _, s := range m {
		s.RequestTimeout(req, timeout)
	}
}
<|endoftext|>


<commit_before>package mailgun

import (
	"context"
	"time"
)

const iso8601date = "2006-01-02"

// Stats on accepted messages
type Accepted struct {
	Incoming int `json:"incoming"`
	Outgoing int `json:"outgoing"`
	Total    int `json:"total"`
}

// Stats on delivered messages
type Delivered struct {
	Smtp  int `json:"smtp"`
	Http  int `json:"http"`
	Total int `json:"total"`
}

// Stats on temporary failures
type Temporary struct {
	Espblock int `json:"espblock"`
}

// Stats on permanent failures
type Permanent struct {
	SuppressBounce      int `json:"suppress-bounce"`
	SuppressUnsubscribe int `json:"suppress-unsubscribe"`
	SuppressComplaint   int `json:"suppress-complaint"`
	Bounce              int `json:"bounce"`
	DelayedBounce       int `json:"delayed-bounce"`
	Total               int `json:"total"`
}

// Stats on failed messages
type Failed struct {
	Temporary Temporary `json:"temporary"`
	Permanent Permanent `json:"permanent"`
}

// Total stats for messages
type Total struct {
	Total int `json:"total"`
}

// Stats as returned by `GetStats()`
type Stats struct {
	Time         string    `json:"time"`
	Accepted     Accepted  `json:"accepted"`
	Delivered    Delivered `json:"delivered"`
	Failed       Failed    `json:"failed"`
	Stored       Total     `json:"stored"`
	Opened       Total     `json:"opened"`
	Clicked      Total     `json:"clicked"`
	Unsubscribed Total     `json:"unsubscribed"`
	Complained   Total     `json:"complained"`
}

type statsTotalResponse struct {
	End        string  `json:"end"`
	Resolution string  `json:"resolution"`
	Start      string  `json:"start"`
	Stats      []Stats `json:"stats"`
}

// Used by GetStats() to specify the resolution stats are for
type Resolution string

// Indicate which resolution a stat response for request is for
const (
	ResolutionHour  = Resolution("hour")
	ResolutionDay   = Resolution("day")
	ResolutionMonth = Resolution("month")
)

// Options for GetStats()
type GetStatOptions struct {
	Resolution Resolution
	Duration   string
	Start      time.Time
	End        time.Time
}

// GetStats returns total stats for a given domain for the specified time period
func (mg *MailgunImpl) GetStats(ctx context.Context, events []string, opts *GetStatOptions) ([]Stats, error) {
	r := newHTTPRequest(generateApiUrl(mg, statsTotalEndpoint))

	if opts != nil {
		if !opts.Start.IsZero() {
			r.addParameter("start", opts.Start.Format(iso8601date))
		}
		if !opts.End.IsZero() {
			r.addParameter("end", opts.End.Format(iso8601date))
		}
		if opts.Resolution != "" {
			r.addParameter("resolution", string(opts.Resolution))
		}
		if opts.Duration != "" {
			r.addParameter("duration", opts.Duration)
		}
	}

	for _, e := range events {
		r.addParameter("event", e)
	}

	r.setClient(mg.Client())
	r.setBasicAuth(basicAuthUser, mg.APIKey())

	var res statsTotalResponse
	err := getResponseFromJSON(ctx, r, &res)
	if err != nil {
		return nil, err
	} else {
		return res.Stats, nil
	}
}
<commit_msg>Stats: fix time format<commit_after>package mailgun

import (
	"context"
	"strconv"
	"time"
)

// Stats on accepted messages
type Accepted struct {
	Incoming int `json:"incoming"`
	Outgoing int `json:"outgoing"`
	Total    int `json:"total"`
}

// Stats on delivered messages
type Delivered struct {
	Smtp  int `json:"smtp"`
	Http  int `json:"http"`
	Total int `json:"total"`
}

// Stats on temporary failures
type Temporary struct {
	Espblock int `json:"espblock"`
}

// Stats on permanent failures
type Permanent struct {
	SuppressBounce      int `json:"suppress-bounce"`
	SuppressUnsubscribe int `json:"suppress-unsubscribe"`
	SuppressComplaint   int `json:"suppress-complaint"`
	Bounce              int `json:"bounce"`
	DelayedBounce       int `json:"delayed-bounce"`
	Total               int `json:"total"`
}

// Stats on failed messages
type Failed struct {
	Temporary Temporary `json:"temporary"`
	Permanent Permanent `json:"permanent"`
}

// Total stats for messages
type Total struct {
	Total int `json:"total"`
}

// Stats as returned by `GetStats()`
type Stats struct {
	Time         string    `json:"time"`
	Accepted     Accepted  `json:"accepted"`
	Delivered    Delivered `json:"delivered"`
	Failed       Failed    `json:"failed"`
	Stored       Total     `json:"stored"`
	Opened       Total     `json:"opened"`
	Clicked      Total     `json:"clicked"`
	Unsubscribed Total     `json:"unsubscribed"`
	Complained   Total     `json:"complained"`
}

type statsTotalResponse struct {
	End        string  `json:"end"`
	Resolution string  `json:"resolution"`
	Start      string  `json:"start"`
	Stats      []Stats `json:"stats"`
}

// Used by GetStats() to specify the resolution stats are for
type Resolution string

// Indicate which resolution a stat response for request is for
const (
	ResolutionHour  = Resolution("hour")
	ResolutionDay   = Resolution("day")
	ResolutionMonth = Resolution("month")
)

// Options for GetStats()
type GetStatOptions struct {
	Resolution Resolution
	Duration   string
	Start      time.Time
	End        time.Time
}

// GetStats returns total stats for a given domain for the specified time period
func (mg *MailgunImpl) GetStats(ctx context.Context, events []string, opts *GetStatOptions) ([]Stats, error) {
	r := newHTTPRequest(generateApiUrl(mg, statsTotalEndpoint))

	if opts != nil {
		if !opts.Start.IsZero() {
			r.addParameter("start", strconv.Itoa(int(opts.Start.Unix())))
		}
		if !opts.End.IsZero() {
			r.addParameter("end", strconv.Itoa(int(opts.End.Unix())))
		}
		if opts.Resolution != "" {
			r.addParameter("resolution", string(opts.Resolution))
		}
		if opts.Duration != "" {
			r.addParameter("duration", opts.Duration)
		}
	}

	for _, e := range events {
		r.addParameter("event", e)
	}

	r.setClient(mg.Client())
	r.setBasicAuth(basicAuthUser, mg.APIKey())

	var res statsTotalResponse
	err := getResponseFromJSON(ctx, r, &res)
	if err != nil {
		return nil, err
	} else {
		return res.Stats, nil
	}
}
<|endoftext|>


<commit_before>package contentstore

import (
	"encoding/csv"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"os"
	"sort"
	"strconv"
	"sync"

	"github.com/kjk/u"
)

// Files we use:
// - index where for each blob we store: sha1 of the content, number of segment
//   file in which the blob is stored, offset within the segment and size of
//   the blob
// - one or more segment files. User can control the max size of segment
//   file (10 MB by default) to pick the right file size/number of files
//   balance for his needs

// Ideas for the future:
// - add a mode where we store big blobs (e.g. over 1MB) in their own files
// - add a way to delete files and re-use deleted space via some sort of
//   best-fit allocator (if we're adding a new blob and have free space
//   due to deletion, pick the free space that is the closest in size to
//   new blob but only if it's e.g. withing 10% in size, to avoid excessive
//   fragmentation)
// - add a way to delete files by rewriting the files (expensive! we have to
//   rewrite the whole index and each segment that contains deleted files)

var (
	errNotFound           = errors.New("not found")
	errInvalidIndexHdr    = errors.New("invalid index file header")
	errInvalidIndexLine   = errors.New("invalid index line")
	errSegmentFileMissing = errors.New("segment file missing")
	errNotValidSha1       = errors.New("not a valid sha1")
	// first line in index file, for additional safety
	idxHdr = "github.com/kjk/contentstore header 1.0"
)

type Blob struct {
	sha1     [20]byte
	nSegment int
	offset   int
	size     int
}

type Store struct {
	sync.Mutex
	baseFilePath   string
	maxSegmentSize int
	blobs          []Blob
	// sha1ToBlob is to quickly find
	// string is really [20]byte cast to string and int is a position within blobs array
	// Note: we could try to be a bit smarter about how we
	sha1ToBlobNo    map[string]int
	idxFile         *os.File
	idxCsvWriter    *csv.Writer
	currSegmentFile *os.File
	currSegment     int
	currSegmentSize int
}

func idxFilePath(baseFilePath string) string {
	return baseFilePath + "_idx.txt"
}

func segmentFilePath(baseFilePath string, nSegment int) string {
	return fmt.Sprintf("%s_%d.txt", baseFilePath, nSegment)
}

func decodeIndexLine(rec []string) (blob Blob, err error) {
	if len(rec) != 4 {
		return blob, errInvalidIndexLine
	}
	sha1, err := hex.DecodeString(rec[0])
	if err != nil {
		return blob, err
	}
	if len(sha1) != 20 {
		return blob, errNotValidSha1
	}
	copy(blob.sha1[:], sha1)
	if blob.nSegment, err = strconv.Atoi(rec[1]); err != nil {
		return blob, err
	}
	if blob.offset, err = strconv.Atoi(rec[2]); err != nil {
		return blob, err
	}
	if blob.size, err = strconv.Atoi(rec[3]); err != nil {
		return blob, err
	}
	return blob, nil
}

// appends x to array of ints
func appendIntIfNotExists(aPtr *[]int, x int) {
	a := *aPtr
	if sort.SearchInts(a, x) >= 0 {
		return
	}
	a = append(a, x)
	sort.Ints(a)
	*aPtr = a
}

func (store *Store) readExistingData() error {
	// at this point idx file must exist
	fidx, err := os.Open(idxFilePath(store.baseFilePath))
	if err != nil {
		return err
	}
	// TODO: would be faster (and easier?) to use a bitset since we know
	// segment numbers are consequitive integers
	segments := make([]int, 0)
	csvReader := csv.NewReader(fidx)
	csvReader.Comma = ','
	csvReader.FieldsPerRecord = -1
	rec, err := csvReader.Read()
	if err != nil || len(rec) != 1 || rec[0] != idxHdr {
		return errInvalidIndexHdr
	}
	for {
		rec, err = csvReader.Read()
		if err != nil {
			break
		}
		blob, err := decodeIndexLine(rec)
		if err != nil {
			break
		}
		appendIntIfNotExists(&segments, blob.nSegment)
		store.blobs = append(store.blobs, blob)
	}
	if err == io.EOF {
		err = nil
	}
	// verify segment files exist
	// TODO: also verify offset + size is <= size of segment file
	for _, nSegment := range segments {
		path := segmentFilePath(store.baseFilePath, nSegment)
		if !u.PathExists(path) {
			return errSegmentFileMissing
		}
	}
	return nil
}

func NewStoreWithLimit(baseFilePath string, maxSegmentSize int) (store *Store, err error) {
	store = &Store{
		baseFilePath:   baseFilePath,
		blobs:          make([]Blob, 0),
		maxSegmentSize: maxSegmentSize,
	}
	idxPath := idxFilePath(baseFilePath)
	idxExists := u.PathExists(idxPath)
	if idxExists {
		// Note: we have enough data in segment files to reconstruct the index
		// in the rare case that index got deleted but segment files did not.
		// The logic to do that is not implemented.
		if err = store.readExistingData(); err != nil {
			return nil, err
		}
	}

	if store.idxFile, err = os.OpenFile(idxPath, os.O_WRONLY|os.O_APPEND, 0644); err != nil {
		return nil, err
	}

	return store, nil
}

func NewStore(baseFilePath string) (*Store, error) {
	return NewStoreWithLimit(baseFilePath, 10*1024*1024)
}

func closeFilePtr(filePtr **os.File) (err error) {
	f := *filePtr
	if f != nil {
		err = f.Close()
		*filePtr = nil
	}
	return err
}

func (store *Store) Close() {
	closeFilePtr(&store.idxFile)
	closeFilePtr(&store.currSegmentFile)
}

func (store *Store) Put(d []byte) (id string, err error) {
	store.Lock()
	defer store.Unlock()

	idBytes := u.Sha1OfBytes(d)
	id = string(idBytes)
	if _, ok := store.sha1ToBlobNo[id]; ok {
		return id, nil
	}
	// TODO: save the blob to current segment
	// TODO: when exceeded segment size, switch to a new segment
	return "", errors.New("NYI")
}

// TODO: cache file descriptor
func readFromFile(path string, offset, size int) ([]byte, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	res := make([]byte, size, size)
	if _, err := f.ReadAt(res, int64(offset)); err != nil {
		return nil, err
	}
	return res, nil
}

func (store *Store) Get(id string) ([]byte, error) {
	store.Lock()
	defer store.Unlock()

	blobNo, ok := store.sha1ToBlobNo[id]
	if !ok {
		return nil, errNotFound
	}
	blob := store.blobs[blobNo]
	path := segmentFilePath(store.baseFilePath, blob.nSegment)
	return readFromFile(path, blob.offset, blob.size)
}
<commit_msg>mostly finished<commit_after>package contentstore

import (
	"encoding/csv"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"os"
	"sort"
	"strconv"
	"sync"

	"github.com/kjk/u"
)

// Files we use:
// - index where for each blob we store: sha1 of the content, number of segment
//   file in which the blob is stored, offset within the segment and size of
//   the blob
// - one or more segment files. User can control the max size of segment
//   file (10 MB by default) to pick the right file size/number of files
//   balance for his needs

// Ideas for the future:
// - add a mode where we store big blobs (e.g. over 1MB) in their own files
// - add a way to delete files and re-use deleted space via some sort of
//   best-fit allocator (if we're adding a new blob and have free space
//   due to deletion, pick the free space that is the closest in size to
//   new blob but only if it's e.g. withing 10% in size, to avoid excessive
//   fragmentation)
// - add a way to delete files by rewriting the files (expensive! we have to
//   rewrite the whole index and each segment that contains deleted files)

var (
	errNotFound           = errors.New("not found")
	errInvalidIndexHdr    = errors.New("invalid index file header")
	errInvalidIndexLine   = errors.New("invalid index line")
	errSegmentFileMissing = errors.New("segment file missing")
	errNotValidSha1       = errors.New("not a valid sha1")
	// first line in index file, for additional safety
	idxHdr = "github.com/kjk/contentstore header 1.0"
)

type Blob struct {
	sha1     [20]byte
	nSegment int
	offset   int
	size     int
}

type Store struct {
	sync.Mutex
	baseFilePath   string
	maxSegmentSize int
	blobs          []Blob
	// sha1ToBlob is to quickly find
	// string is really [20]byte cast to string and int is a position within blobs array
	// Note: we could try to be a bit smarter about how we
	sha1ToBlobNo    map[string]int
	idxFile         *os.File
	idxCsvWriter    *csv.Writer
	currSegmentFile *os.File
	currSegmentNo   int
	currSegmentSize int
	// we cache file descriptor for one segment file (in addition to current
	// segment file) to reduce file open/close for Get()
	cachedSegmentFile *os.File
	cachedSegmentNo   int
}

func idxFilePath(baseFilePath string) string {
	return baseFilePath + "_idx.txt"
}

func segmentFilePath(baseFilePath string, nSegment int) string {
	return fmt.Sprintf("%s_%d.txt", baseFilePath, nSegment)
}

func decodeIndexLine(rec []string) (blob Blob, err error) {
	if len(rec) != 4 {
		return blob, errInvalidIndexLine
	}
	sha1, err := hex.DecodeString(rec[0])
	if err != nil {
		return blob, err
	}
	if len(sha1) != 20 {
		return blob, errNotValidSha1
	}
	copy(blob.sha1[:], sha1)
	if blob.nSegment, err = strconv.Atoi(rec[1]); err != nil {
		return blob, err
	}
	if blob.offset, err = strconv.Atoi(rec[2]); err != nil {
		return blob, err
	}
	if blob.size, err = strconv.Atoi(rec[3]); err != nil {
		return blob, err
	}
	return blob, nil
}

// appends x to array of ints
func appendIntIfNotExists(aPtr *[]int, x int) {
	a := *aPtr
	if sort.SearchInts(a, x) >= 0 {
		return
	}
	a = append(a, x)
	sort.Ints(a)
	*aPtr = a
}

func (store *Store) readIndex() error {
	// at this point idx file must exist
	fidx, err := os.Open(idxFilePath(store.baseFilePath))
	if err != nil {
		return err
	}
	// TODO: would be faster (and easier?) to use a bitset since we know
	// segment numbers are consequitive integers
	segments := make([]int, 0)
	csvReader := csv.NewReader(fidx)
	csvReader.Comma = ','
	csvReader.FieldsPerRecord = -1
	rec, err := csvReader.Read()
	if err != nil || len(rec) != 1 || rec[0] != idxHdr {
		return errInvalidIndexHdr
	}
	for {
		rec, err = csvReader.Read()
		if err != nil {
			break
		}
		blob, err := decodeIndexLine(rec)
		if err != nil {
			break
		}
		appendIntIfNotExists(&segments, blob.nSegment)
		store.blobs = append(store.blobs, blob)
	}
	if err == io.EOF {
		err = nil
	}
	// verify segment files exist
	// TODO: also verify offset + size is <= size of segment file
	for _, nSegment := range segments {
		path := segmentFilePath(store.baseFilePath, nSegment)
		if !u.PathExists(path) {
			return errSegmentFileMissing
		}
		store.currSegmentNo = nSegment
	}
	return nil
}

func NewStoreWithLimit(baseFilePath string, maxSegmentSize int) (store *Store, err error) {
	store = &Store{
		baseFilePath:    baseFilePath,
		blobs:           make([]Blob, 0),
		maxSegmentSize:  maxSegmentSize,
		cachedSegmentNo: -1,
	}
	idxPath := idxFilePath(baseFilePath)
	idxExists := u.PathExists(idxPath)
	if idxExists {
		if err = store.readIndex(); err != nil {
			return nil, err
		}
	}
	if store.idxFile, err = os.OpenFile(idxPath, os.O_WRONLY|os.O_APPEND, 0644); err != nil {
		return nil, err
	}
	segmentPath := segmentFilePath(store.baseFilePath, store.currSegmentNo)
	stat, err := os.Stat(segmentPath)
	if err != nil {
		// TODO: fail if error is different than "file doesn't exist"
		if store.currSegmentNo != 0 {
			store.Close()
			return nil, errSegmentFileMissing
		}
		store.currSegmentFile, err = os.Create(segmentPath)
		if err != nil {
			store.Close()
			return nil, err
		}
	} else {
		store.currSegmentSize = int(stat.Size())
		store.currSegmentFile, err = os.OpenFile(segmentPath, os.O_APPEND|os.O_RDWR, 0644)
		if err != nil {
			store.Close()
			return nil, err
		}
	}
	return store, nil
}

func NewStore(baseFilePath string) (*Store, error) {
	return NewStoreWithLimit(baseFilePath, 10*1024*1024)
}

func closeFilePtr(filePtr **os.File) (err error) {
	f := *filePtr
	if f != nil {
		err = f.Close()
		*filePtr = nil
	}
	return err
}

func (store *Store) Close() {
	closeFilePtr(&store.idxFile)
	closeFilePtr(&store.currSegmentFile)
	closeFilePtr(&store.cachedSegmentFile)
}

func writeBlobRec(csvWriter *csv.Writer, blob *Blob) error {
	sha1Str := hex.EncodeToString(blob.sha1[:])
	rec := []string{
		sha1Str,
		strconv.Itoa(blob.nSegment),
		strconv.Itoa(blob.offset),
		strconv.Itoa(blob.size),
	}
	return csvWriter.Write(rec)
}

func (store *Store) Put(d []byte) (id string, err error) {
	store.Lock()
	defer store.Unlock()

	idBytes := u.Sha1OfBytes(d)
	id = string(idBytes)
	if _, ok := store.sha1ToBlobNo[id]; ok {
		return id, nil
	}
	blob := Blob{
		size:     len(d),
		offset:   store.currSegmentSize,
		nSegment: store.currSegmentNo,
	}
	copy(blob.sha1[:], idBytes)

	if _, err = store.currSegmentFile.Write(d); err != nil {
		return "", err
	}
	if err = store.currSegmentFile.Sync(); err != nil {
		return "", err
	}
	store.currSegmentSize += blob.size
	if store.currSegmentSize >= store.maxSegmentSize {
		// filled current segment => create a new one
		err = store.currSegmentFile.Close()
		if err != nil {
			return "", err
		}
		store.currSegmentNo += 1
		store.currSegmentSize = 0
		path := segmentFilePath(store.baseFilePath, store.currSegmentNo)
		store.currSegmentFile, err = os.Create(path)
		if err != nil {
			return "", err
		}
	}
	if err = writeBlobRec(store.idxCsvWriter, &blob); err != nil {
		return "", err
	}
	store.blobs = append(store.blobs, blob)
	return id, nil
}

func readFromFile(file *os.File, offset, size int) ([]byte, error) {
	res := make([]byte, size, size)
	if _, err := file.ReadAt(res, int64(offset)); err != nil {
		return nil, err
	}
	return res, nil
}

func (store *Store) getSegmentFile(nSegment int) (*os.File, error) {
	if nSegment == store.currSegmentNo {
		return store.currSegmentFile, nil
	}
	if nSegment == store.cachedSegmentNo {
		return store.cachedSegmentFile, nil
	}
	closeFilePtr(&store.cachedSegmentFile)
	path := segmentFilePath(store.baseFilePath, nSegment)
	var err error
	if store.cachedSegmentFile, err = os.Open(path); err != nil {
		return nil, err
	}
	store.cachedSegmentNo = nSegment
	return store.cachedSegmentFile, nil
}

func (store *Store) Get(id string) ([]byte, error) {
	store.Lock()
	defer store.Unlock()

	blobNo, ok := store.sha1ToBlobNo[id]
	if !ok {
		return nil, errNotFound
	}
	blob := store.blobs[blobNo]
	segmentFile, err := store.getSegmentFile(blob.nSegment)
	if err != nil {
		return nil, err
	}
	return readFromFile(segmentFile, blob.offset, blob.size)
}
<|endoftext|>


<commit_before>package lights

import (
	"errors"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
)

// Data Store implementation

// Store is implemented by data storage providers for persistent
// configuration information.
type Store interface {
	// Read a value from the provided collection with a given ID.
	Read(collection, id string) (string, error)
	// Write a value to the provided collection with a given ID.
	Write(collection, id, value string) error
	// Remove a value from the provided collection with a given ID.
	Remove(collection, id string) error
	// Remove all values from the provided collection.
	RemoveAll(collection string) error
	// Load reads all the values out of a collection.
	Load(colection string) ([]string, error)
}

// FileStore implements the Store interface by storing each value in
// a file named after the item ID and folders for each collection.
// Note that IDs and collections must be file name friendly.
// On disk, the files will have a `.txt` file extension added.
type FileStore struct {
	Base string // The path to the file store base
}

// NewFileStore creates a new file-based Store implementation. Pass in
// an optional base path to use for data storage (otherwise the user's home
// directory is used).
func NewFileStore(base ...string) (*FileStore, error) {
	var root string
	if len(base) > 0 {
		root = base[0]
	} else {
		root = filepath.Join(os.Getenv("HOME"), "data")
	}
	root, err := filepath.Abs(root)
	if err != nil {
		return nil, err
	}
	err = os.MkdirAll(root, 0755)
	if err != nil {
		return nil, err
	}
	return &FileStore{root}, nil
}

// Read a value from the provided collection and ID.
func (f *FileStore) Read(collection, id string) (string, error) {
	name := filepath.Join(f.Base, collection, id+".txt")
	text, err := ioutil.ReadFile(name)
	if err != nil {
		return "", err
	}
	return string(text), nil
}

// Write a value to the provided collection and ID.
func (f *FileStore) Write(collection, id, value string) error {
	base := filepath.Join(f.Base, collection)
	err := os.MkdirAll(base, 0755)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(filepath.Join(base, id+".txt"), []byte(value), 0660)
}

// Remove a value from the provided collection and ID.
func (f *FileStore) Remove(collection, id string) error {
	return os.Remove(filepath.Join(f.Base, collection, id+".txt"))
}

// RemoveAll removes all items from a collection.
func (f *FileStore) RemoveAll(collection string) error {
	return os.RemoveAll(filepath.Join(f.Base, collection))
}

// Load all the values for a collection.
func (f *FileStore) Load(collection string) ([]string, error) {
	items := []string{}
	base := filepath.Join(f.Base, collection)
	base, err := filepath.Abs(base)
	if err != nil {
		log.Println("Error getting abs path to", collection, err)
		return nil, err
	}
	log.Println("Loading collection", collection, "from", base)
	filepath.Walk(base, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			// Ignore
			log.Println("store ignoring walk error", err)
			return filepath.SkipDir
		}
		/*
			if info.IsDir() {
				log.Println("skipping dir", path)
				return filepath.SkipDir
			}
		*/
		if filepath.Ext(path) == ".txt" {
			log.Println("loading item", path)
			text, err := ioutil.ReadFile(path)
			if err != nil {
				return err
			}
			items = append(items, string(text))
		} else {
			log.Println("skipping non item", path, filepath.Ext(path))
		}
		return nil
	})
	return items, nil

}

// MockStore is used to test services that rely on Store implementations.
type MockStore struct {
	Data map[string]map[string]string
}

// Read a value from the provided collection with a given ID.
func (s *MockStore) Read(collection, id string) (string, error) {
	c, ok := s.Data[collection]
	if !ok {
		return "", errors.New("No collection found " + collection)
	}
	item, ok := c[id]
	if !ok {
		return "", errors.New("No item with ID found " + id)
	}
	return item, nil
}

// Write a value to the provided collection with a given ID.
func (s *MockStore) Write(collection, id, value string) error {
	c, ok := s.Data[collection]
	if ok {
		c[id] = value
	} else {
		c = map[string]string{id: value}
		if s.Data == nil {
			s.Data = map[string]map[string]string{collection: c}
		} else {
			s.Data[collection] = c
		}
	}
	return nil
}

// Remove a value from the provided collection with a given ID.
func (s *MockStore) Remove(collection, id string) error {
	c, ok := s.Data[collection]
	if ok {
		delete(c, id)
	}
	return nil
}

// RemoveAll clears all items from a collection.
func (s *MockStore) RemoveAll(collection string) error {
	delete(s.Data, collection)
	return nil
}

// Load all the values from a collection.
func (s *MockStore) Load(collection string) ([]string, error) {
	items := []string{}
	for _, item := range s.Data[collection] {
		items = append(items, item)
	}
	return items, nil
}

// Reset removes all data from the store.
func (s *MockStore) Reset() {
	s.Data = map[string]map[string]string{}
}
<commit_msg>Change default data directory to fixed absolute location.<commit_after>package lights

import (
	"errors"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
)

// Data Store implementation

// Store is implemented by data storage providers for persistent
// configuration information.
type Store interface {
	// Read a value from the provided collection with a given ID.
	Read(collection, id string) (string, error)
	// Write a value to the provided collection with a given ID.
	Write(collection, id, value string) error
	// Remove a value from the provided collection with a given ID.
	Remove(collection, id string) error
	// Remove all values from the provided collection.
	RemoveAll(collection string) error
	// Load reads all the values out of a collection.
	Load(colection string) ([]string, error)
}

// FileStore implements the Store interface by storing each value in
// a file named after the item ID and folders for each collection.
// Note that IDs and collections must be file name friendly.
// On disk, the files will have a `.txt` file extension added.
type FileStore struct {
	Base string // The path to the file store base
}

// NewFileStore creates a new file-based Store implementation. Pass in
// an optional base path to use for data storage (otherwise the user's home
// directory is used).
func NewFileStore(base ...string) (*FileStore, error) {
	var root string
	if len(base) > 0 {
		root = base[0]
	} else {
		root = "/var/lib/inception/lighting/data"
	}
	root, err := filepath.Abs(root)
	if err != nil {
		return nil, err
	}
	err = os.MkdirAll(root, 0755)
	if err != nil {
		return nil, err
	}
	return &FileStore{root}, nil
}

// Read a value from the provided collection and ID.
func (f *FileStore) Read(collection, id string) (string, error) {
	name := filepath.Join(f.Base, collection, id+".txt")
	text, err := ioutil.ReadFile(name)
	if err != nil {
		return "", err
	}
	return string(text), nil
}

// Write a value to the provided collection and ID.
func (f *FileStore) Write(collection, id, value string) error {
	base := filepath.Join(f.Base, collection)
	err := os.MkdirAll(base, 0755)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(filepath.Join(base, id+".txt"), []byte(value), 0660)
}

// Remove a value from the provided collection and ID.
func (f *FileStore) Remove(collection, id string) error {
	return os.Remove(filepath.Join(f.Base, collection, id+".txt"))
}

// RemoveAll removes all items from a collection.
func (f *FileStore) RemoveAll(collection string) error {
	return os.RemoveAll(filepath.Join(f.Base, collection))
}

// Load all the values for a collection.
func (f *FileStore) Load(collection string) ([]string, error) {
	items := []string{}
	base := filepath.Join(f.Base, collection)
	base, err := filepath.Abs(base)
	if err != nil {
		log.Println("Error getting abs path to", collection, err)
		return nil, err
	}
	log.Println("Loading collection", collection, "from", base)
	filepath.Walk(base, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			// Ignore
			log.Println("store ignoring walk error", err)
			return filepath.SkipDir
		}
		/*
			if info.IsDir() {
				log.Println("skipping dir", path)
				return filepath.SkipDir
			}
		*/
		if filepath.Ext(path) == ".txt" {
			log.Println("loading item", path)
			text, err := ioutil.ReadFile(path)
			if err != nil {
				return err
			}
			items = append(items, string(text))
		} else {
			log.Println("skipping non item", path, filepath.Ext(path))
		}
		return nil
	})
	return items, nil

}

// MockStore is used to test services that rely on Store implementations.
type MockStore struct {
	Data map[string]map[string]string
}

// Read a value from the provided collection with a given ID.
func (s *MockStore) Read(collection, id string) (string, error) {
	c, ok := s.Data[collection]
	if !ok {
		return "", errors.New("No collection found " + collection)
	}
	item, ok := c[id]
	if !ok {
		return "", errors.New("No item with ID found " + id)
	}
	return item, nil
}

// Write a value to the provided collection with a given ID.
func (s *MockStore) Write(collection, id, value string) error {
	c, ok := s.Data[collection]
	if ok {
		c[id] = value
	} else {
		c = map[string]string{id: value}
		if s.Data == nil {
			s.Data = map[string]map[string]string{collection: c}
		} else {
			s.Data[collection] = c
		}
	}
	return nil
}

// Remove a value from the provided collection with a given ID.
func (s *MockStore) Remove(collection, id string) error {
	c, ok := s.Data[collection]
	if ok {
		delete(c, id)
	}
	return nil
}

// RemoveAll clears all items from a collection.
func (s *MockStore) RemoveAll(collection string) error {
	delete(s.Data, collection)
	return nil
}

// Load all the values from a collection.
func (s *MockStore) Load(collection string) ([]string, error) {
	items := []string{}
	for _, item := range s.Data[collection] {
		items = append(items, item)
	}
	return items, nil
}

// Reset removes all data from the store.
func (s *MockStore) Reset() {
	s.Data = map[string]map[string]string{}
}
<|endoftext|>