<commit_before>package vcs

import (
	"encoding/xml"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"time"
)

// NewSvnRepo creates a new instance of SvnRepo. The remote and local directories
// need to be passed in. The remote location should include the branch for SVN.
// For example, if the package is https://github.com/Masterminds/cookoo/ the remote
// should be https://github.com/Masterminds/cookoo/trunk for the trunk branch.
func NewSvnRepo(remote, local string) (*SvnRepo, error) {
	ins := depInstalled("svn")
	if !ins {
		return nil, NewLocalError("svn is not installed", nil, "")
	}
	ltype, err := DetectVcsFromFS(local)

	// Found a VCS other than Svn. Need to report an error.
	if err == nil && ltype != Svn {
		return nil, ErrWrongVCS
	}

	r := &SvnRepo{}
	r.setRemote(remote)
	r.setLocalPath(local)
	r.Logger = Logger

	// Make sure the local SVN repo is configured the same as the remote when
	// A remote value was passed in.
	if err == nil && r.CheckLocal() {
		// An SVN repo was found so test that the URL there matches
		// the repo passed in here.
		out, err := exec.Command("svn", "info", local).CombinedOutput()
		if err != nil {
			return nil, NewLocalError("Unable to retrieve local repo information", err, string(out))
		}

		detectedRemote, err := detectRemoteFromInfoCommand(string(out))
		if err != nil {
			return nil, NewLocalError("Unable to retrieve local repo information", err, string(out))
		}
		if detectedRemote != "" && remote != "" && detectedRemote != remote {
			return nil, ErrWrongRemote
		}

		// If no remote was passed in but one is configured for the locally
		// checked out Svn repo use that one.
		if remote == "" && detectedRemote != "" {
			r.setRemote(detectedRemote)
		}
	}

	return r, nil
}

// SvnRepo implements the Repo interface for the Svn source control.
type SvnRepo struct {
	base
}

// Vcs retrieves the underlying VCS being implemented.
func (s SvnRepo) Vcs() Type {
	return Svn
}

// Get is used to perform an initial checkout of a repository.
// Note, because SVN isn't distributed this is a checkout without
// a clone.
func (s *SvnRepo) Get() error {
	remote := s.Remote()
	if strings.HasPrefix(remote, "/") {
		remote = "file://" + remote
	} else if runtime.GOOS == "windows" && filepath.VolumeName(remote) != "" {
		remote = "file:///" + remote
	}
	out, err := s.run("svn", "checkout", remote, s.LocalPath())
	if err != nil {
		fmt.Println(string(out))
		fmt.Println(err.Error())
		return NewRemoteError("Unable to get repository", err, string(out))
	}
	return nil
}

// Init will create a svn repository at remote location.
func (s *SvnRepo) Init() error {
	out, err := s.run("svnadmin", "create", s.Remote())

	if err != nil && s.isUnableToCreateDir(err) {

		basePath := filepath.Dir(filepath.FromSlash(s.Remote()))
		if _, err := os.Stat(basePath); os.IsNotExist(err) {
			err = os.MkdirAll(basePath, 0755)
			if err != nil {
				return NewLocalError("Unable to initialize repository", err, "")
			}

			out, err = s.run("svnadmin", "create", s.Remote())
			if err != nil {
				return NewLocalError("Unable to initialize repository", err, string(out))
			}
			return nil
		}

	} else if err != nil {
		return NewLocalError("Unable to initialize repository", err, string(out))
	}

	return nil
}

// Update performs an SVN update to an existing checkout.
func (s *SvnRepo) Update() error {
	out, err := s.RunFromDir("svn", "update")
	if err != nil {
		return NewRemoteError("Unable to update repository", err, string(out))
	}
	return err
}

// UpdateVersion sets the version of a package currently checked out via SVN.
func (s *SvnRepo) UpdateVersion(version string) error {
	out, err := s.RunFromDir("svn", "update", "-r", version)
	if err != nil {
		return NewRemoteError("Unable to update checked out version", err, string(out))
	}
	return nil
}

// Version retrieves the current version.
func (s *SvnRepo) Version() (string, error) {
	type Commit struct {
		Revision string `xml:"revision,attr"`
	}
	type Info struct {
		Commit Commit `xml:"entry>commit"`
	}

	out, err := s.RunFromDir("svn", "info", "--xml")
	if err != nil {
		return "", NewLocalError("Unable to retrieve checked out version", err, string(out))
	}
	s.log(out)
	infos := &Info{}
	err = xml.Unmarshal(out, &infos)
	if err != nil {
		return "", NewLocalError("Unable to retrieve checked out version", err, string(out))
	}

	return infos.Commit.Revision, nil
}

// Current returns the current version-ish. This means:
// * HEAD if on the tip.
// * Otherwise a revision id
func (s *SvnRepo) Current() (string, error) {
	tip, err := s.CommitInfo("HEAD")
	if err != nil {
		return "", err
	}

	curr, err := s.Version()
	if err != nil {
		return "", err
	}

	if tip.Commit == curr {
		return "HEAD", nil
	}

	return curr, nil
}

// Date retrieves the date on the latest commit.
func (s *SvnRepo) Date() (time.Time, error) {
	version, err := s.Version()
	if err != nil {
		return time.Time{}, NewLocalError("Unable to retrieve revision date", err, "")
	}
	out, err := s.RunFromDir("svn", "pget", "svn:date", "--revprop", "-r", version)
	if err != nil {
		return time.Time{}, NewLocalError("Unable to retrieve revision date", err, string(out))
	}
	const longForm = "2006-01-02T15:04:05.000000Z"
	t, err := time.Parse(longForm, strings.TrimSpace(string(out)))
	if err != nil {
		return time.Time{}, NewLocalError("Unable to retrieve revision date", err, string(out))
	}
	return t, nil
}

// CheckLocal verifies the local location is an SVN repo.
func (s *SvnRepo) CheckLocal() bool {
	pth, err := filepath.Abs(s.LocalPath())
	if err != nil {
		s.log(err.Error())
		return false
	}

	if _, err := os.Stat(filepath.Join(pth, ".svn")); err == nil {
		return true
	}

	oldpth := pth
	for oldpth != pth {
		pth = filepath.Dir(pth)
		if _, err := os.Stat(filepath.Join(pth, ".svn")); err == nil {
			return true
		}
	}

	return false
}

// Tags returns []string{} as there are no formal tags in SVN. Tags are a
// convention in SVN. They are typically implemented as a copy of the trunk and
// placed in the /tags/[tag name] directory. Since this is a convention the
// expectation is to checkout a tag the correct subdirectory will be used
// as the path. For more information see:
// http://svnbook.red-bean.com/en/1.7/svn.branchmerge.tags.html
func (s *SvnRepo) Tags() ([]string, error) {
	return []string{}, nil
}

// Branches returns []string{} as there are no formal branches in SVN. Branches
// are a convention. They are typically implemented as a copy of the trunk and
// placed in the /branches/[tag name] directory. Since this is a convention the
// expectation is to checkout a branch the correct subdirectory will be used
// as the path. For more information see:
// http://svnbook.red-bean.com/en/1.7/svn.branchmerge.using.html
func (s *SvnRepo) Branches() ([]string, error) {
	return []string{}, nil
}

// IsReference returns if a string is a reference. A reference is a commit id.
// Branches and tags are part of the path.
func (s *SvnRepo) IsReference(r string) bool {
	out, err := s.RunFromDir("svn", "log", "-r", r)

	// This is a complete hack. There must be a better way to do this. Pull
	// requests welcome. When the reference isn't real you get a line of
	// repeated - followed by an empty line. If the reference is real there
	// is commit information in addition to those. So, we look for responses
	// over 2 lines long.
	lines := strings.Split(string(out), "\n")
	if err == nil && len(lines) > 2 {
		return true
	}

	return false
}

// IsDirty returns if the checkout has been modified from the checked
// out reference.
func (s *SvnRepo) IsDirty() bool {
	out, err := s.RunFromDir("svn", "diff")
	return err != nil || len(out) != 0
}

// CommitInfo retrieves metadata about a commit.
func (s *SvnRepo) CommitInfo(id string) (*CommitInfo, error) {

	// There are cases where Svn log doesn't return anything for HEAD or BASE.
	// svn info does provide details for these but does not have elements like
	// the commit message.
	if id == "HEAD" || id == "BASE" {
		type Commit struct {
			Revision string `xml:"revision,attr"`
		}
		type Info struct {
			Commit Commit `xml:"entry>commit"`
		}

		out, err := s.RunFromDir("svn", "info", "-r", id, "--xml")
		if err != nil {
			return nil, NewLocalError("Unable to retrieve commit information", err, string(out))
		}
		infos := &Info{}
		err = xml.Unmarshal(out, &infos)
		if err != nil {
			return nil, NewLocalError("Unable to retrieve commit information", err, string(out))
		}

		id = infos.Commit.Revision
		if id == "" {
			return nil, ErrRevisionUnavailable
		}
	}

	out, err := s.RunFromDir("svn", "log", "-r", id, "--xml")
	if err != nil {
		return nil, NewRemoteError("Unable to retrieve commit information", err, string(out))
	}

	type Logentry struct {
		Author string `xml:"author"`
		Date   string `xml:"date"`
		Msg    string `xml:"msg"`
	}
	type Log struct {
		XMLName xml.Name   `xml:"log"`
		Logs    []Logentry `xml:"logentry"`
	}

	logs := &Log{}
	err = xml.Unmarshal(out, &logs)
	if err != nil {
		return nil, NewLocalError("Unable to retrieve commit information", err, string(out))
	}
	if len(logs.Logs) == 0 {
		return nil, ErrRevisionUnavailable
	}

	ci := &CommitInfo{
		Commit:  id,
		Author:  logs.Logs[0].Author,
		Message: logs.Logs[0].Msg,
	}

	if len(logs.Logs[0].Date) > 0 {
		ci.Date, err = time.Parse(time.RFC3339Nano, logs.Logs[0].Date)
		if err != nil {
			return nil, NewLocalError("Unable to retrieve commit information", err, string(out))
		}
	}

	return ci, nil
}

// TagsFromCommit retrieves tags from a commit id.
func (s *SvnRepo) TagsFromCommit(id string) ([]string, error) {
	// Svn tags are a convention implemented as paths. See the details on the
	// Tag() method for more information.
	return []string{}, nil
}

// Ping returns if remote location is accessible.
func (s *SvnRepo) Ping() bool {
	_, err := s.run("svn", "--non-interactive", "info", s.Remote())
	return err == nil
}

// ExportDir exports the current revision to the passed in directory.
func (s *SvnRepo) ExportDir(dir string) error {

	out, err := s.RunFromDir("svn", "export", ".", dir)
	s.log(out)
	if err != nil {
		return NewLocalError("Unable to export source", err, string(out))
	}

	return nil
}

// isUnableToCreateDir checks for an error in Init() to see if an error
// where the parent directory of the VCS local path doesn't exist.
func (s *SvnRepo) isUnableToCreateDir(err error) bool {
	msg := err.Error()
	return strings.HasPrefix(msg, "E000002")
}

// detectRemoteFromInfoCommand finds the remote url from the `svn info`
// command's output without using  a regex. We avoid regex because URLs
// are notoriously complex to accurately match with a regex and
// splitting strings is less complex and often faster
func detectRemoteFromInfoCommand(infoOut string) (string, error) {
	sBytes := []byte(infoOut)
	urlIndex := strings.Index(infoOut, "URL: ")
	if urlIndex == -1 {
		return "", fmt.Errorf("Remote not specified in svn info")
	}
	urlEndIndex := strings.Index(string(sBytes[urlIndex:]), "\n")
	if urlEndIndex == -1 {
		urlEndIndex = strings.Index(string(sBytes[urlIndex:]), "\r")
		if urlEndIndex == -1 {
			return "", fmt.Errorf("Unable to parse remote URL for svn info")
		}
	}

	return string(sBytes[(urlIndex + 5):(urlIndex + urlEndIndex)]), nil
}
<commit_msg>Removing debug code. oops<commit_after>package vcs

import (
	"encoding/xml"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"time"
)

// NewSvnRepo creates a new instance of SvnRepo. The remote and local directories
// need to be passed in. The remote location should include the branch for SVN.
// For example, if the package is https://github.com/Masterminds/cookoo/ the remote
// should be https://github.com/Masterminds/cookoo/trunk for the trunk branch.
func NewSvnRepo(remote, local string) (*SvnRepo, error) {
	ins := depInstalled("svn")
	if !ins {
		return nil, NewLocalError("svn is not installed", nil, "")
	}
	ltype, err := DetectVcsFromFS(local)

	// Found a VCS other than Svn. Need to report an error.
	if err == nil && ltype != Svn {
		return nil, ErrWrongVCS
	}

	r := &SvnRepo{}
	r.setRemote(remote)
	r.setLocalPath(local)
	r.Logger = Logger

	// Make sure the local SVN repo is configured the same as the remote when
	// A remote value was passed in.
	if err == nil && r.CheckLocal() {
		// An SVN repo was found so test that the URL there matches
		// the repo passed in here.
		out, err := exec.Command("svn", "info", local).CombinedOutput()
		if err != nil {
			return nil, NewLocalError("Unable to retrieve local repo information", err, string(out))
		}

		detectedRemote, err := detectRemoteFromInfoCommand(string(out))
		if err != nil {
			return nil, NewLocalError("Unable to retrieve local repo information", err, string(out))
		}
		if detectedRemote != "" && remote != "" && detectedRemote != remote {
			return nil, ErrWrongRemote
		}

		// If no remote was passed in but one is configured for the locally
		// checked out Svn repo use that one.
		if remote == "" && detectedRemote != "" {
			r.setRemote(detectedRemote)
		}
	}

	return r, nil
}

// SvnRepo implements the Repo interface for the Svn source control.
type SvnRepo struct {
	base
}

// Vcs retrieves the underlying VCS being implemented.
func (s SvnRepo) Vcs() Type {
	return Svn
}

// Get is used to perform an initial checkout of a repository.
// Note, because SVN isn't distributed this is a checkout without
// a clone.
func (s *SvnRepo) Get() error {
	remote := s.Remote()
	if strings.HasPrefix(remote, "/") {
		remote = "file://" + remote
	} else if runtime.GOOS == "windows" && filepath.VolumeName(remote) != "" {
		remote = "file:///" + remote
	}
	out, err := s.run("svn", "checkout", remote, s.LocalPath())
	if err != nil {
		return NewRemoteError("Unable to get repository", err, string(out))
	}
	return nil
}

// Init will create a svn repository at remote location.
func (s *SvnRepo) Init() error {
	out, err := s.run("svnadmin", "create", s.Remote())

	if err != nil && s.isUnableToCreateDir(err) {

		basePath := filepath.Dir(filepath.FromSlash(s.Remote()))
		if _, err := os.Stat(basePath); os.IsNotExist(err) {
			err = os.MkdirAll(basePath, 0755)
			if err != nil {
				return NewLocalError("Unable to initialize repository", err, "")
			}

			out, err = s.run("svnadmin", "create", s.Remote())
			if err != nil {
				return NewLocalError("Unable to initialize repository", err, string(out))
			}
			return nil
		}

	} else if err != nil {
		return NewLocalError("Unable to initialize repository", err, string(out))
	}

	return nil
}

// Update performs an SVN update to an existing checkout.
func (s *SvnRepo) Update() error {
	out, err := s.RunFromDir("svn", "update")
	if err != nil {
		return NewRemoteError("Unable to update repository", err, string(out))
	}
	return err
}

// UpdateVersion sets the version of a package currently checked out via SVN.
func (s *SvnRepo) UpdateVersion(version string) error {
	out, err := s.RunFromDir("svn", "update", "-r", version)
	if err != nil {
		return NewRemoteError("Unable to update checked out version", err, string(out))
	}
	return nil
}

// Version retrieves the current version.
func (s *SvnRepo) Version() (string, error) {
	type Commit struct {
		Revision string `xml:"revision,attr"`
	}
	type Info struct {
		Commit Commit `xml:"entry>commit"`
	}

	out, err := s.RunFromDir("svn", "info", "--xml")
	if err != nil {
		return "", NewLocalError("Unable to retrieve checked out version", err, string(out))
	}
	s.log(out)
	infos := &Info{}
	err = xml.Unmarshal(out, &infos)
	if err != nil {
		return "", NewLocalError("Unable to retrieve checked out version", err, string(out))
	}

	return infos.Commit.Revision, nil
}

// Current returns the current version-ish. This means:
// * HEAD if on the tip.
// * Otherwise a revision id
func (s *SvnRepo) Current() (string, error) {
	tip, err := s.CommitInfo("HEAD")
	if err != nil {
		return "", err
	}

	curr, err := s.Version()
	if err != nil {
		return "", err
	}

	if tip.Commit == curr {
		return "HEAD", nil
	}

	return curr, nil
}

// Date retrieves the date on the latest commit.
func (s *SvnRepo) Date() (time.Time, error) {
	version, err := s.Version()
	if err != nil {
		return time.Time{}, NewLocalError("Unable to retrieve revision date", err, "")
	}
	out, err := s.RunFromDir("svn", "pget", "svn:date", "--revprop", "-r", version)
	if err != nil {
		return time.Time{}, NewLocalError("Unable to retrieve revision date", err, string(out))
	}
	const longForm = "2006-01-02T15:04:05.000000Z"
	t, err := time.Parse(longForm, strings.TrimSpace(string(out)))
	if err != nil {
		return time.Time{}, NewLocalError("Unable to retrieve revision date", err, string(out))
	}
	return t, nil
}

// CheckLocal verifies the local location is an SVN repo.
func (s *SvnRepo) CheckLocal() bool {
	pth, err := filepath.Abs(s.LocalPath())
	if err != nil {
		s.log(err.Error())
		return false
	}

	if _, err := os.Stat(filepath.Join(pth, ".svn")); err == nil {
		return true
	}

	oldpth := pth
	for oldpth != pth {
		pth = filepath.Dir(pth)
		if _, err := os.Stat(filepath.Join(pth, ".svn")); err == nil {
			return true
		}
	}

	return false
}

// Tags returns []string{} as there are no formal tags in SVN. Tags are a
// convention in SVN. They are typically implemented as a copy of the trunk and
// placed in the /tags/[tag name] directory. Since this is a convention the
// expectation is to checkout a tag the correct subdirectory will be used
// as the path. For more information see:
// http://svnbook.red-bean.com/en/1.7/svn.branchmerge.tags.html
func (s *SvnRepo) Tags() ([]string, error) {
	return []string{}, nil
}

// Branches returns []string{} as there are no formal branches in SVN. Branches
// are a convention. They are typically implemented as a copy of the trunk and
// placed in the /branches/[tag name] directory. Since this is a convention the
// expectation is to checkout a branch the correct subdirectory will be used
// as the path. For more information see:
// http://svnbook.red-bean.com/en/1.7/svn.branchmerge.using.html
func (s *SvnRepo) Branches() ([]string, error) {
	return []string{}, nil
}

// IsReference returns if a string is a reference. A reference is a commit id.
// Branches and tags are part of the path.
func (s *SvnRepo) IsReference(r string) bool {
	out, err := s.RunFromDir("svn", "log", "-r", r)

	// This is a complete hack. There must be a better way to do this. Pull
	// requests welcome. When the reference isn't real you get a line of
	// repeated - followed by an empty line. If the reference is real there
	// is commit information in addition to those. So, we look for responses
	// over 2 lines long.
	lines := strings.Split(string(out), "\n")
	if err == nil && len(lines) > 2 {
		return true
	}

	return false
}

// IsDirty returns if the checkout has been modified from the checked
// out reference.
func (s *SvnRepo) IsDirty() bool {
	out, err := s.RunFromDir("svn", "diff")
	return err != nil || len(out) != 0
}

// CommitInfo retrieves metadata about a commit.
func (s *SvnRepo) CommitInfo(id string) (*CommitInfo, error) {

	// There are cases where Svn log doesn't return anything for HEAD or BASE.
	// svn info does provide details for these but does not have elements like
	// the commit message.
	if id == "HEAD" || id == "BASE" {
		type Commit struct {
			Revision string `xml:"revision,attr"`
		}
		type Info struct {
			Commit Commit `xml:"entry>commit"`
		}

		out, err := s.RunFromDir("svn", "info", "-r", id, "--xml")
		if err != nil {
			return nil, NewLocalError("Unable to retrieve commit information", err, string(out))
		}
		infos := &Info{}
		err = xml.Unmarshal(out, &infos)
		if err != nil {
			return nil, NewLocalError("Unable to retrieve commit information", err, string(out))
		}

		id = infos.Commit.Revision
		if id == "" {
			return nil, ErrRevisionUnavailable
		}
	}

	out, err := s.RunFromDir("svn", "log", "-r", id, "--xml")
	if err != nil {
		return nil, NewRemoteError("Unable to retrieve commit information", err, string(out))
	}

	type Logentry struct {
		Author string `xml:"author"`
		Date   string `xml:"date"`
		Msg    string `xml:"msg"`
	}
	type Log struct {
		XMLName xml.Name   `xml:"log"`
		Logs    []Logentry `xml:"logentry"`
	}

	logs := &Log{}
	err = xml.Unmarshal(out, &logs)
	if err != nil {
		return nil, NewLocalError("Unable to retrieve commit information", err, string(out))
	}
	if len(logs.Logs) == 0 {
		return nil, ErrRevisionUnavailable
	}

	ci := &CommitInfo{
		Commit:  id,
		Author:  logs.Logs[0].Author,
		Message: logs.Logs[0].Msg,
	}

	if len(logs.Logs[0].Date) > 0 {
		ci.Date, err = time.Parse(time.RFC3339Nano, logs.Logs[0].Date)
		if err != nil {
			return nil, NewLocalError("Unable to retrieve commit information", err, string(out))
		}
	}

	return ci, nil
}

// TagsFromCommit retrieves tags from a commit id.
func (s *SvnRepo) TagsFromCommit(id string) ([]string, error) {
	// Svn tags are a convention implemented as paths. See the details on the
	// Tag() method for more information.
	return []string{}, nil
}

// Ping returns if remote location is accessible.
func (s *SvnRepo) Ping() bool {
	_, err := s.run("svn", "--non-interactive", "info", s.Remote())
	return err == nil
}

// ExportDir exports the current revision to the passed in directory.
func (s *SvnRepo) ExportDir(dir string) error {

	out, err := s.RunFromDir("svn", "export", ".", dir)
	s.log(out)
	if err != nil {
		return NewLocalError("Unable to export source", err, string(out))
	}

	return nil
}

// isUnableToCreateDir checks for an error in Init() to see if an error
// where the parent directory of the VCS local path doesn't exist.
func (s *SvnRepo) isUnableToCreateDir(err error) bool {
	msg := err.Error()
	return strings.HasPrefix(msg, "E000002")
}

// detectRemoteFromInfoCommand finds the remote url from the `svn info`
// command's output without using  a regex. We avoid regex because URLs
// are notoriously complex to accurately match with a regex and
// splitting strings is less complex and often faster
func detectRemoteFromInfoCommand(infoOut string) (string, error) {
	sBytes := []byte(infoOut)
	urlIndex := strings.Index(infoOut, "URL: ")
	if urlIndex == -1 {
		return "", fmt.Errorf("Remote not specified in svn info")
	}
	urlEndIndex := strings.Index(string(sBytes[urlIndex:]), "\n")
	if urlEndIndex == -1 {
		urlEndIndex = strings.Index(string(sBytes[urlIndex:]), "\r")
		if urlEndIndex == -1 {
			return "", fmt.Errorf("Unable to parse remote URL for svn info")
		}
	}

	return string(sBytes[(urlIndex + 5):(urlIndex + urlEndIndex)]), nil
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"

	"encoding/json"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/batch"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/helper/structure"
	"github.com/hashicorp/terraform/helper/validation"
)

func resourceAwsBatchJobDefinition() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsBatchJobDefinitionCreate,
		Read:   resourceAwsBatchJobDefinitionRead,
		Delete: resourceAwsBatchJobDefinitionDelete,

		Schema: map[string]*schema.Schema{
			"name": {
				Type:         schema.TypeString,
				Required:     true,
				ForceNew:     true,
				ValidateFunc: validateBatchName,
			},
			"container_properties": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				StateFunc: func(v interface{}) string {
					json, _ := structure.NormalizeJsonString(v)
					return json
				},
				DiffSuppressFunc: suppressEquivalentJsonDiffs,
				ValidateFunc:     validateAwsBatchJobContainerProperties,
			},
			"parameters": {
				Type:     schema.TypeMap,
				Optional: true,
				ForceNew: true,
				Elem:     schema.TypeString,
			},
			"retry_strategy": {
				Type:     schema.TypeList,
				Optional: true,
				ForceNew: true,
				MaxItems: 1,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"attempts": {
							Type:     schema.TypeInt,
							Required: true,
						},
					},
				},
			},
			"timeout": {
				Type:     schema.TypeList,
				Optional: true,
				ForceNew: true,
				MaxItems: 1,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"attempt_duration_seconds": {
							Type:         schema.TypeInt,
							Required:     true,
							ValidateFunc: validation.IntAtLeast(60),
						},
					},
				},
			},
			"type": {
				Type:         schema.TypeString,
				Required:     true,
				ForceNew:     true,
				ValidateFunc: validation.StringInSlice([]string{batch.JobDefinitionTypeContainer}, true),
			},
			"revision": {
				Type:     schema.TypeInt,
				Computed: true,
			},
			"arn": {
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func resourceAwsBatchJobDefinitionCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).batchconn
	name := d.Get("name").(string)

	input := &batch.RegisterJobDefinitionInput{
		JobDefinitionName: aws.String(name),
		Type:              aws.String(d.Get("type").(string)),
	}

	if v, ok := d.GetOk("container_properties"); ok {
		props, err := expandBatchJobContainerProperties(v.(string))
		if err != nil {
			return fmt.Errorf("%s %q", err, name)
		}
		input.ContainerProperties = props
	}

	if v, ok := d.GetOk("parameters"); ok {
		input.Parameters = expandJobDefinitionParameters(v.(map[string]interface{}))
	}

	if v, ok := d.GetOk("retry_strategy"); ok {
		input.RetryStrategy = expandJobDefinitionRetryStrategy(v.([]interface{}))
	}

	if v, ok := d.GetOk("timeout"); ok {
		input.Timeout = expandJobDefinitionTimeout(v.([]interface{}))
	}

	out, err := conn.RegisterJobDefinition(input)
	if err != nil {
		return fmt.Errorf("%s %q", err, name)
	}
	d.SetId(*out.JobDefinitionArn)
	d.Set("arn", out.JobDefinitionArn)
	return resourceAwsBatchJobDefinitionRead(d, meta)
}

func resourceAwsBatchJobDefinitionRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).batchconn
	arn := d.Get("arn").(string)
	job, err := getJobDefinition(conn, arn)
	if err != nil {
		return fmt.Errorf("%s %q", err, arn)
	}
	if job == nil {
		d.SetId("")
		return nil
	}
	d.Set("arn", job.JobDefinitionArn)
	d.Set("container_properties", job.ContainerProperties)
	d.Set("parameters", aws.StringValueMap(job.Parameters))
	d.Set("retry_strategy", flattenRetryStrategy(job.RetryStrategy))
	d.Set("timeout", flattenTimeout(job.Timeout))
	d.Set("revision", job.Revision)
	d.Set("type", job.Type)
	return nil
}

func resourceAwsBatchJobDefinitionDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).batchconn
	arn := d.Get("arn").(string)
	_, err := conn.DeregisterJobDefinition(&batch.DeregisterJobDefinitionInput{
		JobDefinition: aws.String(arn),
	})
	if err != nil {
		return fmt.Errorf("%s %q", err, arn)
	}
	d.SetId("")
	return nil
}

func getJobDefinition(conn *batch.Batch, arn string) (*batch.JobDefinition, error) {
	describeOpts := &batch.DescribeJobDefinitionsInput{
		JobDefinitions: []*string{aws.String(arn)},
	}
	resp, err := conn.DescribeJobDefinitions(describeOpts)
	if err != nil {
		return nil, err
	}

	numJobDefinitions := len(resp.JobDefinitions)
	switch {
	case numJobDefinitions == 0:
		return nil, nil
	case numJobDefinitions == 1:
		if *resp.JobDefinitions[0].Status == "ACTIVE" {
			return resp.JobDefinitions[0], nil
		}
		return nil, nil
	case numJobDefinitions > 1:
		return nil, fmt.Errorf("Multiple Job Definitions with name %s", arn)
	}
	return nil, nil
}

func validateAwsBatchJobContainerProperties(v interface{}, k string) (ws []string, errors []error) {
	value := v.(string)
	_, err := expandBatchJobContainerProperties(value)
	if err != nil {
		errors = append(errors, fmt.Errorf("AWS Batch Job container_properties is invalid: %s", err))
	}
	return
}

func expandBatchJobContainerProperties(rawProps string) (*batch.ContainerProperties, error) {
	var props *batch.ContainerProperties

	err := json.Unmarshal([]byte(rawProps), &props)
	if err != nil {
		return nil, fmt.Errorf("Error decoding JSON: %s", err)
	}

	return props, nil
}

func expandJobDefinitionParameters(params map[string]interface{}) map[string]*string {
	var jobParams = make(map[string]*string)
	for k, v := range params {
		jobParams[k] = aws.String(v.(string))
	}

	return jobParams
}

func expandJobDefinitionRetryStrategy(item []interface{}) *batch.RetryStrategy {
	data := item[0].(map[string]interface{})
	return &batch.RetryStrategy{
		Attempts: aws.Int64(int64(data["attempts"].(int))),
	}
}

func flattenRetryStrategy(item *batch.RetryStrategy) []map[string]interface{} {
	data := []map[string]interface{}{}
	if item != nil {
		data = append(data, map[string]interface{}{
			"attempts": item.Attempts,
		})
	}
	return data
}

func expandJobDefinitionTimeout(item []interface{}) *batch.JobTimeout {
	data := item[0].(map[string]interface{})
	return &batch.JobTimeout{
		AttemptDurationSeconds: aws.Int64(int64(data["attempt_duration_seconds"].(int))),
	}
}

func flattenTimeout(item *batch.JobTimeout) []map[string]interface{} {
	data := []map[string]interface{}{}
	if item != nil {
		data = append(data, map[string]interface{}{
			"attempt_duration_seconds": item.AttemptDurationSeconds,
		})
	}
	return data
}
<commit_msg>Add error checking to retry_strategy and timeout<commit_after>package aws

import (
	"fmt"

	"encoding/json"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/batch"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/helper/structure"
	"github.com/hashicorp/terraform/helper/validation"
)

func resourceAwsBatchJobDefinition() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsBatchJobDefinitionCreate,
		Read:   resourceAwsBatchJobDefinitionRead,
		Delete: resourceAwsBatchJobDefinitionDelete,

		Schema: map[string]*schema.Schema{
			"name": {
				Type:         schema.TypeString,
				Required:     true,
				ForceNew:     true,
				ValidateFunc: validateBatchName,
			},
			"container_properties": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				StateFunc: func(v interface{}) string {
					json, _ := structure.NormalizeJsonString(v)
					return json
				},
				DiffSuppressFunc: suppressEquivalentJsonDiffs,
				ValidateFunc:     validateAwsBatchJobContainerProperties,
			},
			"parameters": {
				Type:     schema.TypeMap,
				Optional: true,
				ForceNew: true,
				Elem:     schema.TypeString,
			},
			"retry_strategy": {
				Type:     schema.TypeList,
				Optional: true,
				ForceNew: true,
				MaxItems: 1,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"attempts": {
							Type:     schema.TypeInt,
							Required: true,
						},
					},
				},
			},
			"timeout": {
				Type:     schema.TypeList,
				Optional: true,
				ForceNew: true,
				MaxItems: 1,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"attempt_duration_seconds": {
							Type:         schema.TypeInt,
							Required:     true,
							ValidateFunc: validation.IntAtLeast(60),
						},
					},
				},
			},
			"type": {
				Type:         schema.TypeString,
				Required:     true,
				ForceNew:     true,
				ValidateFunc: validation.StringInSlice([]string{batch.JobDefinitionTypeContainer}, true),
			},
			"revision": {
				Type:     schema.TypeInt,
				Computed: true,
			},
			"arn": {
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func resourceAwsBatchJobDefinitionCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).batchconn
	name := d.Get("name").(string)

	input := &batch.RegisterJobDefinitionInput{
		JobDefinitionName: aws.String(name),
		Type:              aws.String(d.Get("type").(string)),
	}

	if v, ok := d.GetOk("container_properties"); ok {
		props, err := expandBatchJobContainerProperties(v.(string))
		if err != nil {
			return fmt.Errorf("%s %q", err, name)
		}
		input.ContainerProperties = props
	}

	if v, ok := d.GetOk("parameters"); ok {
		input.Parameters = expandJobDefinitionParameters(v.(map[string]interface{}))
	}

	if v, ok := d.GetOk("retry_strategy"); ok {
		input.RetryStrategy = expandJobDefinitionRetryStrategy(v.([]interface{}))
	}

	if v, ok := d.GetOk("timeout"); ok {
		input.Timeout = expandJobDefinitionTimeout(v.([]interface{}))
	}

	out, err := conn.RegisterJobDefinition(input)
	if err != nil {
		return fmt.Errorf("%s %q", err, name)
	}
	d.SetId(*out.JobDefinitionArn)
	d.Set("arn", out.JobDefinitionArn)
	return resourceAwsBatchJobDefinitionRead(d, meta)
}

func resourceAwsBatchJobDefinitionRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).batchconn
	arn := d.Get("arn").(string)
	job, err := getJobDefinition(conn, arn)
	if err != nil {
		return fmt.Errorf("%s %q", err, arn)
	}
	if job == nil {
		d.SetId("")
		return nil
	}
	d.Set("arn", job.JobDefinitionArn)
	d.Set("container_properties", job.ContainerProperties)
	d.Set("parameters", aws.StringValueMap(job.Parameters))

	if err := d.Set("retry_strategy", flattenBatchRetryStrategy(job.RetryStrategy)); err != nil {
		return fmt.Errorf("error setting retry_strategy: %s", err)
	}

	if err := d.Set("timeout", flattenBatchJobTimeout(job.Timeout)); err != nil {
		return fmt.Errorf("error setting timeout: %s", err)
	}

	d.Set("revision", job.Revision)
	d.Set("type", job.Type)
	return nil
}

func resourceAwsBatchJobDefinitionDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).batchconn
	arn := d.Get("arn").(string)
	_, err := conn.DeregisterJobDefinition(&batch.DeregisterJobDefinitionInput{
		JobDefinition: aws.String(arn),
	})
	if err != nil {
		return fmt.Errorf("%s %q", err, arn)
	}
	d.SetId("")
	return nil
}

func getJobDefinition(conn *batch.Batch, arn string) (*batch.JobDefinition, error) {
	describeOpts := &batch.DescribeJobDefinitionsInput{
		JobDefinitions: []*string{aws.String(arn)},
	}
	resp, err := conn.DescribeJobDefinitions(describeOpts)
	if err != nil {
		return nil, err
	}

	numJobDefinitions := len(resp.JobDefinitions)
	switch {
	case numJobDefinitions == 0:
		return nil, nil
	case numJobDefinitions == 1:
		if *resp.JobDefinitions[0].Status == "ACTIVE" {
			return resp.JobDefinitions[0], nil
		}
		return nil, nil
	case numJobDefinitions > 1:
		return nil, fmt.Errorf("Multiple Job Definitions with name %s", arn)
	}
	return nil, nil
}

func validateAwsBatchJobContainerProperties(v interface{}, k string) (ws []string, errors []error) {
	value := v.(string)
	_, err := expandBatchJobContainerProperties(value)
	if err != nil {
		errors = append(errors, fmt.Errorf("AWS Batch Job container_properties is invalid: %s", err))
	}
	return
}

func expandBatchJobContainerProperties(rawProps string) (*batch.ContainerProperties, error) {
	var props *batch.ContainerProperties

	err := json.Unmarshal([]byte(rawProps), &props)
	if err != nil {
		return nil, fmt.Errorf("Error decoding JSON: %s", err)
	}

	return props, nil
}

func expandJobDefinitionParameters(params map[string]interface{}) map[string]*string {
	var jobParams = make(map[string]*string)
	for k, v := range params {
		jobParams[k] = aws.String(v.(string))
	}

	return jobParams
}

func expandJobDefinitionRetryStrategy(item []interface{}) *batch.RetryStrategy {
	data := item[0].(map[string]interface{})
	return &batch.RetryStrategy{
		Attempts: aws.Int64(int64(data["attempts"].(int))),
	}
}

func flattenBatchRetryStrategy(item *batch.RetryStrategy) []map[string]interface{} {
	data := []map[string]interface{}{}
	if item != nil {
		data = append(data, map[string]interface{}{
			"attempts": item.Attempts,
		})
	}
	return data
}

func expandJobDefinitionTimeout(item []interface{}) *batch.JobTimeout {
	data := item[0].(map[string]interface{})
	return &batch.JobTimeout{
		AttemptDurationSeconds: aws.Int64(int64(data["attempt_duration_seconds"].(int))),
	}
}

func flattenBatchJobTimeout(item *batch.JobTimeout) []map[string]interface{} {
	data := []map[string]interface{}{}
	if item != nil {
		data = append(data, map[string]interface{}{
			"attempt_duration_seconds": item.AttemptDurationSeconds,
		})
	}
	return data
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"

	"github.com/hashicorp/terraform/helper/schema"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/codepipeline"
)

func resourceAwsCodePipelineWebhook() *schema.Resource {
	return &schema.Resource{
		Create:        resourceAwsCodePipelineWebhookCreate,
		Read:          resourceAwsCodePipelineWebhookRead,
		Update:        nil,
		Delete:        resourceAwsCodePipelineWebhookDelete,
		SchemaVersion: 1,

		Schema: map[string]*schema.Schema{
			"auth": {
				Type:     schema.TypeList,
				MaxItems: 1,
				MinItems: 1,
				Required: true,
				ForceNew: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"type": {
							Type:     schema.TypeString,
							Required: true,
						},
						"secret_token": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"allowed_ip_range": {
							Type:     schema.TypeString,
							Optional: true,
						},
					},
				},
			},
			"filter": {
				Type:     schema.TypeSet,
				Optional: true,
				ForceNew: true,
				MinItems: 1,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"json_path": {
							Type:     schema.TypeString,
							Required: true,
						},

						"match_equals": {
							Type:     schema.TypeString,
							Required: true,
						},
					},
				},
			},
			"name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"url": {
				Type:     schema.TypeString,
				Computed: true,
			},

			"target": {
				Type:     schema.TypeList,
				MaxItems: 1,
				MinItems: 1,
				Required: true,
				ForceNew: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"action": {
							Type:     schema.TypeString,
							Required: true,
						},
						"pipeline": {
							Type:     schema.TypeString,
							Required: true,
						},
					},
				},
			},
		},
	}
}

func flattenWebhookAttr(d *schema.ResourceData, attr string) (map[string]interface{}, error) {
	if v, ok := d.GetOk(attr); ok {
		l := v.([]interface{})
		if len(l) <= 0 {
			return nil, fmt.Errorf("Attribute %s is missing", attr)
		}

		data := l[0].(map[string]interface{})
		return data, nil
	}

	return nil, fmt.Errorf("Could not find attribute %s", attr)
}

func resourceAwsCodePipelineWebhookCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).codepipelineconn

	auth, err := flattenWebhookAttr(d, "auth")
	if err != nil {
		return err
	}

	target, err := flattenWebhookAttr(d, "target")
	if err != nil {
		return err
	}

	filters := d.Get("filter").(*schema.Set)

	var rules []*codepipeline.WebhookFilterRule

	for _, f := range filters.List() {
		r := f.(map[string]interface{})
		filter := codepipeline.WebhookFilterRule{
			JsonPath:    aws.String(r["json_path"].(string)),
			MatchEquals: aws.String(r["match_equals"].(string)),
		}

		rules = append(rules, &filter)
	}

	if len(rules) <= 0 {
		return fmt.Errorf("One or more webhook filter rule is required (%d rules from %d filter blocks)", len(rules), len(filters.List()))
	}

	var authConfig codepipeline.WebhookAuthConfiguration
	switch auth["type"].(string) {
	case "IP":
		ipRange := auth["allowed_ip_range"].(string)
		if ipRange == "" {
			return fmt.Errorf("An IP range must be set when using IP-based auth")
		}

		authConfig.AllowedIPRange = &ipRange

		break
	case "GITHUB_HMAC":
		secretToken := auth["secret_token"].(string)
		if secretToken == "" {
			return fmt.Errorf("Secret token must be set when using GITHUB_HMAC")
		}

		authConfig.SecretToken = &secretToken
		break
	case "UNAUTHENTICATED":
		break
	default:
		return fmt.Errorf("Invalid authentication type %s", auth["type"])
	}

	request := &codepipeline.PutWebhookInput{
		Webhook: &codepipeline.WebhookDefinition{
			Authentication: aws.String(auth["type"].(string)),
			Filters:        rules,
			Name:           aws.String(d.Get("name").(string)),
			TargetAction:   aws.String(target["action"].(string)),
			TargetPipeline: aws.String(target["pipeline"].(string)),
		},
	}

	request.Webhook.AuthenticationConfiguration = &authConfig
	webhook, err := conn.PutWebhook(request)
	if err != nil {
		return fmt.Errorf("Error creating webhook: %s", err)
	}

	arn := *webhook.Webhook.Arn
	d.SetId(arn)

	url := *webhook.Webhook.Url
	d.Set("url", url)

	return resourceAwsCodePipelineWebhookRead(d, meta)
}

func getAllCodePipelineWebhooks(conn *codepipeline.CodePipeline) ([]*codepipeline.ListWebhookItem, error) {
	var webhooks []*codepipeline.ListWebhookItem
	var nextToken string

	for {
		input := &codepipeline.ListWebhooksInput{
			MaxResults: aws.Int64(int64(60)),
		}
		if nextToken != "" {
			input.NextToken = aws.String(nextToken)
		}

		out, err := conn.ListWebhooks(input)
		if err != nil {
			return webhooks, err
		}

		webhooks = append(webhooks, out.Webhooks...)

		if out.NextToken == nil {
			break
		}

		nextToken = aws.StringValue(out.NextToken)
	}

	return webhooks, nil
}

func setCodePipelineWebhookFilters(webhook codepipeline.WebhookDefinition, d *schema.ResourceData) error {
	filters := []interface{}{}
	for _, filter := range webhook.Filters {
		f := map[string]interface{}{
			"json_path":    *filter.JsonPath,
			"match_equals": *filter.MatchEquals,
		}
		filters = append(filters, f)
	}

	if err := d.Set("filter", filters); err != nil {
		return err
	}

	return nil
}

func setCodePipelineWebhookAuthentication(webhook codepipeline.WebhookDefinition, d *schema.ResourceData) error {
	var result []interface{}

	auth := map[string]interface{}{}

	authType := *webhook.Authentication
	auth["type"] = authType

	if webhook.AuthenticationConfiguration.AllowedIPRange != nil {
		ipRange := *webhook.AuthenticationConfiguration.AllowedIPRange
		auth["allowed_ip_range"] = ipRange
	}

	if webhook.AuthenticationConfiguration.SecretToken != nil {
		secretToken := *webhook.AuthenticationConfiguration.SecretToken
		auth["secret_token"] = secretToken
	}

	result = append(result, auth)
	if err := d.Set("auth", result); err != nil {
		return err
	}

	return nil
}

func resourceAwsCodePipelineWebhookRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).codepipelineconn
	arn := d.Id()
	webhooks, err := getAllCodePipelineWebhooks(conn)
	if err != nil {
		return fmt.Errorf("Error fetching webhooks: %s", err)
	}

	if len(webhooks) == 0 {
		return fmt.Errorf("No webhooks returned!")
	}

	var found codepipeline.WebhookDefinition
	for _, w := range webhooks {
		a := *w.Arn
		if a == arn {
			found = *w.Definition
			break
		}
	}

	name := *found.Name
	if name == "" {
		return fmt.Errorf("Webhook not found: %s", arn)
	}

	d.Set("name", name)

	if err = setCodePipelineWebhookAuthentication(found, d); err != nil {
		return err
	}

	if err = setCodePipelineWebhookFilters(found, d); err != nil {
		return err
	}

	var t []interface{}
	target := map[string]interface{}{
		"action":   *found.TargetAction,
		"pipeline": *found.TargetPipeline,
	}
	t = append(t, target)

	if err := d.Set("target", t); err != nil {
		return err
	}

	return nil
}

func resourceAwsCodePipelineWebhookDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).codepipelineconn
	name := d.Get("name").(string)

	input := codepipeline.DeleteWebhookInput{
		Name: &name,
	}
	_, err := conn.DeleteWebhook(&input)

	if err != nil {
		return fmt.Errorf("Could not delete webhook: %s", err)
	}

	return nil
}
<commit_msg>SetId to blank to remove from state.<commit_after>package aws

import (
	"fmt"

	"github.com/hashicorp/terraform/helper/schema"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/codepipeline"
)

func resourceAwsCodePipelineWebhook() *schema.Resource {
	return &schema.Resource{
		Create:        resourceAwsCodePipelineWebhookCreate,
		Read:          resourceAwsCodePipelineWebhookRead,
		Update:        nil,
		Delete:        resourceAwsCodePipelineWebhookDelete,
		SchemaVersion: 1,

		Schema: map[string]*schema.Schema{
			"auth": {
				Type:     schema.TypeList,
				MaxItems: 1,
				MinItems: 1,
				Required: true,
				ForceNew: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"type": {
							Type:     schema.TypeString,
							Required: true,
						},
						"secret_token": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"allowed_ip_range": {
							Type:     schema.TypeString,
							Optional: true,
						},
					},
				},
			},
			"filter": {
				Type:     schema.TypeSet,
				Optional: true,
				ForceNew: true,
				MinItems: 1,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"json_path": {
							Type:     schema.TypeString,
							Required: true,
						},

						"match_equals": {
							Type:     schema.TypeString,
							Required: true,
						},
					},
				},
			},
			"name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"url": {
				Type:     schema.TypeString,
				Computed: true,
			},

			"target": {
				Type:     schema.TypeList,
				MaxItems: 1,
				MinItems: 1,
				Required: true,
				ForceNew: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"action": {
							Type:     schema.TypeString,
							Required: true,
						},
						"pipeline": {
							Type:     schema.TypeString,
							Required: true,
						},
					},
				},
			},
		},
	}
}

func flattenWebhookAttr(d *schema.ResourceData, attr string) (map[string]interface{}, error) {
	if v, ok := d.GetOk(attr); ok {
		l := v.([]interface{})
		if len(l) <= 0 {
			return nil, fmt.Errorf("Attribute %s is missing", attr)
		}

		data := l[0].(map[string]interface{})
		return data, nil
	}

	return nil, fmt.Errorf("Could not find attribute %s", attr)
}

func resourceAwsCodePipelineWebhookCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).codepipelineconn

	auth, err := flattenWebhookAttr(d, "auth")
	if err != nil {
		return err
	}

	target, err := flattenWebhookAttr(d, "target")
	if err != nil {
		return err
	}

	filters := d.Get("filter").(*schema.Set)

	var rules []*codepipeline.WebhookFilterRule

	for _, f := range filters.List() {
		r := f.(map[string]interface{})
		filter := codepipeline.WebhookFilterRule{
			JsonPath:    aws.String(r["json_path"].(string)),
			MatchEquals: aws.String(r["match_equals"].(string)),
		}

		rules = append(rules, &filter)
	}

	if len(rules) <= 0 {
		return fmt.Errorf("One or more webhook filter rule is required (%d rules from %d filter blocks)", len(rules), len(filters.List()))
	}

	var authConfig codepipeline.WebhookAuthConfiguration
	switch auth["type"].(string) {
	case "IP":
		ipRange := auth["allowed_ip_range"].(string)
		if ipRange == "" {
			return fmt.Errorf("An IP range must be set when using IP-based auth")
		}

		authConfig.AllowedIPRange = &ipRange

		break
	case "GITHUB_HMAC":
		secretToken := auth["secret_token"].(string)
		if secretToken == "" {
			return fmt.Errorf("Secret token must be set when using GITHUB_HMAC")
		}

		authConfig.SecretToken = &secretToken
		break
	case "UNAUTHENTICATED":
		break
	default:
		return fmt.Errorf("Invalid authentication type %s", auth["type"])
	}

	request := &codepipeline.PutWebhookInput{
		Webhook: &codepipeline.WebhookDefinition{
			Authentication: aws.String(auth["type"].(string)),
			Filters:        rules,
			Name:           aws.String(d.Get("name").(string)),
			TargetAction:   aws.String(target["action"].(string)),
			TargetPipeline: aws.String(target["pipeline"].(string)),
		},
	}

	request.Webhook.AuthenticationConfiguration = &authConfig
	webhook, err := conn.PutWebhook(request)
	if err != nil {
		return fmt.Errorf("Error creating webhook: %s", err)
	}

	arn := *webhook.Webhook.Arn
	d.SetId(arn)

	url := *webhook.Webhook.Url
	d.Set("url", url)

	return resourceAwsCodePipelineWebhookRead(d, meta)
}

func getAllCodePipelineWebhooks(conn *codepipeline.CodePipeline) ([]*codepipeline.ListWebhookItem, error) {
	var webhooks []*codepipeline.ListWebhookItem
	var nextToken string

	for {
		input := &codepipeline.ListWebhooksInput{
			MaxResults: aws.Int64(int64(60)),
		}
		if nextToken != "" {
			input.NextToken = aws.String(nextToken)
		}

		out, err := conn.ListWebhooks(input)
		if err != nil {
			return webhooks, err
		}

		webhooks = append(webhooks, out.Webhooks...)

		if out.NextToken == nil {
			break
		}

		nextToken = aws.StringValue(out.NextToken)
	}

	return webhooks, nil
}

func setCodePipelineWebhookFilters(webhook codepipeline.WebhookDefinition, d *schema.ResourceData) error {
	filters := []interface{}{}
	for _, filter := range webhook.Filters {
		f := map[string]interface{}{
			"json_path":    *filter.JsonPath,
			"match_equals": *filter.MatchEquals,
		}
		filters = append(filters, f)
	}

	if err := d.Set("filter", filters); err != nil {
		return err
	}

	return nil
}

func setCodePipelineWebhookAuthentication(webhook codepipeline.WebhookDefinition, d *schema.ResourceData) error {
	var result []interface{}

	auth := map[string]interface{}{}

	authType := *webhook.Authentication
	auth["type"] = authType

	if webhook.AuthenticationConfiguration.AllowedIPRange != nil {
		ipRange := *webhook.AuthenticationConfiguration.AllowedIPRange
		auth["allowed_ip_range"] = ipRange
	}

	if webhook.AuthenticationConfiguration.SecretToken != nil {
		secretToken := *webhook.AuthenticationConfiguration.SecretToken
		auth["secret_token"] = secretToken
	}

	result = append(result, auth)
	if err := d.Set("auth", result); err != nil {
		return err
	}

	return nil
}

func resourceAwsCodePipelineWebhookRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).codepipelineconn
	arn := d.Id()
	webhooks, err := getAllCodePipelineWebhooks(conn)
	if err != nil {
		return fmt.Errorf("Error fetching webhooks: %s", err)
	}

	if len(webhooks) == 0 {
		return fmt.Errorf("No webhooks returned!")
	}

	var found codepipeline.WebhookDefinition
	for _, w := range webhooks {
		a := *w.Arn
		if a == arn {
			found = *w.Definition
			break
		}
	}

	name := *found.Name
	if name == "" {
		return fmt.Errorf("Webhook not found: %s", arn)
	}

	d.Set("name", name)

	if err = setCodePipelineWebhookAuthentication(found, d); err != nil {
		return err
	}

	if err = setCodePipelineWebhookFilters(found, d); err != nil {
		return err
	}

	var t []interface{}
	target := map[string]interface{}{
		"action":   *found.TargetAction,
		"pipeline": *found.TargetPipeline,
	}
	t = append(t, target)

	if err := d.Set("target", t); err != nil {
		return err
	}

	return nil
}

func resourceAwsCodePipelineWebhookDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).codepipelineconn
	name := d.Get("name").(string)

	input := codepipeline.DeleteWebhookInput{
		Name: &name,
	}
	_, err := conn.DeleteWebhook(&input)

	if err != nil {
		return fmt.Errorf("Could not delete webhook: %s", err)
	}

	d.SetId("")

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/centurylinkcloud/clc-go-cli/auth"
	"github.com/centurylinkcloud/clc-go-cli/command_loader"
	"github.com/centurylinkcloud/clc-go-cli/config"
	"github.com/centurylinkcloud/clc-go-cli/formatter_provider"
	"github.com/centurylinkcloud/clc-go-cli/model_adjuster"
	"github.com/centurylinkcloud/clc-go-cli/model_loader"
	"github.com/centurylinkcloud/clc-go-cli/model_validator"
	"github.com/centurylinkcloud/clc-go-cli/options"
	"github.com/centurylinkcloud/clc-go-cli/parser"

	"io"
)

func run(args []string) string {
	if len(args) == 0 {
		return ussage()
	}
	cmdArg := ""
	optionArgs := args[1:]
	if len(args) >= 2 {
		cmdArg = args[1]
		optionArgs = args[2:]
	}
	cmd, err := command_loader.LoadCommand(args[0], cmdArg)
	if err != nil {
		return err.Error()
	}
	parsedArgs, err := parser.ParseArguments(optionArgs)
	if err != nil {
		return err.Error()
	}
	options, err := options.ExtractFrom(parsedArgs)
	if err != nil {
		return err.Error()
	}
	if options.Help {
		return cmd.ShowHelp()
	}
	err = model_loader.LoadModel(parsedArgs, cmd.InputModel())
	if err != nil {
		return err.Error()
	}
	err = model_validator.ValidateModel(cmd.InputModel())
	if err != nil {
		return err.Error()
	}
	err = model_adjuster.ApplyDefaultBehaviour(cmd.InputModel())
	if err != nil {
		return err.Error()
	}
	config, err := config.LoadConfig()
	if err != nil {
		return err.Error()
	}
	cn, err := auth.AuthenticateCommand(options, config)
	if err != nil {
		return err.Error()
	}
	err = cmd.Execute(cn)
	if err != nil {
		if err == io.EOF {
			return ""
		}
		return err.Error()
	}
	f, err := formatter_provider.GetOutputFormatter(options)
	if err != nil {
		return err.Error()
	}
	output, err := f.FormatOutput(cmd.OutputModel())
	if err != nil {
		return err.Error()
	}
	return output
}

func ussage() string {
	res := "Ussage: clc <resource> <command> [options and parameters], for example 'clc server create --name my-server ...'\n"
	res += "To get help and list all avaliable resources or commands, you can use 'clc --help' or 'clc <resource> --help' or 'clc <resource> <command> --help'\n"
	return res
}
<commit_msg>Save the "last command result" on each command execution.<commit_after>package main

import (
	"github.com/centurylinkcloud/clc-go-cli/auth"
	"github.com/centurylinkcloud/clc-go-cli/command_loader"
	"github.com/centurylinkcloud/clc-go-cli/config"
	"github.com/centurylinkcloud/clc-go-cli/formatter_provider"
	"github.com/centurylinkcloud/clc-go-cli/model_adjuster"
	"github.com/centurylinkcloud/clc-go-cli/model_loader"
	"github.com/centurylinkcloud/clc-go-cli/model_validator"
	"github.com/centurylinkcloud/clc-go-cli/options"
	"github.com/centurylinkcloud/clc-go-cli/parser"
	"github.com/centurylinkcloud/clc-go-cli/state"

	"io"
)

func run(args []string) string {
	if len(args) == 0 {
		return ussage()
	}
	cmdArg := ""
	optionArgs := args[1:]
	if len(args) >= 2 {
		cmdArg = args[1]
		optionArgs = args[2:]
	}
	cmd, err := command_loader.LoadCommand(args[0], cmdArg)
	if err != nil {
		return err.Error()
	}
	parsedArgs, err := parser.ParseArguments(optionArgs)
	if err != nil {
		return err.Error()
	}
	options, err := options.ExtractFrom(parsedArgs)
	if err != nil {
		return err.Error()
	}
	if options.Help {
		return cmd.ShowHelp()
	}
	err = model_loader.LoadModel(parsedArgs, cmd.InputModel())
	if err != nil {
		return err.Error()
	}
	err = model_validator.ValidateModel(cmd.InputModel())
	if err != nil {
		return err.Error()
	}
	err = model_adjuster.ApplyDefaultBehaviour(cmd.InputModel())
	if err != nil {
		return err.Error()
	}
	config, err := config.LoadConfig()
	if err != nil {
		return err.Error()
	}
	cn, err := auth.AuthenticateCommand(options, config)
	if err != nil {
		return err.Error()
	}
	err = cmd.Execute(cn)
	if err != nil {
		if err == io.EOF {
			return ""
		}
		return err.Error()
	}
	err = state.SaveLastResult(cmd.OutputModel())
	if err != nil {
		return err.Error()
	}
	f, err := formatter_provider.GetOutputFormatter(options)
	if err != nil {
		return err.Error()
	}
	output, err := f.FormatOutput(cmd.OutputModel())
	if err != nil {
		return err.Error()
	}
	return output
}

func ussage() string {
	res := "Ussage: clc <resource> <command> [options and parameters], for example 'clc server create --name my-server ...'\n"
	res += "To get help and list all avaliable resources or commands, you can use 'clc --help' or 'clc <resource> --help' or 'clc <resource> <command> --help'\n"
	return res
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file implements the visitor that computes the (line, column)-(line-column) range for each function.

package main

import (
	"bufio"
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"os"
	"path/filepath"
	"text/tabwriter"

	"code.google.com/p/go.tools/cover"
)

// funcOutput takes two file names as arguments, a coverage profile to read as input and an output
// file to write ("" means to write to standard output). The function reads the profile and produces
// as output the coverage data broken down by function, like this:
//
//	fmt/format.go:	init			100.0%
//	fmt/format.go:	computePadding		84.6%
//	...
//	fmt/scan.go:	doScan			100.0%
//	fmt/scan.go:	advance			96.2%
//	fmt/scan.go:	doScanf			96.8%
//	total:		(statements)		91.4%

func funcOutput(profile, outputFile string) error {
	profiles, err := cover.ParseProfiles(profile)
	if err != nil {
		return err
	}

	var out *bufio.Writer
	if outputFile == "" {
		out = bufio.NewWriter(os.Stdout)
	} else {
		fd, err := os.Create(outputFile)
		if err != nil {
			return err
		}
		defer fd.Close()
		out = bufio.NewWriter(fd)
	}
	defer out.Flush()

	tabber := tabwriter.NewWriter(out, 1, 8, 1, '\t', 0)
	defer tabber.Flush()

	var total, covered int64
	for _, profile := range profiles {
		fn := profile.FileName
		file, err := findFile(fn)
		if err != nil {
			return err
		}
		funcs, err := findFuncs(file)
		if err != nil {
			return err
		}
		// Now match up functions and profile blocks.
		for _, f := range funcs {
			c, t := f.coverage(profile)
			fmt.Fprintf(tabber, "%s:\t%s\t%.1f%%\n", fn, f.name, 100.0*float64(c)/float64(t))
			total += t
			covered += c
		}
	}
	fmt.Fprintf(tabber, "total:\t(statements)\t%.1f%%\n", 100.0*float64(covered)/float64(total))

	return nil
}

// findFuncs parses the file and returns a slice of FuncExtent descriptors.
func findFuncs(name string) ([]*FuncExtent, error) {
	fset := token.NewFileSet()
	parsedFile, err := parser.ParseFile(fset, name, nil, 0)
	if err != nil {
		return nil, err
	}
	visitor := &FuncVisitor{
		fset:    fset,
		name:    name,
		astFile: parsedFile,
	}
	ast.Walk(visitor, visitor.astFile)
	return visitor.funcs, nil
}

// FuncExtent describes a function's extent in the source by file and position.
type FuncExtent struct {
	name      string
	startLine int
	startCol  int
	endLine   int
	endCol    int
}

// FuncVisitor implements the visitor that builds the function position list for a file.
type FuncVisitor struct {
	fset    *token.FileSet
	name    string // Name of file.
	astFile *ast.File
	funcs   []*FuncExtent
}

// Visit implements the ast.Visitor interface.
func (v *FuncVisitor) Visit(node ast.Node) ast.Visitor {
	switch n := node.(type) {
	case *ast.FuncDecl:
		start := v.fset.Position(n.Pos())
		end := v.fset.Position(n.End())
		fe := &FuncExtent{
			name:      n.Name.Name,
			startLine: start.Line,
			startCol:  start.Column,
			endLine:   end.Line,
			endCol:    end.Column,
		}
		v.funcs = append(v.funcs, fe)
	}
	return v
}

// coverage returns the fraction of the statements in the function that were covered, as a numerator and denominator.
func (f *FuncExtent) coverage(profile *cover.Profile) (num, den int64) {
	// We could avoid making this n^2 overall by doing a single scan and annotating the functions,
	// but the sizes of the data structures is never very large and the scan is almost instantaneous.
	var covered, total int64
	// The blocks are sorted, so we can stop counting as soon as we reach the end of the relevant block.
	for _, b := range profile.Blocks {
		if b.StartLine > f.endLine || (b.StartLine == f.endLine && b.StartCol >= f.endCol) {
			// Past the end of the function.
			break
		}
		if b.EndLine < f.startLine || (b.EndLine == f.startLine && b.EndCol <= f.startCol) {
			// Before the beginning of the function
			continue
		}
		total += int64(b.NumStmt)
		if b.Count > 0 {
			covered += int64(b.NumStmt)
		}
	}
	if total == 0 {
		total = 1 // Avoid zero denominator.
	}
	return covered, total
}

// findFile finds the location of the named file in GOROOT, GOPATH etc.
func findFile(file string) (string, error) {
	dir, file := filepath.Split(file)
	pkg, err := build.Import(dir, ".", build.FindOnly)
	if err != nil {
		return "", fmt.Errorf("can't find %q: %v", file, err)
	}
	return filepath.Join(pkg.Dir, file), nil
}
<commit_msg>cmd/cover: add start lines numbers to each function in -func mode<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file implements the visitor that computes the (line, column)-(line-column) range for each function.

package main

import (
	"bufio"
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"os"
	"path/filepath"
	"text/tabwriter"

	"code.google.com/p/go.tools/cover"
)

// funcOutput takes two file names as arguments, a coverage profile to read as input and an output
// file to write ("" means to write to standard output). The function reads the profile and produces
// as output the coverage data broken down by function, like this:
//
//	fmt/format.go:30:	init			100.0%
//	fmt/format.go:57:	clearflags		100.0%
//	...
//	fmt/scan.go:1046:	doScan			100.0%
//	fmt/scan.go:1075:	advance			96.2%
//	fmt/scan.go:1119:	doScanf			96.8%
//	total:		(statements)			91.9%

func funcOutput(profile, outputFile string) error {
	profiles, err := cover.ParseProfiles(profile)
	if err != nil {
		return err
	}

	var out *bufio.Writer
	if outputFile == "" {
		out = bufio.NewWriter(os.Stdout)
	} else {
		fd, err := os.Create(outputFile)
		if err != nil {
			return err
		}
		defer fd.Close()
		out = bufio.NewWriter(fd)
	}
	defer out.Flush()

	tabber := tabwriter.NewWriter(out, 1, 8, 1, '\t', 0)
	defer tabber.Flush()

	var total, covered int64
	for _, profile := range profiles {
		fn := profile.FileName
		file, err := findFile(fn)
		if err != nil {
			return err
		}
		funcs, err := findFuncs(file)
		if err != nil {
			return err
		}
		// Now match up functions and profile blocks.
		for _, f := range funcs {
			c, t := f.coverage(profile)
			fmt.Fprintf(tabber, "%s:%d:\t%s\t%.1f%%\n", fn, f.startLine, f.name, 100.0*float64(c)/float64(t))
			total += t
			covered += c
		}
	}
	fmt.Fprintf(tabber, "total:\t(statements)\t%.1f%%\n", 100.0*float64(covered)/float64(total))

	return nil
}

// findFuncs parses the file and returns a slice of FuncExtent descriptors.
func findFuncs(name string) ([]*FuncExtent, error) {
	fset := token.NewFileSet()
	parsedFile, err := parser.ParseFile(fset, name, nil, 0)
	if err != nil {
		return nil, err
	}
	visitor := &FuncVisitor{
		fset:    fset,
		name:    name,
		astFile: parsedFile,
	}
	ast.Walk(visitor, visitor.astFile)
	return visitor.funcs, nil
}

// FuncExtent describes a function's extent in the source by file and position.
type FuncExtent struct {
	name      string
	startLine int
	startCol  int
	endLine   int
	endCol    int
}

// FuncVisitor implements the visitor that builds the function position list for a file.
type FuncVisitor struct {
	fset    *token.FileSet
	name    string // Name of file.
	astFile *ast.File
	funcs   []*FuncExtent
}

// Visit implements the ast.Visitor interface.
func (v *FuncVisitor) Visit(node ast.Node) ast.Visitor {
	switch n := node.(type) {
	case *ast.FuncDecl:
		start := v.fset.Position(n.Pos())
		end := v.fset.Position(n.End())
		fe := &FuncExtent{
			name:      n.Name.Name,
			startLine: start.Line,
			startCol:  start.Column,
			endLine:   end.Line,
			endCol:    end.Column,
		}
		v.funcs = append(v.funcs, fe)
	}
	return v
}

// coverage returns the fraction of the statements in the function that were covered, as a numerator and denominator.
func (f *FuncExtent) coverage(profile *cover.Profile) (num, den int64) {
	// We could avoid making this n^2 overall by doing a single scan and annotating the functions,
	// but the sizes of the data structures is never very large and the scan is almost instantaneous.
	var covered, total int64
	// The blocks are sorted, so we can stop counting as soon as we reach the end of the relevant block.
	for _, b := range profile.Blocks {
		if b.StartLine > f.endLine || (b.StartLine == f.endLine && b.StartCol >= f.endCol) {
			// Past the end of the function.
			break
		}
		if b.EndLine < f.startLine || (b.EndLine == f.startLine && b.EndCol <= f.startCol) {
			// Before the beginning of the function
			continue
		}
		total += int64(b.NumStmt)
		if b.Count > 0 {
			covered += int64(b.NumStmt)
		}
	}
	if total == 0 {
		total = 1 // Avoid zero denominator.
	}
	return covered, total
}

// findFile finds the location of the named file in GOROOT, GOPATH etc.
func findFile(file string) (string, error) {
	dir, file := filepath.Split(file)
	pkg, err := build.Import(dir, ".", build.FindOnly)
	if err != nil {
		return "", fmt.Errorf("can't find %q: %v", file, err)
	}
	return filepath.Join(pkg.Dir, file), nil
}
<|endoftext|>


<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"github.com/globocom/tsuru/cmd"
	"os"
)

const (
	version = "0.4"
	header  = "Supported-Crane"
)

func buildManager(name string) *cmd.Manager {
	m := cmd.BuildBaseManager(name, version, header)
	m.Register(&ServiceCreate{})
	m.Register(&ServiceRemove{})
	m.Register(&ServiceList{})
	m.Register(&ServiceUpdate{})
	m.Register(&ServiceDocGet{})
	m.Register(&ServiceDocAdd{})
	m.Register(&ServiceTemplate{})
	return m
}

func main() {
	name := cmd.ExtractProgramName(os.Args[0])
	manager := buildManager(name)
	args := os.Args[1:]
	manager.Run(args)
}
<commit_msg>cmd/crane: version 0.4.1<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"github.com/globocom/tsuru/cmd"
	"os"
)

const (
	version = "0.4.1"
	header  = "Supported-Crane"
)

func buildManager(name string) *cmd.Manager {
	m := cmd.BuildBaseManager(name, version, header)
	m.Register(&ServiceCreate{})
	m.Register(&ServiceRemove{})
	m.Register(&ServiceList{})
	m.Register(&ServiceUpdate{})
	m.Register(&ServiceDocGet{})
	m.Register(&ServiceDocAdd{})
	m.Register(&ServiceTemplate{})
	return m
}

func main() {
	name := cmd.ExtractProgramName(os.Args[0])
	manager := buildManager(name)
	args := os.Args[1:]
	manager.Run(args)
}
<|endoftext|>


<commit_before>/*
 * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"bytes"
	"context"
	"encoding/json"
	"os"
	"path/filepath"
	"sync"
	"time"

	jsoniter "github.com/json-iterator/go"
	"github.com/minio/minio/cmd/logger"
	"github.com/minio/minio/pkg/hash"
)

const (
	dataUsageObjName       = "data-usage"
	dataUsageCrawlInterval = 12 * time.Hour
)

func initDataUsageStats() {
	go runDataUsageInfoUpdateRoutine()
}

func runDataUsageInfoUpdateRoutine() {
	// Wait until the object layer is ready
	var objAPI ObjectLayer
	for {
		objAPI = newObjectLayerWithoutSafeModeFn()
		if objAPI == nil {
			time.Sleep(time.Second)
			continue
		}
		break
	}

	runDataUsageInfo(context.Background(), objAPI, GlobalServiceDoneCh)
}

// timeToNextCrawl returns the duration until next crawl should occur
// this is validated by verifying the LastUpdate time.
func timeToCrawl(ctx context.Context, objAPI ObjectLayer) time.Duration {
	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objAPI)
	if err != nil {
		// Upon an error wait for like 10
		// seconds to start the crawler.
		return 10 * time.Second
	}
	// File indeed doesn't exist when LastUpdate is zero
	// so we have never crawled, start crawl right away.
	if dataUsageInfo.LastUpdate.IsZero() {
		return 1 * time.Second
	}
	waitDuration := dataUsageInfo.LastUpdate.Sub(UTCNow())
	if waitDuration > dataUsageCrawlInterval {
		// Waited long enough start crawl in a 1 second
		return 1 * time.Second
	}
	// No crawling needed, ask the routine to wait until
	// the daily interval 12hrs - delta between last update
	// with current time.
	return dataUsageCrawlInterval - waitDuration
}

func runDataUsageInfo(ctx context.Context, objAPI ObjectLayer, endCh <-chan struct{}) {
	locker := objAPI.NewNSLock(ctx, minioMetaBucket, "leader-data-usage-info")
	for {
		err := locker.GetLock(newDynamicTimeout(time.Millisecond, time.Millisecond))
		if err != nil {
			time.Sleep(5 * time.Minute)
			continue
		}
		// Break without unlocking, this node will acquire
		// data usage calculator role for its lifetime.
		break
	}

	for {
		wait := timeToCrawl(ctx, objAPI)
		select {
		case <-endCh:
			locker.Unlock()
			return
		case <-time.NewTimer(wait).C:
			// Crawl only when no previous crawl has occurred,
			// or its been too long since last crawl.
			err := storeDataUsageInBackend(ctx, objAPI, objAPI.CrawlAndGetDataUsage(ctx, endCh))
			logger.LogIf(ctx, err)
		}
	}
}

func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dataUsageInfo DataUsageInfo) error {
	dataUsageJSON, err := json.Marshal(dataUsageInfo)
	if err != nil {
		return err
	}

	size := int64(len(dataUsageJSON))
	r, err := hash.NewReader(bytes.NewReader(dataUsageJSON), size, "", "", size, false)
	if err != nil {
		return err
	}

	_, err = objAPI.PutObject(ctx, minioMetaBackgroundOpsBucket, dataUsageObjName, NewPutObjReader(r, nil, nil), ObjectOptions{})
	return err
}

func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsageInfo, error) {
	var dataUsageInfoJSON bytes.Buffer

	err := objAPI.GetObject(ctx, minioMetaBackgroundOpsBucket, dataUsageObjName, 0, -1, &dataUsageInfoJSON, "", ObjectOptions{})
	if err != nil {
		if isErrObjectNotFound(err) {
			return DataUsageInfo{}, nil
		}
		return DataUsageInfo{}, toObjectErr(err, minioMetaBackgroundOpsBucket, dataUsageObjName)
	}

	var dataUsageInfo DataUsageInfo
	var json = jsoniter.ConfigCompatibleWithStandardLibrary
	err = json.Unmarshal(dataUsageInfoJSON.Bytes(), &dataUsageInfo)
	if err != nil {
		return DataUsageInfo{}, err
	}

	return dataUsageInfo, nil
}

// Item represents each file while walking.
type Item struct {
	Path string
	Typ  os.FileMode
}

type getSizeFn func(item Item) (int64, error)

func updateUsage(basePath string, doneCh <-chan struct{}, waitForLowActiveIO func(), getSize getSizeFn) DataUsageInfo {
	var dataUsageInfo = DataUsageInfo{
		BucketsSizes:          make(map[string]uint64),
		ObjectsSizesHistogram: make(map[string]uint64),
	}

	numWorkers := 4

	var mutex sync.Mutex // Mutex to update dataUsageInfo

	fastWalk(basePath, numWorkers, doneCh, func(path string, typ os.FileMode) error {
		// Wait for I/O to go down.
		waitForLowActiveIO()

		bucket, entry := path2BucketObjectWithBasePath(basePath, path)
		if bucket == "" {
			return nil
		}

		if isReservedOrInvalidBucket(bucket, false) {
			return filepath.SkipDir
		}

		if entry == "" && typ&os.ModeDir != 0 {
			mutex.Lock()
			dataUsageInfo.BucketsCount++
			dataUsageInfo.BucketsSizes[bucket] = 0
			mutex.Unlock()
			return nil
		}

		if typ&os.ModeDir != 0 {
			return nil
		}

		size, err := getSize(Item{path, typ})
		if err != nil {
			return errSkipFile
		}

		mutex.Lock()
		dataUsageInfo.ObjectsCount++
		dataUsageInfo.ObjectsTotalSize += uint64(size)
		dataUsageInfo.BucketsSizes[bucket] += uint64(size)
		dataUsageInfo.ObjectsSizesHistogram[objSizeToHistoInterval(uint64(size))]++
		mutex.Unlock()
		return nil
	})

	return dataUsageInfo
}
<commit_msg>Add metadata parsing to be inside mutex to slow down (#8952)<commit_after>/*
 * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"bytes"
	"context"
	"encoding/json"
	"math/rand"
	"os"
	"path/filepath"
	"sync"
	"time"

	jsoniter "github.com/json-iterator/go"
	"github.com/minio/minio/cmd/logger"
	"github.com/minio/minio/pkg/hash"
)

const (
	dataUsageObjName       = "data-usage"
	dataUsageCrawlInterval = 12 * time.Hour
)

func initDataUsageStats() {
	go runDataUsageInfoUpdateRoutine()
}

func runDataUsageInfoUpdateRoutine() {
	// Wait until the object layer is ready
	var objAPI ObjectLayer
	for {
		objAPI = newObjectLayerWithoutSafeModeFn()
		if objAPI == nil {
			time.Sleep(time.Second)
			continue
		}
		break
	}

	runDataUsageInfo(context.Background(), objAPI, GlobalServiceDoneCh)
}

// timeToNextCrawl returns the duration until next crawl should occur
// this is validated by verifying the LastUpdate time.
func timeToCrawl(ctx context.Context, objAPI ObjectLayer) time.Duration {
	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objAPI)
	if err != nil {
		// Upon an error wait for like 10
		// seconds to start the crawler.
		return 10 * time.Second
	}
	// File indeed doesn't exist when LastUpdate is zero
	// so we have never crawled, start crawl right away.
	if dataUsageInfo.LastUpdate.IsZero() {
		return 1 * time.Second
	}
	waitDuration := dataUsageInfo.LastUpdate.Sub(UTCNow())
	if waitDuration > dataUsageCrawlInterval {
		// Waited long enough start crawl in a 1 second
		return 1 * time.Second
	}
	// No crawling needed, ask the routine to wait until
	// the daily interval 12hrs - delta between last update
	// with current time.
	return dataUsageCrawlInterval - waitDuration
}

func runDataUsageInfo(ctx context.Context, objAPI ObjectLayer, endCh <-chan struct{}) {
	locker := objAPI.NewNSLock(ctx, minioMetaBucket, "leader-data-usage-info")
	for {
		err := locker.GetLock(newDynamicTimeout(time.Millisecond, time.Millisecond))
		if err != nil {
			time.Sleep(5 * time.Minute)
			continue
		}
		// Break without unlocking, this node will acquire
		// data usage calculator role for its lifetime.
		break
	}

	for {
		wait := timeToCrawl(ctx, objAPI)
		select {
		case <-endCh:
			locker.Unlock()
			return
		case <-time.NewTimer(wait).C:
			// Crawl only when no previous crawl has occurred,
			// or its been too long since last crawl.
			err := storeDataUsageInBackend(ctx, objAPI, objAPI.CrawlAndGetDataUsage(ctx, endCh))
			logger.LogIf(ctx, err)
		}
	}
}

func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dataUsageInfo DataUsageInfo) error {
	dataUsageJSON, err := json.Marshal(dataUsageInfo)
	if err != nil {
		return err
	}

	size := int64(len(dataUsageJSON))
	r, err := hash.NewReader(bytes.NewReader(dataUsageJSON), size, "", "", size, false)
	if err != nil {
		return err
	}

	_, err = objAPI.PutObject(ctx, minioMetaBackgroundOpsBucket, dataUsageObjName, NewPutObjReader(r, nil, nil), ObjectOptions{})
	return err
}

func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsageInfo, error) {
	var dataUsageInfoJSON bytes.Buffer

	err := objAPI.GetObject(ctx, minioMetaBackgroundOpsBucket, dataUsageObjName, 0, -1, &dataUsageInfoJSON, "", ObjectOptions{})
	if err != nil {
		if isErrObjectNotFound(err) {
			return DataUsageInfo{}, nil
		}
		return DataUsageInfo{}, toObjectErr(err, minioMetaBackgroundOpsBucket, dataUsageObjName)
	}

	var dataUsageInfo DataUsageInfo
	var json = jsoniter.ConfigCompatibleWithStandardLibrary
	err = json.Unmarshal(dataUsageInfoJSON.Bytes(), &dataUsageInfo)
	if err != nil {
		return DataUsageInfo{}, err
	}

	return dataUsageInfo, nil
}

// Item represents each file while walking.
type Item struct {
	Path string
	Typ  os.FileMode
}

type getSizeFn func(item Item) (int64, error)

func updateUsage(basePath string, doneCh <-chan struct{}, waitForLowActiveIO func(), getSize getSizeFn) DataUsageInfo {
	var dataUsageInfo = DataUsageInfo{
		BucketsSizes:          make(map[string]uint64),
		ObjectsSizesHistogram: make(map[string]uint64),
	}

	numWorkers := 4
	walkInterval := 1 * time.Millisecond

	var mutex sync.Mutex // Mutex to update dataUsageInfo

	r := rand.New(rand.NewSource(UTCNow().UnixNano()))

	fastWalk(basePath, numWorkers, doneCh, func(path string, typ os.FileMode) error {
		// Wait for I/O to go down.
		waitForLowActiveIO()

		// Randomize sleep intervals, to stagger the walk.
		defer time.Sleep(time.Duration(r.Float64() * float64(walkInterval)))

		bucket, entry := path2BucketObjectWithBasePath(basePath, path)
		if bucket == "" {
			return nil
		}

		if isReservedOrInvalidBucket(bucket, false) {
			return filepath.SkipDir
		}

		if entry == "" && typ&os.ModeDir != 0 {
			mutex.Lock()
			dataUsageInfo.BucketsCount++
			dataUsageInfo.BucketsSizes[bucket] = 0
			mutex.Unlock()
			return nil
		}

		mutex.Lock()
		defer mutex.Unlock()

		if typ&os.ModeDir != 0 {
			return nil
		}

		size, err := getSize(Item{path, typ})
		if err != nil {
			return errSkipFile
		}

		dataUsageInfo.ObjectsCount++
		dataUsageInfo.ObjectsTotalSize += uint64(size)
		dataUsageInfo.BucketsSizes[bucket] += uint64(size)
		dataUsageInfo.ObjectsSizesHistogram[objSizeToHistoInterval(uint64(size))]++
		return nil
	})

	return dataUsageInfo
}
<|endoftext|>


<commit_before>package cmd

import (
	client "bigv.io/client/lib"
	"flag"
	"fmt"
	"os"
	"strings"
)

// type der is used to create API requests and direct output to views,
// except probably when those API requests don't require authorisation (e.g. /definitions, new user)
type Dispatcher struct {
	//Config *Config
	Flags      *flag.FlagSet
	cmds       Commands
	debugLevel int
}

// NewDispatcher creates a new Dispatcher given a config.
func NewDispatcher(config ConfigManager) (d *Dispatcher) {
	d = new(Dispatcher)
	bigv, err := client.New(config.Get("endpoint"))
	if err != nil {
		exit(err)
	}

	d.debugLevel = config.GetDebugLevel()

	d.cmds = NewCommandSet(config, bigv)
	return d
}

// NewderWithCommands is for writing tests with mock CommandSets
func NewDispatcherWithCommands(config ConfigManager, commands Commands) *Dispatcher {
	d := NewDispatcher(config)
	d.cmds = commands
	return d
}

// EnsureAuth makes sure a valid token is stored in config.
// This should be called by anything that needs auth.

// TODO(telyn): Write a test for Do. Somehow.

// Do takes the command line arguments and figures out what to do
func (d *Dispatcher) Do(args []string) {
	//	help := d.Flags.Lookup("help")
	///	fmt.Printf("%+v", help)
	if d.debugLevel >= 1 {
		fmt.Fprintf(os.Stderr, "Args passed to Do: %#v\n", args)
	}

	if /*help == true || */ len(args) == 0 || strings.HasPrefix(args[0], "-") {
		fmt.Printf("No command specified.\n\n")
		d.cmds.Help(args)
		return
	}

	// short-circuit commands that don't take arguments
	switch strings.ToLower(args[0]) {
	case "config":
		d.cmds.Config(args[1:])
		return
	case "help":
		d.cmds.Help(args[1:])
		return
	}

	// do this
	if len(args) == 1 {
		d.cmds.Help(args)
		return
	}

	switch strings.ToLower(args[0]) {
	case "debug":
		d.cmds.Debug(args[1:])
		return
	case "delete-vm":
		d.cmds.DeleteVM(args[1:])
		return
	case "show-account":
		d.cmds.ShowAccount(args[1:])
		return
	case "show-vm":
		d.cmds.ShowVM(args[1:])
		return
	}
	fmt.Fprintf(os.Stderr, "Unrecognised command '%s'\r\n", args[0])
	d.cmds.Help(args)
}
<commit_msg>Actually set bigv debug level<commit_after>package cmd

import (
	client "bigv.io/client/lib"
	"flag"
	"fmt"
	"os"
	"strings"
)

// type der is used to create API requests and direct output to views,
// except probably when those API requests don't require authorisation (e.g. /definitions, new user)
type Dispatcher struct {
	//Config *Config
	Flags      *flag.FlagSet
	cmds       Commands
	debugLevel int
}

// NewDispatcher creates a new Dispatcher given a config.
func NewDispatcher(config ConfigManager) (d *Dispatcher) {
	d = new(Dispatcher)
	bigv, err := client.New(config.Get("endpoint"))
	if err != nil {
		exit(err)
	}

	d.debugLevel = config.GetDebugLevel()
	bigv.SetDebugLevel(d.debugLevel)

	d.cmds = NewCommandSet(config, bigv)
	return d
}

// NewderWithCommands is for writing tests with mock CommandSets
func NewDispatcherWithCommands(config ConfigManager, commands Commands) *Dispatcher {
	d := NewDispatcher(config)
	d.cmds = commands
	return d
}

// EnsureAuth makes sure a valid token is stored in config.
// This should be called by anything that needs auth.

// TODO(telyn): Write a test for Do. Somehow.

// Do takes the command line arguments and figures out what to do
func (d *Dispatcher) Do(args []string) {
	//	help := d.Flags.Lookup("help")
	///	fmt.Printf("%+v", help)
	if d.debugLevel >= 1 {
		fmt.Fprintf(os.Stderr, "Args passed to Do: %#v\n", args)
	}

	if /*help == true || */ len(args) == 0 || strings.HasPrefix(args[0], "-") {
		fmt.Printf("No command specified.\n\n")
		d.cmds.Help(args)
		return
	}

	// short-circuit commands that don't take arguments
	switch strings.ToLower(args[0]) {
	case "config":
		d.cmds.Config(args[1:])
		return
	case "help":
		d.cmds.Help(args[1:])
		return
	}

	// do this
	if len(args) == 1 {
		d.cmds.Help(args)
		return
	}

	switch strings.ToLower(args[0]) {
	case "debug":
		d.cmds.Debug(args[1:])
		return
	case "delete-vm":
		d.cmds.DeleteVM(args[1:])
		return
	case "show-account":
		d.cmds.ShowAccount(args[1:])
		return
	case "show-vm":
		d.cmds.ShowVM(args[1:])
		return
	}
	fmt.Fprintf(os.Stderr, "Unrecognised command '%s'\r\n", args[0])
	d.cmds.Help(args)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"os"

	"github.com/Nhanderu/gridt"
	"github.com/alecthomas/kingpin"
	"golang.org/x/crypto/ssh/terminal"
)

const (
	topToBottom = "top-to-bottom"
	leftToRight = "left-to-right"
)

var (
	args      *[]string
	file      *string
	separator *string
	direction *string
)

func init() {

	kingpin.CommandLine.HelpFlag.Short('h')
	kingpin.Version("2.0.0").VersionFlag.Short('v')

	file = kingpin.Flag("file", "Get values as lines from file.").Short('f').String()
	separator = kingpin.
		Flag("separator", "What separates every value column.").
		Short('s').
		Default("  ").
		String()
	direction = kingpin.
		Flag("direction", `Whether it writes from "top-to-bottom" or "left-to-right".`).
		Short('d').
		Default(topToBottom).
		Enum(topToBottom, leftToRight)

	kingpin.Parse()
}

func main() {

	width, _, err := terminal.GetSize(0)
	if err != nil {
		eprintln(err.Error())
		os.Exit(1)
	}

	var scanner *bufio.Scanner
	if *file != "" {
		f, err := os.Open(*file)
		if err != nil {
			eprintf("Error %s.", err.Error())
			os.Exit(1)
		}
		defer f.Close()
		scanner = bufio.NewScanner(f)
	} else {
		scanner = bufio.NewScanner(os.Stdin)
	}

	var values []string
	if scanner != nil {
		scanner.Split(bufio.ScanLines)
		for scanner.Scan() {
			values = append(values, scanner.Text())
		}
	}

	if len(values) == 0 {
		eprintln("No values were given.")
		os.Exit(1)
	}

	d := gridt.TopToBottom
	if *direction == leftToRight {
		d = gridt.LeftToRight
	}

	grid, ok := gridt.New(d, *separator, values...).FitIntoWidth(width)
	if !ok {
		eprintln("The given values does not fit in the terminal width.")
		os.Exit(1)
	}

	fmt.Println(grid.String())
}

func eprintln(a ...interface{}) (n int, err error) {
	return fmt.Fprintln(os.Stderr, a...)
}

func eprintf(format string, a ...interface{}) (n int, err error) {
	return fmt.Fprintf(os.Stderr, format, a...)
}
<commit_msg>Use stdout fd for terminal size<commit_after>package main

import (
	"bufio"
	"fmt"
	"os"

	"github.com/Nhanderu/gridt"
	"github.com/alecthomas/kingpin"
	"golang.org/x/crypto/ssh/terminal"
)

const (
	topToBottom = "top-to-bottom"
	leftToRight = "left-to-right"
)

var (
	args      *[]string
	file      *string
	separator *string
	direction *string
)

func init() {

	kingpin.CommandLine.HelpFlag.Short('h')
	kingpin.Version("2.0.0").VersionFlag.Short('v')

	file = kingpin.Flag("file", "Get values as lines from file.").Short('f').String()
	separator = kingpin.
		Flag("separator", "What separates every value column.").
		Short('s').
		Default("  ").
		String()
	direction = kingpin.
		Flag("direction", `Whether it writes from "top-to-bottom" or "left-to-right".`).
		Short('d').
		Default(topToBottom).
		Enum(topToBottom, leftToRight)

	kingpin.Parse()
}

func main() {

	width, _, err := terminal.GetSize(1)
	if err != nil {
		eprintf("Error getting terminal size: %s.", err.Error())
		os.Exit(1)
	}

	var scanner *bufio.Scanner
	if *file != "" {
		f, err := os.Open(*file)
		if err != nil {
			eprintf("Error opening file: %s.", err.Error())
			os.Exit(1)
		}
		defer f.Close()
		scanner = bufio.NewScanner(f)
	} else {
		scanner = bufio.NewScanner(os.Stdin)
	}

	var values []string
	if scanner != nil {
		scanner.Split(bufio.ScanLines)
		for scanner.Scan() {
			values = append(values, scanner.Text())
		}
	}

	if len(values) == 0 {
		eprintln("Error: no values were given.")
		os.Exit(1)
	}

	d := gridt.TopToBottom
	if *direction == leftToRight {
		d = gridt.LeftToRight
	}

	grid, ok := gridt.New(d, *separator, values...).FitIntoWidth(width)
	if !ok {
		eprintln("Error: the given values does not fit in the terminal width.")
		os.Exit(1)
	}

	fmt.Println(grid.String())
}

func eprintln(a ...interface{}) (n int, err error) {
	return fmt.Fprintln(os.Stderr, a...)
}

func eprintf(format string, a ...interface{}) (n int, err error) {
	return fmt.Fprintf(os.Stderr, format, a...)
}
<|endoftext|>