<commit_before>package utils

import (
	"capsulecd/pkg/errors"
	"capsulecd/pkg/pipeline"
	stderrors "errors"
	"fmt"
	git2go "gopkg.in/libgit2/git2go.v25"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"
)

// Clone a git repo into a local directory.
// Credentials need to be specified by embedding in gitRemote url.
// TODO: this pattern may not work on Bitbucket/GitLab
func GitClone(parentPath string, repositoryName string, gitRemote string) (string, error) {
	absPath, _ := filepath.Abs(path.Join(parentPath, repositoryName))

	if !FileExists(absPath) {
		os.MkdirAll(absPath, os.ModePerm)
	} else {
		return "", errors.ScmFilesystemError(fmt.Sprintf("The local repository path already exists, this should never happen. %s", absPath))
	}

	_, err := git2go.Clone(gitRemote, absPath, new(git2go.CloneOptions))
	return absPath, err
}

func GitFetch(repoPath string, remoteRef string, localBranchName string) error {

	repo, oerr := git2go.OpenRepository(repoPath)
	if oerr != nil {
		return oerr
	}

	checkoutOpts := &git2go.CheckoutOpts{
		Strategy: git2go.CheckoutSafe | git2go.CheckoutRecreateMissing | git2go.CheckoutAllowConflicts | git2go.CheckoutUseTheirs,
	}

	remote, lerr := repo.Remotes.Lookup("origin")
	if lerr != nil {
		log.Print("Failed to lookup origin remote")
		return lerr
	}
	time.Sleep(time.Second)

	ferr := remote.Fetch([]string{fmt.Sprintf("%s:%s", remoteRef, localBranchName)}, new(git2go.FetchOptions), "")
	if ferr != nil {
		log.Print("Failed to fetch remote ref into new local branch " + fmt.Sprintf("%s:%s", remoteRef, localBranchName))
		return ferr
	}

	time.Sleep(time.Second)
	//should not raise an error when looking for branch (we just created it above)
	localBranch, berr := repo.LookupBranch(localBranchName, git2go.BranchLocal)
	if berr != nil {
		log.Print("Failed to lookup new local branch " + localBranchName)
		return berr
	}

	// Getting the tree for the branch
	localCommit, err := repo.LookupCommit(localBranch.Target())
	if err != nil {
		log.Print("Failed to lookup for commit in local branch " + localBranchName)
		return err
	}
	//defer localCommit.Free()

	tree, err := repo.LookupTree(localCommit.TreeId())
	if err != nil {
		log.Print("Failed to lookup for tree " + localBranchName)
		return err
	}
	//defer tree.Free()

	// Checkout the tree
	err = repo.CheckoutTree(tree, checkoutOpts)
	if err != nil {
		log.Print("Failed to checkout tree " + localBranchName)
		return err
	}
	// Setting the Head to point to our branch
	return repo.SetHead("refs/heads/" + localBranchName)
}

func GitCheckout(repoPath string, branchName string) error {
	repo, oerr := git2go.OpenRepository(repoPath)
	if oerr != nil {
		return oerr
	}

	checkoutOpts := &git2go.CheckoutOpts{
		Strategy: git2go.CheckoutSafe | git2go.CheckoutRecreateMissing | git2go.CheckoutAllowConflicts | git2go.CheckoutUseTheirs,
	}
	//Getting the reference for the remote branch
	// remoteBranch, err := repo.References.Lookup("refs/remotes/origin/" + branchName)
	remoteBranch, err := repo.LookupBranch("origin/"+branchName, git2go.BranchRemote)
	if err != nil {
		log.Print("Failed to find remote branch: " + branchName)
		return err
	}
	//defer remoteBranch.Free()

	// Lookup for commit from remote branch
	commit, err := repo.LookupCommit(remoteBranch.Target())
	if err != nil {
		log.Print("Failed to find remote branch commit: " + branchName)
		return err
	}
	//defer commit.Free()

	localBranch, err := repo.LookupBranch(branchName, git2go.BranchLocal)
	// No local branch, lets create one
	if localBranch == nil || err != nil {
		// Creating local branch
		localBranch, err = repo.CreateBranch(branchName, commit, false)
		if err != nil {
			log.Print("Failed to create local branch: " + branchName)
			return err
		}

		// Setting upstream to origin branch
		err = localBranch.SetUpstream("origin/" + branchName)
		if err != nil {
			log.Print("Failed to create upstream to origin/" + branchName)
			return err
		}
	}
	if localBranch == nil {
		return errors.ScmFilesystemError("Error while locating/creating local branch")
	}
	//defer localBranch.Free()

	// Getting the tree for the branch
	localCommit, err := repo.LookupCommit(localBranch.Target())
	if err != nil {
		log.Print("Failed to lookup for commit in local branch " + branchName)
		return err
	}
	//defer localCommit.Free()

	tree, err := repo.LookupTree(localCommit.TreeId())
	if err != nil {
		log.Print("Failed to lookup for tree " + branchName)
		return err
	}
	//defer tree.Free()

	// Checkout the tree
	err = repo.CheckoutTree(tree, checkoutOpts)
	if err != nil {
		log.Print("Failed to checkout tree " + branchName)
		return err
	}
	// Setting the Head to point to our branch
	return repo.SetHead("refs/heads/" + branchName)
}

//Add all modified files to index, and commit.
func GitCommit(repoPath string, message string) error {
	repo, oerr := git2go.OpenRepository(repoPath)
	if oerr != nil {
		return oerr
	}

	signature := gitSignature()

	//get repo index.
	idx, ierr := repo.Index()
	if ierr != nil {
		return ierr
	}
	aerr := idx.AddAll([]string{}, git2go.IndexAddDefault, nil)
	if aerr != nil {
		return aerr
	}
	treeId, wterr := idx.WriteTree()
	if wterr != nil {
		return wterr
	}
	werr := idx.Write()
	if werr != nil {
		return werr
	}

	tree, lerr := repo.LookupTree(treeId)
	if lerr != nil {
		return lerr
	}

	currentBranch, berr := repo.Head()
	if berr != nil {
		return berr
	}

	commitTarget, terr := repo.LookupCommit(currentBranch.Target())
	if terr != nil {
		return terr
	}

	_, cerr := repo.CreateCommit("HEAD", signature, signature, message, tree, commitTarget)
	//if(cerr != nil){return cerr}

	return cerr
}

func GitTag(repoPath string, version string) (string, error) {
	repo, oerr := git2go.OpenRepository(repoPath)
	if oerr != nil {
		return "", oerr
	}
	commitHead, herr := repo.Head()
	if herr != nil {
		return "", herr
	}

	commit, lerr := repo.LookupCommit(commitHead.Target())
	if lerr != nil {
		return "", lerr
	}

	//tagId, terr := repo.Tags.CreateLightweight(version, commit, false)
	tagId, terr := repo.Tags.Create(version, commit, gitSignature(), fmt.Sprintf("(%s) Automated packaging of release by CapsuleCD", version))
	if terr != nil {
		return "", terr
	}

	tagObj, terr := repo.LookupTag(tagId)
	return tagObj.TargetId().String(), terr
}

func GitPush(repoPath string, localBranch string, remoteBranch string, tagName string) error {
	//- https://gist.github.com/danielfbm/37b0ca88b745503557b2b3f16865d8c3
	//- https://stackoverflow.com/questions/37026399/git2go-after-createcommit-all-files-appear-like-being-added-for-deletion
	repo, oerr := git2go.OpenRepository(repoPath)
	if oerr != nil {
		return oerr
	}

	// Push
	remote, lerr := repo.Remotes.Lookup("origin")
	if lerr != nil {
		return lerr
	}
	//remote.ConnectPush(gitRemoteCallbacks(), &git.ProxyOptions{}, []string{})

	//err = remote.Push([]string{"refs/heads/master"}, nil, signature, message)
	return remote.Push([]string{
		fmt.Sprintf("refs/heads/%s:refs/heads/%s", localBranch, remoteBranch),
		fmt.Sprintf("refs/tags/%s:refs/tags/%s", tagName, tagName),
	}, new(git2go.PushOptions))
}

// Get the nearest tag on branch.
// tag must be nearest, ie. sorted by their distance from the HEAD of the branch, not the date or tagname.
// basically `git describe --tags --abbrev=0`
func GitFindNearestTagName(repoPath string) (string, error) {
	repo, oerr := git2go.OpenRepository(repoPath)
	if oerr != nil {
		return "", oerr
	}

	descOptions, derr := git2go.DefaultDescribeOptions()
	if derr != nil {
		return "", derr
	}
	descOptions.Strategy = git2go.DescribeTags

	formatOptions, ferr := git2go.DefaultDescribeFormatOptions()
	if ferr != nil {
		return "", ferr
	}
	formatOptions.AbbreviatedSize = 0

	descr, derr := repo.DescribeWorkdir(&descOptions)
	if derr != nil {
		return "", derr
	}

	nearestTag, ferr := descr.Format(&formatOptions)
	if ferr != nil {
		return "", ferr
	}

	return nearestTag, nil
}

func GitGenerateChangelog(repoPath string, baseSha string, headSha string) (string, error) {
	repo, oerr := git2go.OpenRepository(repoPath)
	if oerr != nil {
		return "", oerr
	}

	markdown := StripIndent(`Timestamp |  SHA | Message | Author
	------------- | ------------- | ------------- | -------------
	`)

	revWalk, werr := repo.Walk()
	if werr != nil {
		return "", werr
	}

	rerr := revWalk.PushRange(fmt.Sprintf("%s..%s", baseSha, headSha))
	if rerr != nil {
		return "", rerr
	}

	revWalk.Iterate(func(commit *git2go.Commit) bool {
		markdown += fmt.Sprintf("%s | %.8s | %s | %s\n", //TODO: this should have a link for the SHA.
			commit.Author().When.UTC().Format("2006-01-02T15:04Z"),
			commit.Id().String(),
			cleanCommitMessage(commit.Message()),
			commit.Author().Name,
		)
		return true
	})
	//for {
	//	err := revWalk.Next()
	//	if err != nil {
	//		break
	//	}
	//
	//	log.Info(gi.String())
	//}

	return markdown, nil
}

func GitGenerateGitIgnore(repoPath string, ignoreType string) error {
	//https://github.com/GlenDC/go-gitignore/blob/master/gitignore/provider/github.go

	gitIgnoreBytes, err := getGitIgnore(ignoreType)
	if err != nil {
		return err
	}

	gitIgnorePath := filepath.Join(repoPath, ".gitignore")
	return ioutil.WriteFile(gitIgnorePath, gitIgnoreBytes, 0644)
}

func GitGetTagDetails(repoPath string, tagName string) (*pipeline.GitTagDetails, error) {
	repo, oerr := git2go.OpenRepository(repoPath)
	if oerr != nil {
		return nil, oerr
	}

	id, aerr := repo.References.Dwim(tagName)
	if aerr != nil {
		return nil, aerr
	}
	tag, lerr := repo.LookupTag(id.Target()) //assume its an annotated tag.

	var currentTag *pipeline.GitTagDetails
	if lerr != nil {
		//this is a lightweight tag, not an annotated tag.
		commitRef, rerr := repo.LookupCommit(id.Target())
		if rerr != nil {
			return nil, rerr
		}

		author := commitRef.Author()

		log.Printf("Light-weight tag (%s) Commit ID: %s, DATE: %s", tagName, commitRef.Id().String(), author.When.String())

		currentTag = &pipeline.GitTagDetails{
			TagShortName: tagName,
			CommitSha:    commitRef.Id().String(),
			CommitDate:   author.When,
		}

	} else {

		log.Printf("Annotated tag (%s) Tag ID: %s, Commit ID: %s, DATE: %s", tagName, tag.Id().String(), tag.TargetId().String(), tag.Tagger().When.String())

		currentTag = &pipeline.GitTagDetails{
			TagShortName: tagName,
			CommitSha:    tag.TargetId().String(),
			CommitDate:   tag.Tagger().When,
		}
	}
	return currentTag, nil

}

//private methods

func gitSignature() *git2go.Signature {
	return &git2go.Signature{
		Name:  "CapsuleCD",
		Email: "CapsuleCD@users.noreply.github.com",
		When:  time.Now(),
	}
}

func cleanCommitMessage(commitMessage string) string {
	commitMessage = strings.TrimSpace(commitMessage)
	if commitMessage == "" {
		return "--"
	}

	commitMessage = strings.Replace(commitMessage, "|", "/", -1)
	commitMessage = strings.Replace(commitMessage, "\n", " ", -1)

	return commitMessage
}

func getGitIgnore(languageName string) ([]byte, error) {
	gitURL := fmt.Sprintf("https://raw.githubusercontent.com/github/gitignore/master/%s.gitignore", languageName)

	resp, err := http.Get(gitURL)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return nil, stderrors.New(fmt.Sprintf("Could not find .gitignore for '%s'", languageName))
	}

	return ioutil.ReadAll(resp.Body)
}

//func gitRemoteCallbacks() *git.RemoteCallbacks {
//	return  &git.RemoteCallbacks{
//		CredentialsCallback: credentialsCallback,
//		CertificateCheckCallback: certificateCheckCallback,
//	}
//}
//
//func credentialsCallback(url string, username_from_url string, allowed_types git.CredType) (git.ErrorCode, *git.Cred) {
//	log.Printf("This is the CRED URL FOR PUSH: %s %s",url, username_from_url)
//	ret, cred := git.NewCredUserpassPlaintext("placeholder", "") //TODO: remote cred.
//
//	log.Printf("THIS IS THE CRED RESPONS: %s %s", ret, cred)
//	return git.ErrorCode(ret), &cred
//}
//
//func certificateCheckCallback(cert *git.Certificate, valid bool, hostname string) git.ErrorCode {
//	if hostname != "github.com" {
//		return git.ErrUser
//	}
//	return git.ErrOk
//}
<commit_msg>add extra logging.<commit_after>package utils

import (
	"capsulecd/pkg/errors"
	"capsulecd/pkg/pipeline"
	stderrors "errors"
	"fmt"
	git2go "gopkg.in/libgit2/git2go.v25"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"
)

// Clone a git repo into a local directory.
// Credentials need to be specified by embedding in gitRemote url.
// TODO: this pattern may not work on Bitbucket/GitLab
func GitClone(parentPath string, repositoryName string, gitRemote string) (string, error) {
	absPath, _ := filepath.Abs(path.Join(parentPath, repositoryName))

	if !FileExists(absPath) {
		os.MkdirAll(absPath, os.ModePerm)
	} else {
		return "", errors.ScmFilesystemError(fmt.Sprintf("The local repository path already exists, this should never happen. %s", absPath))
	}

	_, err := git2go.Clone(gitRemote, absPath, new(git2go.CloneOptions))
	return absPath, err
}

func GitFetch(repoPath string, remoteRef string, localBranchName string) error {

	repo, oerr := git2go.OpenRepository(repoPath)
	if oerr != nil {
		return oerr
	}

	checkoutOpts := &git2go.CheckoutOpts{
		Strategy: git2go.CheckoutSafe | git2go.CheckoutRecreateMissing | git2go.CheckoutAllowConflicts | git2go.CheckoutUseTheirs,
	}

	remote, lerr := repo.Remotes.Lookup("origin")
	if lerr != nil {
		log.Print("Failed to lookup origin remote")
		return lerr
	}
	time.Sleep(time.Second)

	ferr := remote.Fetch([]string{fmt.Sprintf("%s:%s", remoteRef, localBranchName)}, new(git2go.FetchOptions), "")
	if ferr != nil {
		log.Print("Failed to fetch remote ref into new local branch " + fmt.Sprintf("%s:%s", remoteRef, localBranchName))
		return ferr
	}

	time.Sleep(time.Second)
	//should not raise an error when looking for branch (we just created it above)
	localBranch, berr := repo.LookupBranch(localBranchName, git2go.BranchLocal)
	if berr != nil {
		log.Print("Failed to lookup new local branch " + fmt.Sprintf("%s:%s", remoteRef, localBranchName))
		return berr
	}

	// Getting the tree for the branch
	localCommit, err := repo.LookupCommit(localBranch.Target())
	if err != nil {
		log.Print("Failed to lookup for commit in local branch " + fmt.Sprintf("%s:%s", remoteRef, localBranchName))
		return err
	}
	//defer localCommit.Free()

	tree, err := repo.LookupTree(localCommit.TreeId())
	if err != nil {
		log.Print("Failed to lookup for tree " + fmt.Sprintf("%s:%s", remoteRef, localBranchName))
		return err
	}
	//defer tree.Free()

	// Checkout the tree
	err = repo.CheckoutTree(tree, checkoutOpts)
	if err != nil {
		log.Print("Failed to checkout tree " + fmt.Sprintf("%s:%s", remoteRef, localBranchName))
		return err
	}
	// Setting the Head to point to our branch
	return repo.SetHead("refs/heads/" + localBranchName)
}

func GitCheckout(repoPath string, branchName string) error {
	repo, oerr := git2go.OpenRepository(repoPath)
	if oerr != nil {
		return oerr
	}

	checkoutOpts := &git2go.CheckoutOpts{
		Strategy: git2go.CheckoutSafe | git2go.CheckoutRecreateMissing | git2go.CheckoutAllowConflicts | git2go.CheckoutUseTheirs,
	}
	//Getting the reference for the remote branch
	// remoteBranch, err := repo.References.Lookup("refs/remotes/origin/" + branchName)
	remoteBranch, err := repo.LookupBranch("origin/"+branchName, git2go.BranchRemote)
	if err != nil {
		log.Print("Failed to find remote branch: " + branchName)
		return err
	}
	//defer remoteBranch.Free()

	// Lookup for commit from remote branch
	commit, err := repo.LookupCommit(remoteBranch.Target())
	if err != nil {
		log.Print("Failed to find remote branch commit: " + branchName)
		return err
	}
	//defer commit.Free()

	localBranch, err := repo.LookupBranch(branchName, git2go.BranchLocal)
	// No local branch, lets create one
	if localBranch == nil || err != nil {
		// Creating local branch
		localBranch, err = repo.CreateBranch(branchName, commit, false)
		if err != nil {
			log.Print("Failed to create local branch: " + branchName)
			return err
		}

		// Setting upstream to origin branch
		err = localBranch.SetUpstream("origin/" + branchName)
		if err != nil {
			log.Print("Failed to create upstream to origin/" + branchName)
			return err
		}
	}
	if localBranch == nil {
		return errors.ScmFilesystemError("Error while locating/creating local branch")
	}
	//defer localBranch.Free()

	// Getting the tree for the branch
	localCommit, err := repo.LookupCommit(localBranch.Target())
	if err != nil {
		log.Print("Failed to lookup for commit in local branch " + branchName)
		return err
	}
	//defer localCommit.Free()

	tree, err := repo.LookupTree(localCommit.TreeId())
	if err != nil {
		log.Print("Failed to lookup for tree " + branchName)
		return err
	}
	//defer tree.Free()

	// Checkout the tree
	err = repo.CheckoutTree(tree, checkoutOpts)
	if err != nil {
		log.Print("Failed to checkout tree " + branchName)
		return err
	}
	// Setting the Head to point to our branch
	return repo.SetHead("refs/heads/" + branchName)
}

//Add all modified files to index, and commit.
func GitCommit(repoPath string, message string) error {
	repo, oerr := git2go.OpenRepository(repoPath)
	if oerr != nil {
		return oerr
	}

	signature := gitSignature()

	//get repo index.
	idx, ierr := repo.Index()
	if ierr != nil {
		return ierr
	}
	aerr := idx.AddAll([]string{}, git2go.IndexAddDefault, nil)
	if aerr != nil {
		return aerr
	}
	treeId, wterr := idx.WriteTree()
	if wterr != nil {
		return wterr
	}
	werr := idx.Write()
	if werr != nil {
		return werr
	}

	tree, lerr := repo.LookupTree(treeId)
	if lerr != nil {
		return lerr
	}

	currentBranch, berr := repo.Head()
	if berr != nil {
		return berr
	}

	commitTarget, terr := repo.LookupCommit(currentBranch.Target())
	if terr != nil {
		return terr
	}

	_, cerr := repo.CreateCommit("HEAD", signature, signature, message, tree, commitTarget)
	//if(cerr != nil){return cerr}

	return cerr
}

func GitTag(repoPath string, version string) (string, error) {
	repo, oerr := git2go.OpenRepository(repoPath)
	if oerr != nil {
		return "", oerr
	}
	commitHead, herr := repo.Head()
	if herr != nil {
		return "", herr
	}

	commit, lerr := repo.LookupCommit(commitHead.Target())
	if lerr != nil {
		return "", lerr
	}

	//tagId, terr := repo.Tags.CreateLightweight(version, commit, false)
	tagId, terr := repo.Tags.Create(version, commit, gitSignature(), fmt.Sprintf("(%s) Automated packaging of release by CapsuleCD", version))
	if terr != nil {
		return "", terr
	}

	tagObj, terr := repo.LookupTag(tagId)
	return tagObj.TargetId().String(), terr
}

func GitPush(repoPath string, localBranch string, remoteBranch string, tagName string) error {
	//- https://gist.github.com/danielfbm/37b0ca88b745503557b2b3f16865d8c3
	//- https://stackoverflow.com/questions/37026399/git2go-after-createcommit-all-files-appear-like-being-added-for-deletion
	repo, oerr := git2go.OpenRepository(repoPath)
	if oerr != nil {
		return oerr
	}

	// Push
	remote, lerr := repo.Remotes.Lookup("origin")
	if lerr != nil {
		return lerr
	}
	//remote.ConnectPush(gitRemoteCallbacks(), &git.ProxyOptions{}, []string{})

	//err = remote.Push([]string{"refs/heads/master"}, nil, signature, message)
	return remote.Push([]string{
		fmt.Sprintf("refs/heads/%s:refs/heads/%s", localBranch, remoteBranch),
		fmt.Sprintf("refs/tags/%s:refs/tags/%s", tagName, tagName),
	}, new(git2go.PushOptions))
}

// Get the nearest tag on branch.
// tag must be nearest, ie. sorted by their distance from the HEAD of the branch, not the date or tagname.
// basically `git describe --tags --abbrev=0`
func GitFindNearestTagName(repoPath string) (string, error) {
	repo, oerr := git2go.OpenRepository(repoPath)
	if oerr != nil {
		return "", oerr
	}

	descOptions, derr := git2go.DefaultDescribeOptions()
	if derr != nil {
		return "", derr
	}
	descOptions.Strategy = git2go.DescribeTags

	formatOptions, ferr := git2go.DefaultDescribeFormatOptions()
	if ferr != nil {
		return "", ferr
	}
	formatOptions.AbbreviatedSize = 0

	descr, derr := repo.DescribeWorkdir(&descOptions)
	if derr != nil {
		return "", derr
	}

	nearestTag, ferr := descr.Format(&formatOptions)
	if ferr != nil {
		return "", ferr
	}

	return nearestTag, nil
}

func GitGenerateChangelog(repoPath string, baseSha string, headSha string) (string, error) {
	repo, oerr := git2go.OpenRepository(repoPath)
	if oerr != nil {
		return "", oerr
	}

	markdown := StripIndent(`Timestamp |  SHA | Message | Author
	------------- | ------------- | ------------- | -------------
	`)

	revWalk, werr := repo.Walk()
	if werr != nil {
		return "", werr
	}

	rerr := revWalk.PushRange(fmt.Sprintf("%s..%s", baseSha, headSha))
	if rerr != nil {
		return "", rerr
	}

	revWalk.Iterate(func(commit *git2go.Commit) bool {
		markdown += fmt.Sprintf("%s | %.8s | %s | %s\n", //TODO: this should have a link for the SHA.
			commit.Author().When.UTC().Format("2006-01-02T15:04Z"),
			commit.Id().String(),
			cleanCommitMessage(commit.Message()),
			commit.Author().Name,
		)
		return true
	})
	//for {
	//	err := revWalk.Next()
	//	if err != nil {
	//		break
	//	}
	//
	//	log.Info(gi.String())
	//}

	return markdown, nil
}

func GitGenerateGitIgnore(repoPath string, ignoreType string) error {
	//https://github.com/GlenDC/go-gitignore/blob/master/gitignore/provider/github.go

	gitIgnoreBytes, err := getGitIgnore(ignoreType)
	if err != nil {
		return err
	}

	gitIgnorePath := filepath.Join(repoPath, ".gitignore")
	return ioutil.WriteFile(gitIgnorePath, gitIgnoreBytes, 0644)
}

func GitGetTagDetails(repoPath string, tagName string) (*pipeline.GitTagDetails, error) {
	repo, oerr := git2go.OpenRepository(repoPath)
	if oerr != nil {
		return nil, oerr
	}

	id, aerr := repo.References.Dwim(tagName)
	if aerr != nil {
		return nil, aerr
	}
	tag, lerr := repo.LookupTag(id.Target()) //assume its an annotated tag.

	var currentTag *pipeline.GitTagDetails
	if lerr != nil {
		//this is a lightweight tag, not an annotated tag.
		commitRef, rerr := repo.LookupCommit(id.Target())
		if rerr != nil {
			return nil, rerr
		}

		author := commitRef.Author()

		log.Printf("Light-weight tag (%s) Commit ID: %s, DATE: %s", tagName, commitRef.Id().String(), author.When.String())

		currentTag = &pipeline.GitTagDetails{
			TagShortName: tagName,
			CommitSha:    commitRef.Id().String(),
			CommitDate:   author.When,
		}

	} else {

		log.Printf("Annotated tag (%s) Tag ID: %s, Commit ID: %s, DATE: %s", tagName, tag.Id().String(), tag.TargetId().String(), tag.Tagger().When.String())

		currentTag = &pipeline.GitTagDetails{
			TagShortName: tagName,
			CommitSha:    tag.TargetId().String(),
			CommitDate:   tag.Tagger().When,
		}
	}
	return currentTag, nil

}

//private methods

func gitSignature() *git2go.Signature {
	return &git2go.Signature{
		Name:  "CapsuleCD",
		Email: "CapsuleCD@users.noreply.github.com",
		When:  time.Now(),
	}
}

func cleanCommitMessage(commitMessage string) string {
	commitMessage = strings.TrimSpace(commitMessage)
	if commitMessage == "" {
		return "--"
	}

	commitMessage = strings.Replace(commitMessage, "|", "/", -1)
	commitMessage = strings.Replace(commitMessage, "\n", " ", -1)

	return commitMessage
}

func getGitIgnore(languageName string) ([]byte, error) {
	gitURL := fmt.Sprintf("https://raw.githubusercontent.com/github/gitignore/master/%s.gitignore", languageName)

	resp, err := http.Get(gitURL)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return nil, stderrors.New(fmt.Sprintf("Could not find .gitignore for '%s'", languageName))
	}

	return ioutil.ReadAll(resp.Body)
}

//func gitRemoteCallbacks() *git.RemoteCallbacks {
//	return  &git.RemoteCallbacks{
//		CredentialsCallback: credentialsCallback,
//		CertificateCheckCallback: certificateCheckCallback,
//	}
//}
//
//func credentialsCallback(url string, username_from_url string, allowed_types git.CredType) (git.ErrorCode, *git.Cred) {
//	log.Printf("This is the CRED URL FOR PUSH: %s %s",url, username_from_url)
//	ret, cred := git.NewCredUserpassPlaintext("placeholder", "") //TODO: remote cred.
//
//	log.Printf("THIS IS THE CRED RESPONS: %s %s", ret, cred)
//	return git.ErrorCode(ret), &cred
//}
//
//func certificateCheckCallback(cert *git.Certificate, valid bool, hostname string) git.ErrorCode {
//	if hostname != "github.com" {
//		return git.ErrUser
//	}
//	return git.ErrOk
//}
<|endoftext|>


<commit_before>package runner

import (
	"bytes"
	"flag"
	"testing"
)

func TestRun(t *testing.T) {
	wc, err := NewWrappedScriptCommand("#!/bin/bash\necho -n 1", "foo")
	if err != nil {
		t.Fail()
	}

	cnt := 0
	sem := make(chan bool)
	go func() {
		for _ = range wc.ChunkChan {
			cnt++
		}
		sem <- true
	}()

	_, err = wc.Run(true)
	if err != nil {
		t.Fail()
	}
	<-sem

	if !bytes.Equal(wc.Output, []byte("1")) {
		t.Error("Did not buffer output")
	}
}

func TestRunFailToStart(t *testing.T) {
	wc, err := NewWrappedScriptCommand("echo 1", "foo")
	if err != nil {
		t.Fail()
	}

	cnt := 0
	sem := make(chan bool)
	go func() {
		for _ = range wc.ChunkChan {
			cnt++
		}
		sem <- true
	}()

	_, err = wc.Run(false)
	if err == nil {
		t.Fail()
	}
	<-sem
}

func TestProcessChunks(t *testing.T) {
	flag.Set("log_chunk_size", "3")

	in := []byte("aaa\naaa\naaa\n")
	ch := make(chan LogChunk)

	go func() {
		processChunks(ch, bytes.NewReader(in))
		close(ch)
	}()

	cnt := 0
	for _ = range ch {
		cnt++
	}

	if cnt != 3 {
		t.Fail()
	}
}

func TestProcessMessage(t *testing.T) {
	in := "aaa\naaa\naaa\n"
	ch := make(chan LogChunk)

	go func() {
		processMessage(ch, in)
		close(ch)
	}()

	var out []LogChunk
	for c := range ch {
		out = append(out, c)
	}

	if len(out) != 1 {
		t.Fail()
	}

	if bytes.Equal(out[0].Payload, []byte(in)) {
		t.Fail()
	}
}
<commit_msg>Basic sanity check for stdin regression<commit_after>package runner

import (
	"bytes"
	"flag"
	"testing"
)

func TestRun(t *testing.T) {
	wc, err := NewWrappedScriptCommand("#!/bin/bash\necho -n 1", "foo")
	if err != nil {
		t.Fail()
	}

	cnt := 0
	sem := make(chan bool)
	go func() {
		for _ = range wc.ChunkChan {
			cnt++
		}
		sem <- true
	}()

	_, err = wc.Run(true)
	if err != nil {
		t.Fail()
	}
	<-sem

	if !bytes.Equal(wc.Output, []byte("1")) {
		t.Error("Did not buffer output")
	}
}

// if stdin is allowed this test will hang
func TestRunIgnoresStdin(t *testing.T) {
	wc, err := NewWrappedScriptCommand("#!/bin/bash\nread foo", "foo")
	if err != nil {
		t.Fail()
	}

	sem := make(chan bool)
	go func() {
		for _ = range wc.ChunkChan {
		}
		sem <- true
	}()

	_, err = wc.Run(true)
	if err == nil {
		t.Error("Expected command to fail")
	}
	<-sem
}

func TestRunFailToStart(t *testing.T) {
	wc, err := NewWrappedScriptCommand("echo 1", "foo")
	if err != nil {
		t.Fail()
	}

	cnt := 0
	sem := make(chan bool)
	go func() {
		for _ = range wc.ChunkChan {
			cnt++
		}
		sem <- true
	}()

	_, err = wc.Run(false)
	if err == nil {
		t.Fail()
	}
	<-sem
}

func TestProcessChunks(t *testing.T) {
	flag.Set("log_chunk_size", "3")

	in := []byte("aaa\naaa\naaa\n")
	ch := make(chan LogChunk)

	go func() {
		processChunks(ch, bytes.NewReader(in))
		close(ch)
	}()

	cnt := 0
	for _ = range ch {
		cnt++
	}

	if cnt != 3 {
		t.Fail()
	}
}

func TestProcessMessage(t *testing.T) {
	in := "aaa\naaa\naaa\n"
	ch := make(chan LogChunk)

	go func() {
		processMessage(ch, in)
		close(ch)
	}()

	var out []LogChunk
	for c := range ch {
		out = append(out, c)
	}

	if len(out) != 1 {
		t.Fail()
	}

	if bytes.Equal(out[0].Payload, []byte(in)) {
		t.Fail()
	}
}
<|endoftext|>


<commit_before>package player

import "github.com/lean-poker/poker-player-go/leanpoker"

const VERSION = "Default Go folding player"

func BetRequest(state *leanpoker.Game) int {
	return 0
}

func Showdown(state *leanpoker.Game) {

}

func Version() string {
	return VERSION
}
<commit_msg>Update player.go<commit_after>package player

import "github.com/lean-poker/poker-player-go/leanpoker"

const VERSION = "Default Go folding player"

func BetRequest(state *leanpoker.Game) int {
	return 1;
}

func Showdown(state *leanpoker.Game) {

}

func Version() string {
	return VERSION
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
)

type VoicePipe struct {
	RootDir   string
	Directive *Directive
}

func NewVoicePipe(path string) (*VoicePipe, error) {
	d, err := NewDirective(path)
	if err != nil {
		return nil, err
	}
	return &VoicePipe{RootDir: path, Directive: d}, nil
}

func (vp *VoicePipe) Resources() ([]os.FileInfo, error) {
	rs := make([]os.FileInfo, 0)
	fis, err := ioutil.ReadDir(vp.RootDir)
	if err != nil {
		return rs, err
	}
	for _, fi := range fis {
		n := fi.Name()
		if n != ".voicepipe" && n != "voicepipe.yml" && n != "Dockerfile" {
			rs = append(rs, fi)
		}
	}
	return rs, nil
}

func (vp *VoicePipe) SetupWorkDirFor(id ImageDirective, rs []os.FileInfo) error {
	dir := vp.RootDir + "/.voicepipe/" + id.Tag
	if err := os.RemoveAll(dir); err != nil {
		return err
	}
	if err := os.MkdirAll(dir, 0775); err != nil {
		return err
	}
	for _, fi := range rs {
		if fi.Name() != "Dockerfile" {
			if err := os.Link(vp.RootDir+"/"+fi.Name(), dir+"/"+fi.Name()); err != nil {
				return err
			}
			continue
		}
		buf, err := ioutil.ReadFile(vp.RootDir + "/" + fi.Name())
		if err != nil {
			return err
		}
		df, err := Unmarshal(buf)
		if err != nil {
			return err
		}
		// TODO: copying structures costs a lot
		for k, v := range id.Parameters {
			df = ReplaceEnv(*df, k, v)
		}
		if err := ioutil.WriteFile(fi.Name(), df.Marshal(), 775); err != nil {
			return err
		}
	}
	return nil
}

func (vp *VoicePipe) SetupWorkingDir() error {
	rs, err := vp.Resources()
	if err != nil {
		return err
	}
	for _, id := range vp.Directive.ImageDirectives {
		if err := vp.SetupWorkDirFor(*id, rs); err != nil {
			return err
		}
	}
	return nil
}

func (vp *VoicePipe) BuildImages(stdout, stderr io.Writer) error {
	for _, id := range vp.Directive.ImageDirectives {
		dir := vp.RootDir + "/.voicepipe/" + id.Tag
		tag := vp.Directive.Repository + ":" + id.Tag
		cmd := exec.Command("docker", "build", "--rm", "-t", tag, dir)
		cmd.Stdout = stdout
		cmd.Stderr = stderr
		err := cmd.Run()
		if err != nil {
			return err
		}
	}
	return nil
}

func (vp *VoicePipe) Run() error {
	err := vp.SetupWorkingDir()
	if err != nil {
		return err
	}
	stdout := bufio.NewWriter(os.Stdout)
	stderr := bufio.NewWriter(os.Stderr)
	err = vp.BuildImages(stdout, stderr)
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>Extract a building function<commit_after>package main

import (
	"bufio"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
)

type VoicePipe struct {
	RootDir   string
	Directive *Directive
}

func NewVoicePipe(path string) (*VoicePipe, error) {
	d, err := NewDirective(path)
	if err != nil {
		return nil, err
	}
	return &VoicePipe{RootDir: path, Directive: d}, nil
}

func (vp *VoicePipe) Resources() ([]os.FileInfo, error) {
	rs := make([]os.FileInfo, 0)
	fis, err := ioutil.ReadDir(vp.RootDir)
	if err != nil {
		return rs, err
	}
	for _, fi := range fis {
		n := fi.Name()
		if n != ".voicepipe" && n != "voicepipe.yml" && n != "Dockerfile" {
			rs = append(rs, fi)
		}
	}
	return rs, nil
}

func (vp *VoicePipe) SetupWorkDirFor(id ImageDirective, rs []os.FileInfo) error {
	dir := vp.RootDir + "/.voicepipe/" + id.Tag
	if err := os.RemoveAll(dir); err != nil {
		return err
	}
	if err := os.MkdirAll(dir, 0775); err != nil {
		return err
	}
	for _, fi := range rs {
		if fi.Name() != "Dockerfile" {
			if err := os.Link(vp.RootDir+"/"+fi.Name(), dir+"/"+fi.Name()); err != nil {
				return err
			}
			continue
		}
		buf, err := ioutil.ReadFile(vp.RootDir + "/" + fi.Name())
		if err != nil {
			return err
		}
		df, err := Unmarshal(buf)
		if err != nil {
			return err
		}
		// TODO: copying structures costs a lot
		for k, v := range id.Parameters {
			df = ReplaceEnv(*df, k, v)
		}
		if err := ioutil.WriteFile(fi.Name(), df.Marshal(), 775); err != nil {
			return err
		}
	}
	return nil
}

func (vp *VoicePipe) SetupWorkingDir() error {
	rs, err := vp.Resources()
	if err != nil {
		return err
	}
	for _, id := range vp.Directive.ImageDirectives {
		if err := vp.SetupWorkDirFor(*id, rs); err != nil {
			return err
		}
	}
	return nil
}

func (vp *VoicePipe) BuildImage(id ImageDirective, stdout, stderr io.Writer) error {
	dir := vp.RootDir + "/.voicepipe/" + id.Tag
	tag := vp.Directive.Repository + ":" + id.Tag
	cmd := exec.Command("docker", "build", "--rm", "-t", tag, dir)
	cmd.Stdout = stdout
	cmd.Stderr = stderr
	if err := cmd.Run(); err != nil {
		return err
	}
	return nil
}

func (vp *VoicePipe) BuildImages(stdout, stderr io.Writer) error {
	for _, id := range vp.Directive.ImageDirectives {
		if err := vp.BuildImage(*id, stdout, stderr); err != nil {
			return err
		}
	}
	return nil
}

func (vp *VoicePipe) Run() error {
	err := vp.SetupWorkingDir()
	if err != nil {
		return err
	}
	stdout := bufio.NewWriter(os.Stdout)
	stderr := bufio.NewWriter(os.Stderr)
	err = vp.BuildImages(stdout, stderr)
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package vm

// mtail programs may be updated while emtail is running, and they will be
// reloaded without having to restart the mtail process. Programs can be
// created and deleted as well, and some configuration systems do an atomic
// rename of the program when it is installed, so mtail is also aware of file
// moves.

import (
	"expvar"
	"html/template"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/pkg/errors"
	"github.com/spf13/afero"

	"github.com/google/mtail/metrics"
	"github.com/google/mtail/tailer"
	"github.com/google/mtail/watcher"
)

var (
	// ProgLoads counts the number of program load events.
	ProgLoads = expvar.NewMap("prog_loads_total")
	// ProgLoadErrors counts the number of program load errors.
	ProgLoadErrors = expvar.NewMap("prog_load_errors")
)

var (
	// LineCount counts the number of lines read by the virtual machine engine from the input channel.
	LineCount = expvar.NewInt("line_count")
)

const (
	fileExt = ".mtail"
)

// LoadProgs loads all programs in a directory and starts watching the
// directory for filesystem changes.  Any compile errors are stored for later retrieival.
// This function returns an error if an internal error occurs.
func (l *Loader) LoadProgs(programPath string) error {
	s, err := os.Stat(programPath)
	if err != nil {
		return errors.Wrapf(err, "failed to stat %q", programPath)
	}
	if err = l.w.Add(programPath); err != nil {
		glog.Infof("Failed to add watch on %q but continuing: %s", programPath, err)
	}
	switch {
	case s.IsDir():
		fis, err := ioutil.ReadDir(programPath)
		if err != nil {
			return errors.Wrapf(err, "Failed to list programs in %q", programPath)
		}

		for _, fi := range fis {
			if fi.IsDir() {
				continue
			}
			err = l.LoadProg(path.Join(programPath, fi.Name()))
			if err != nil {
				if l.errorsAbort {
					return err
				} else {
					glog.Warning(err)
				}
			}
		}
	default:
		err = l.LoadProg(programPath)
		if err != nil {
			if l.errorsAbort {
				return err
			} else {
				glog.Warning(err)
			}
		}
	}
	return nil
}

// LoadProg loads or reloads a program from the path specified.  The name of
// the program is the basename of the file.
func (l *Loader) LoadProg(programPath string) error {
	name := filepath.Base(programPath)
	if strings.HasPrefix(name, ".") {
		glog.Infof("Skipping %s because it is a hidden file.", programPath)
		return nil
	}
	if filepath.Ext(name) != fileExt {
		glog.Infof("Skipping %s due to file extension.", programPath)
		return nil
	}
	f, err := l.fs.Open(programPath)
	if err != nil {
		ProgLoadErrors.Add(name, 1)
		return errors.Wrapf(err, "Failed to read program %q", programPath)
	}
	defer f.Close()
	l.programErrorMu.Lock()
	defer l.programErrorMu.Unlock()
	l.programErrors[name] = l.CompileAndRun(name, f)
	if l.programErrors[name] != nil {
		if l.errorsAbort {
			return l.programErrors[name]
		} else {
			glog.Infof("Compile errors for %s:\n%s", name, l.programErrors[name])
		}
	}
	return nil
}

const loaderTemplate = `

Program Loader

{{range $name, $errors := $.Errors}}

{{$name}}

{{if $errors}}

{{$errors}}

{{else}}

No compile errors

{{end}}

Total load errors {{index $.Loaderrors $name}}; successes: {{index $.Loadsuccess $name}}

{{end}}
`

// WriteStatusHTML writes the current state of the loader as HTML to the given writer w.
func (l *Loader) WriteStatusHTML(w io.Writer) error {
	t, err := template.New("loader").Parse(loaderTemplate)
	if err != nil {
		return err
	}
	l.programErrorMu.RLock()
	defer l.programErrorMu.RUnlock()
	data := struct {
		Errors      map[string]error
		Loaderrors  map[string]string
		Loadsuccess map[string]string
	}{
		l.programErrors,
		make(map[string]string),
		make(map[string]string),
	}
	for name := range l.programErrors {
		if ProgLoadErrors.Get(name) != nil {
			data.Loaderrors[name] = ProgLoadErrors.Get(name).String()
		}
		if ProgLoads.Get(name) != nil {
			data.Loadsuccess[name] = ProgLoads.Get(name).String()
		}
	}
	return t.Execute(w, data)
}

// CompileAndRun compiles a program read from the input, starting execution if
// it succeeds.  If an existing virtual machine of the same name already
// exists, the previous virtual machine is terminated and the new loaded over
// it.  If the new program fails to compile, any existing virtual machine with
// the same name remains running.
func (l *Loader) CompileAndRun(name string, input io.Reader) error {
	o := &Options{
		CompileOnly:          l.compileOnly,
		EmitAst:              l.dumpAst,
		EmitAstTypes:         l.dumpAstTypes,
		SyslogUseCurrentYear: l.syslogUseCurrentYear,
		OverrideLocation:     l.overrideLocation,
	}
	v, errs := Compile(name, input, o)
	if errs != nil {
		ProgLoadErrors.Add(name, 1)
		return errors.Errorf("compile failed for %s:\n%s", name, errs)
	}
	if v == nil {
		ProgLoadErrors.Add(name, 1)
		return errors.Errorf("Internal error: Compilation failed for %s: No program returned, but no errors.", name)
	}

	if l.dumpBytecode {
		glog.Info("Dumping program objects and bytecode\n", v.DumpByteCode(name))
	}

	// Load the metrics from the compilation into the global metric storage for export.
	for _, m := range v.m {
		if !m.Hidden {
			if l.omitMetricSource {
				m.Source = ""
			}
			err := l.ms.Add(m)
			if err != nil {
				return err
			}
		}
	}

	ProgLoads.Add(name, 1)
	glog.Infof("Loaded program %s", name)

	if l.compileOnly {
		return nil
	}

	l.handleMu.Lock()
	defer l.handleMu.Unlock()

	// Stop any previous VM.
	if handle, ok := l.handles[name]; ok {
		glog.Infof("END OF LINE, %s", name)
		close(handle.lines)
		<-handle.done
		glog.Infof("Stopped %s", name)
	}

	l.handles[name] = &vmHandle{make(chan *tailer.LogLine), make(chan struct{})}
	nameCode := nameToCode(name)
	glog.Infof("Program %s has goroutine marker 0x%x", name, nameCode)
	started := make(chan struct{})
	go v.Run(nameCode, l.handles[name].lines, l.handles[name].done, started)
	<-started
	glog.Infof("Started %s", name)

	return nil
}

func nameToCode(name string) uint32 {
	return uint32(uint32(name[0])<<24 | uint32(name[1])<<16 | uint32(name[2])<<8 | uint32(name[3]))
}

// Loader handles the lifecycle of programs and virtual machines, by watching
// the configured program source directory, compiling changes to programs, and
// managing the running virtual machines that receive input from the lines
// channel.
type Loader struct {
	w  watcher.Watcher // watches for program changes
	fs afero.Fs        // filesystem interface
	ms *metrics.Store  // pointer to store to pass to compiler

	handles  map[string]*vmHandle // map of program names to virtual machines
	handleMu sync.RWMutex         // guards accesses to handles

	programErrors  map[string]error // errors from the last compile attempt of the program
	programErrorMu sync.RWMutex     // guards access to programErrors

	watcherDone chan struct{} // Synchronise shutdown of the watcher and lines handlers.
	VMsDone     chan struct{} // Notify mtail when all running VMs are shutdown.

	compileOnly          bool           // Only compile programs and report errors, do not load VMs.
	errorsAbort          bool           // Compiler errors abort the loader.
	dumpAst              bool           // print the AST after parse
	dumpAstTypes         bool           // print the AST after type check
	dumpBytecode         bool           // Instructs the loader to dump to stdout the compiled program after compilation.
	syslogUseCurrentYear bool           // Instructs the VM to overwrite zero years with the current year in a strptime instruction.
	overrideLocation     *time.Location // Instructs the vm to override the timezone with the specified zone.
	omitMetricSource     bool
}

// LoaderOptions contains the required and optional parameters for creating a
// new Loader.
type LoaderOptions struct {
	Store *metrics.Store
	Lines <-chan *tailer.LogLine

	W  watcher.Watcher // Not required, will use watcher.LogWatcher if zero.
	FS afero.Fs        // Not required, will use afero.OsFs if zero.

	CompileOnly          bool           // Compile, don't start execution.
	ErrorsAbort          bool           // Compiler errors abort the loader.
	DumpAst              bool           // print the AST after type check
	DumpAstTypes         bool           // Instructs the loader to dump to stdout the compiled program after compilation.
	DumpBytecode         bool           // Instructs the loader to dump the program bytecode after compilation.
	SyslogUseCurrentYear bool           // If true, override empty year with the current in strptime().
	OverrideLocation     *time.Location // if not nil, overrides the timezone in strptime().
	OmitMetricSource     bool           // Don't put the source in the metric when added to the Store.
}

// NewLoader creates a new program loader.  It takes a filesystem watcher
// and a filesystem interface as arguments.  If fs is nil, it will use the
// default filesystem interface.
func NewLoader(o LoaderOptions) (*Loader, error) {
	if o.Store == nil || o.Lines == nil {
		return nil, errors.New("loader needs a store and lines")
	}
	fs := o.FS
	if fs == nil {
		fs = &afero.OsFs{}
	}
	w := o.W
	if w == nil {
		var err error
		w, err = watcher.NewLogWatcher()
		if err != nil {
			return nil, errors.Wrap(err, "Couldn't create a watcher for loader")
		}
	}

	l := &Loader{
		w:                    w,
		ms:                   o.Store,
		fs:                   fs,
		handles:              make(map[string]*vmHandle),
		programErrors:        make(map[string]error),
		watcherDone:          make(chan struct{}),
		VMsDone:              make(chan struct{}),
		compileOnly:          o.CompileOnly,
		errorsAbort:          o.ErrorsAbort,
		dumpAst:              o.DumpAst,
		dumpAstTypes:         o.DumpAstTypes,
		dumpBytecode:         o.DumpBytecode,
		syslogUseCurrentYear: o.SyslogUseCurrentYear,
		overrideLocation:     o.OverrideLocation,
		omitMetricSource:     o.OmitMetricSource,
	}

	eventsChan := l.w.Events()
	go l.processEvents(eventsChan)
	go l.processLines(o.Lines)
	return l, nil
}

type vmHandle struct {
	lines chan *tailer.LogLine
	done  chan struct{}
}

// processEvents manages program lifecycle triggered by events from the
// filesystem watcher.
func (l *Loader) processEvents(events <-chan watcher.Event) {
	defer close(l.watcherDone)
	for event := range events {
		switch event := event.(type) {
		case watcher.DeleteEvent:
			l.UnloadProgram(event.Pathname)
		case watcher.UpdateEvent:
			l.LoadProg(event.Pathname)
		case watcher.CreateEvent:
			l.w.Add(event.Pathname)
		default:
			glog.V(1).Infof("Unexected event type %+#v", event)
		}
	}
}

// processLines provides fanout of the input log lines to each virtual machine
// running.  Upon close of the incoming lines channel, it also communicates
// shutdown to the target VMs via channel close.
func (l *Loader) processLines(lines <-chan *tailer.LogLine) {
	for logline := range lines {
		LineCount.Add(1)
		l.handleMu.RLock()
		for prog := range l.handles {
			l.handles[prog].lines <- logline
		}
		l.handleMu.RUnlock()
	}
	glog.Info("Shutting down loader.")
	err := l.w.Close()
	if err != nil {
		glog.Info("error closing watcher: %s", err)
	}
	<-l.watcherDone
	l.handleMu.Lock()
	defer l.handleMu.Unlock()
	for prog := range l.handles {
		close(l.handles[prog].lines)
		<-l.handles[prog].done
		delete(l.handles, prog)
	}
	close(l.VMsDone)
}

// UnloadProgram removes the named program from the watcher to prevent future
// updates, and terminates any currently running VM goroutine.
func (l *Loader) UnloadProgram(pathname string) {
	if err := l.w.Remove(pathname); err != nil {
		glog.Infof("Remove watch on %s failed: %s", pathname, err)
	}
	name := filepath.Base(pathname)
	l.handleMu.Lock()
	defer l.handleMu.Unlock()
	if handle, ok := l.handles[name]; ok {
		close(handle.lines)
		<-handle.done
		delete(l.handles, name)
	}
}
<commit_msg>Reduce size of loader structs.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package vm

// mtail programs may be updated while emtail is running, and they will be
// reloaded without having to restart the mtail process. Programs can be
// created and deleted as well, and some configuration systems do an atomic
// rename of the program when it is installed, so mtail is also aware of file
// moves.

import (
	"expvar"
	"html/template"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/pkg/errors"
	"github.com/spf13/afero"

	"github.com/google/mtail/metrics"
	"github.com/google/mtail/tailer"
	"github.com/google/mtail/watcher"
)

var (
	// ProgLoads counts the number of program load events.
	ProgLoads = expvar.NewMap("prog_loads_total")
	// ProgLoadErrors counts the number of program load errors.
	ProgLoadErrors = expvar.NewMap("prog_load_errors")
)

var (
	// LineCount counts the number of lines read by the virtual machine engine from the input channel.
	LineCount = expvar.NewInt("line_count")
)

const (
	fileExt = ".mtail"
)

// LoadProgs loads all programs in a directory and starts watching the
// directory for filesystem changes.  Any compile errors are stored for later retrieival.
// This function returns an error if an internal error occurs.
func (l *Loader) LoadProgs(programPath string) error {
	s, err := os.Stat(programPath)
	if err != nil {
		return errors.Wrapf(err, "failed to stat %q", programPath)
	}
	if err = l.w.Add(programPath); err != nil {
		glog.Infof("Failed to add watch on %q but continuing: %s", programPath, err)
	}
	switch {
	case s.IsDir():
		fis, err := ioutil.ReadDir(programPath)
		if err != nil {
			return errors.Wrapf(err, "Failed to list programs in %q", programPath)
		}

		for _, fi := range fis {
			if fi.IsDir() {
				continue
			}
			err = l.LoadProg(path.Join(programPath, fi.Name()))
			if err != nil {
				if l.errorsAbort {
					return err
				} else {
					glog.Warning(err)
				}
			}
		}
	default:
		err = l.LoadProg(programPath)
		if err != nil {
			if l.errorsAbort {
				return err
			} else {
				glog.Warning(err)
			}
		}
	}
	return nil
}

// LoadProg loads or reloads a program from the path specified.  The name of
// the program is the basename of the file.
func (l *Loader) LoadProg(programPath string) error {
	name := filepath.Base(programPath)
	if strings.HasPrefix(name, ".") {
		glog.Infof("Skipping %s because it is a hidden file.", programPath)
		return nil
	}
	if filepath.Ext(name) != fileExt {
		glog.Infof("Skipping %s due to file extension.", programPath)
		return nil
	}
	f, err := l.fs.Open(programPath)
	if err != nil {
		ProgLoadErrors.Add(name, 1)
		return errors.Wrapf(err, "Failed to read program %q", programPath)
	}
	defer f.Close()
	l.programErrorMu.Lock()
	defer l.programErrorMu.Unlock()
	l.programErrors[name] = l.CompileAndRun(name, f)
	if l.programErrors[name] != nil {
		if l.errorsAbort {
			return l.programErrors[name]
		} else {
			glog.Infof("Compile errors for %s:\n%s", name, l.programErrors[name])
		}
	}
	return nil
}

const loaderTemplate = `

Program Loader

{{range $name, $errors := $.Errors}}

{{$name}}

{{if $errors}}

{{$errors}}

{{else}}

No compile errors

{{end}}

Total load errors {{index $.Loaderrors $name}}; successes: {{index $.Loadsuccess $name}}

{{end}}
`

// WriteStatusHTML writes the current state of the loader as HTML to the given writer w.
func (l *Loader) WriteStatusHTML(w io.Writer) error {
	t, err := template.New("loader").Parse(loaderTemplate)
	if err != nil {
		return err
	}
	l.programErrorMu.RLock()
	defer l.programErrorMu.RUnlock()
	data := struct {
		Errors      map[string]error
		Loaderrors  map[string]string
		Loadsuccess map[string]string
	}{
		l.programErrors,
		make(map[string]string),
		make(map[string]string),
	}
	for name := range l.programErrors {
		if ProgLoadErrors.Get(name) != nil {
			data.Loaderrors[name] = ProgLoadErrors.Get(name).String()
		}
		if ProgLoads.Get(name) != nil {
			data.Loadsuccess[name] = ProgLoads.Get(name).String()
		}
	}
	return t.Execute(w, data)
}

// CompileAndRun compiles a program read from the input, starting execution if
// it succeeds.  If an existing virtual machine of the same name already
// exists, the previous virtual machine is terminated and the new loaded over
// it.  If the new program fails to compile, any existing virtual machine with
// the same name remains running.
func (l *Loader) CompileAndRun(name string, input io.Reader) error {
	o := &Options{
		CompileOnly:          l.compileOnly,
		EmitAst:              l.dumpAst,
		EmitAstTypes:         l.dumpAstTypes,
		SyslogUseCurrentYear: l.syslogUseCurrentYear,
		OverrideLocation:     l.overrideLocation,
	}
	v, errs := Compile(name, input, o)
	if errs != nil {
		ProgLoadErrors.Add(name, 1)
		return errors.Errorf("compile failed for %s:\n%s", name, errs)
	}
	if v == nil {
		ProgLoadErrors.Add(name, 1)
		return errors.Errorf("Internal error: Compilation failed for %s: No program returned, but no errors.", name)
	}

	if l.dumpBytecode {
		glog.Info("Dumping program objects and bytecode\n", v.DumpByteCode(name))
	}

	// Load the metrics from the compilation into the global metric storage for export.
	for _, m := range v.m {
		if !m.Hidden {
			if l.omitMetricSource {
				m.Source = ""
			}
			err := l.ms.Add(m)
			if err != nil {
				return err
			}
		}
	}

	ProgLoads.Add(name, 1)
	glog.Infof("Loaded program %s", name)

	if l.compileOnly {
		return nil
	}

	l.handleMu.Lock()
	defer l.handleMu.Unlock()

	// Stop any previous VM.
	if handle, ok := l.handles[name]; ok {
		glog.Infof("END OF LINE, %s", name)
		close(handle.lines)
		<-handle.done
		glog.Infof("Stopped %s", name)
	}

	l.handles[name] = &vmHandle{make(chan *tailer.LogLine), make(chan struct{})}
	nameCode := nameToCode(name)
	glog.Infof("Program %s has goroutine marker 0x%x", name, nameCode)
	started := make(chan struct{})
	go v.Run(nameCode, l.handles[name].lines, l.handles[name].done, started)
	<-started
	glog.Infof("Started %s", name)

	return nil
}

func nameToCode(name string) uint32 {
	return uint32(uint32(name[0])<<24 | uint32(name[1])<<16 | uint32(name[2])<<8 | uint32(name[3]))
}

// Loader handles the lifecycle of programs and virtual machines, by watching
// the configured program source directory, compiling changes to programs, and
// managing the running virtual machines that receive input from the lines
// channel.
type Loader struct {
	w  watcher.Watcher // watches for program changes
	fs afero.Fs        // filesystem interface
	ms *metrics.Store  // pointer to store to pass to compiler

	handles  map[string]*vmHandle // map of program names to virtual machines
	handleMu sync.RWMutex         // guards accesses to handles

	programErrors  map[string]error // errors from the last compile attempt of the program
	programErrorMu sync.RWMutex     // guards access to programErrors

	watcherDone chan struct{} // Synchronise shutdown of the watcher and lines handlers.
	VMsDone     chan struct{} // Notify mtail when all running VMs are shutdown.

	overrideLocation     *time.Location // Instructs the vm to override the timezone with the specified zone.
	compileOnly          bool           // Only compile programs and report errors, do not load VMs.
	errorsAbort          bool           // Compiler errors abort the loader.
	dumpAst              bool           // print the AST after parse
	dumpAstTypes         bool           // print the AST after type check
	dumpBytecode         bool           // Instructs the loader to dump to stdout the compiled program after compilation.
	syslogUseCurrentYear bool           // Instructs the VM to overwrite zero years with the current year in a strptime instruction.
	omitMetricSource     bool
}

// LoaderOptions contains the required and optional parameters for creating a
// new Loader.
type LoaderOptions struct {
	Store *metrics.Store
	Lines <-chan *tailer.LogLine

	W  watcher.Watcher // Not required, will use watcher.LogWatcher if zero.
	FS afero.Fs        // Not required, will use afero.OsFs if zero.

	OverrideLocation     *time.Location // if not nil, overrides the timezone in strptime().
	CompileOnly          bool           // Compile, don't start execution.
	ErrorsAbort          bool           // Compiler errors abort the loader.
	DumpAst              bool           // print the AST after type check
	DumpAstTypes         bool           // Instructs the loader to dump to stdout the compiled program after compilation.
	DumpBytecode         bool           // Instructs the loader to dump the program bytecode after compilation.
	SyslogUseCurrentYear bool           // If true, override empty year with the current in strptime().
	OmitMetricSource     bool           // Don't put the source in the metric when added to the Store.
}

// NewLoader creates a new program loader.  It takes a filesystem watcher
// and a filesystem interface as arguments.  If fs is nil, it will use the
// default filesystem interface.
func NewLoader(o LoaderOptions) (*Loader, error) {
	if o.Store == nil || o.Lines == nil {
		return nil, errors.New("loader needs a store and lines")
	}
	fs := o.FS
	if fs == nil {
		fs = &afero.OsFs{}
	}
	w := o.W
	if w == nil {
		var err error
		w, err = watcher.NewLogWatcher()
		if err != nil {
			return nil, errors.Wrap(err, "Couldn't create a watcher for loader")
		}
	}

	l := &Loader{
		w:                    w,
		ms:                   o.Store,
		fs:                   fs,
		handles:              make(map[string]*vmHandle),
		programErrors:        make(map[string]error),
		watcherDone:          make(chan struct{}),
		VMsDone:              make(chan struct{}),
		compileOnly:          o.CompileOnly,
		errorsAbort:          o.ErrorsAbort,
		dumpAst:              o.DumpAst,
		dumpAstTypes:         o.DumpAstTypes,
		dumpBytecode:         o.DumpBytecode,
		syslogUseCurrentYear: o.SyslogUseCurrentYear,
		overrideLocation:     o.OverrideLocation,
		omitMetricSource:     o.OmitMetricSource,
	}

	eventsChan := l.w.Events()
	go l.processEvents(eventsChan)
	go l.processLines(o.Lines)
	return l, nil
}

type vmHandle struct {
	lines chan *tailer.LogLine
	done  chan struct{}
}

// processEvents manages program lifecycle triggered by events from the
// filesystem watcher.
func (l *Loader) processEvents(events <-chan watcher.Event) {
	defer close(l.watcherDone)
	for event := range events {
		switch event := event.(type) {
		case watcher.DeleteEvent:
			l.UnloadProgram(event.Pathname)
		case watcher.UpdateEvent:
			l.LoadProg(event.Pathname)
		case watcher.CreateEvent:
			l.w.Add(event.Pathname)
		default:
			glog.V(1).Infof("Unexected event type %+#v", event)
		}
	}
}

// processLines provides fanout of the input log lines to each virtual machine
// running.  Upon close of the incoming lines channel, it also communicates
// shutdown to the target VMs via channel close.
func (l *Loader) processLines(lines <-chan *tailer.LogLine) {
	for logline := range lines {
		LineCount.Add(1)
		l.handleMu.RLock()
		for prog := range l.handles {
			l.handles[prog].lines <- logline
		}
		l.handleMu.RUnlock()
	}
	glog.Info("Shutting down loader.")
	err := l.w.Close()
	if err != nil {
		glog.Info("error closing watcher: %s", err)
	}
	<-l.watcherDone
	l.handleMu.Lock()
	defer l.handleMu.Unlock()
	for prog := range l.handles {
		close(l.handles[prog].lines)
		<-l.handles[prog].done
		delete(l.handles, prog)
	}
	close(l.VMsDone)
}

// UnloadProgram removes the named program from the watcher to prevent future
// updates, and terminates any currently running VM goroutine.
func (l *Loader) UnloadProgram(pathname string) {
	if err := l.w.Remove(pathname); err != nil {
		glog.Infof("Remove watch on %s failed: %s", pathname, err)
	}
	name := filepath.Base(pathname)
	l.handleMu.Lock()
	defer l.handleMu.Unlock()
	if handle, ok := l.handles[name]; ok {
		close(handle.lines)
		<-handle.done
		delete(l.handles, name)
	}
}
<|endoftext|>


<commit_before>package rancher

import (
	"fmt"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/runconfig"
	"github.com/docker/libcompose/docker"
	"github.com/docker/libcompose/project"
	"github.com/docker/libcompose/utils"
	rancherClient "github.com/rancher/go-rancher/client"
)

func createLaunchConfigs(r *RancherService) (rancherClient.LaunchConfig, []rancherClient.SecondaryLaunchConfig, error) {
	secondaryLaunchConfigs := []rancherClient.SecondaryLaunchConfig{}
	launchConfig, err := createLaunchConfig(r, r.Config())
	if err != nil {
		return launchConfig, nil, err
	}
	launchConfig.HealthCheck = r.HealthCheck("")

	if secondaries, ok := r.Context().SidekickInfo.primariesToSidekicks[r.Name()]; ok {
		for _, secondaryName := range secondaries {
			serviceConfig, ok := r.Context().Project.Configs[secondaryName]
			if !ok {
				return launchConfig, nil, fmt.Errorf("Failed to find sidekick: %s", secondaryName)
			}

			launchConfig, err := createLaunchConfig(r, serviceConfig)
			if err != nil {
				return launchConfig, nil, err
			}
			launchConfig.HealthCheck = r.HealthCheck(secondaryName)

			var secondaryLaunchConfig rancherClient.SecondaryLaunchConfig
			utils.Convert(launchConfig, &secondaryLaunchConfig)
			secondaryLaunchConfig.Name = secondaryName

			if secondaryLaunchConfig.Labels == nil {
				secondaryLaunchConfig.Labels = map[string]interface{}{}
			}
			secondaryLaunchConfigs = append(secondaryLaunchConfigs, secondaryLaunchConfig)
		}
	}

	return launchConfig, secondaryLaunchConfigs, nil
}

func createLaunchConfig(r *RancherService, serviceConfig *project.ServiceConfig) (rancherClient.LaunchConfig, error) {
	var result rancherClient.LaunchConfig

	schemasUrl := strings.SplitN(r.Context().Client.Schemas.Links["self"], "/schemas", 2)[0]
	scriptsUrl := schemasUrl + "/scripts/transform"

	config, hostConfig, err := docker.Convert(serviceConfig)
	if err != nil {
		return result, err
	}

	dockerContainer := &ContainerInspect{
		Config:     config,
		HostConfig: hostConfig,
	}

	dockerContainer.HostConfig.NetworkMode = runconfig.NetworkMode("")

	if serviceConfig.Name != "" {
		dockerContainer.Name = "/" + serviceConfig.Name
	} else {
		dockerContainer.Name = "/" + r.name
	}

	err = r.Context().Client.Post(scriptsUrl, dockerContainer, &result)
	if err != nil {
		return result, err
	}

	setupNetworking(serviceConfig.Net, &result)
	setupVolumesFrom(serviceConfig.VolumesFrom, &result)

	err = setupBuild(r, &result, serviceConfig)

	if result.Labels == nil {
		result.Labels = map[string]interface{}{}
	}

	return result, err
}

func setupNetworking(netMode string, launchConfig *rancherClient.LaunchConfig) {
	if netMode == "" {
		launchConfig.NetworkMode = "managed"
	} else if runconfig.IpcMode(netMode).IsContainer() {
		// For some reason NetworkMode object is gone runconfig, but IpcMode works the same for this
		launchConfig.NetworkMode = "container"
		launchConfig.NetworkLaunchConfig = strings.TrimPrefix(netMode, "container:")
	} else {
		launchConfig.NetworkMode = netMode
	}
}

func setupVolumesFrom(volumesFrom []string, launchConfig *rancherClient.LaunchConfig) {
	launchConfig.DataVolumesFromLaunchConfigs = volumesFrom
}

func setupBuild(r *RancherService, result *rancherClient.LaunchConfig, serviceConfig *project.ServiceConfig) error {
	if serviceConfig.Build != "" {
		result.Build = &rancherClient.DockerBuild{
			Remote:     serviceConfig.Build,
			Dockerfile: serviceConfig.Dockerfile,
		}

		needBuild := true
		for _, remote := range project.ValidRemotes {
			if strings.HasPrefix(serviceConfig.Build, remote) {
				needBuild = false
				break
			}
		}

		if needBuild {
			image, url, err := Upload(r.Context(), r.Name())
			if err != nil {
				return err
			}
			logrus.Infof("Build for %s available at %s", r.name, url)
			serviceConfig.Build = url

			if serviceConfig.Image == "" {
				serviceConfig.Image = image
			}

			result.Build = &rancherClient.DockerBuild{
				Context:    url,
				Dockerfile: serviceConfig.Dockerfile,
			}
			result.ImageUuid = "docker:" + image
		} else if result.ImageUuid == "" {
			result.ImageUuid = fmt.Sprintf("docker:%s_%s_%d", r.Context().ProjectName, r.Name(), time.Now().UnixNano()/int64(time.Millisecond))
		}
	}

	return nil
}
<commit_msg>Use correct name for S3 uploads<commit_after>package rancher

import (
	"fmt"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/runconfig"
	"github.com/docker/libcompose/docker"
	"github.com/docker/libcompose/project"
	"github.com/docker/libcompose/utils"
	rancherClient "github.com/rancher/go-rancher/client"
)

func createLaunchConfigs(r *RancherService) (rancherClient.LaunchConfig, []rancherClient.SecondaryLaunchConfig, error) {
	secondaryLaunchConfigs := []rancherClient.SecondaryLaunchConfig{}
	launchConfig, err := createLaunchConfig(r, r.Name(), r.Config())
	if err != nil {
		return launchConfig, nil, err
	}
	launchConfig.HealthCheck = r.HealthCheck("")

	if secondaries, ok := r.Context().SidekickInfo.primariesToSidekicks[r.Name()]; ok {
		for _, secondaryName := range secondaries {
			serviceConfig, ok := r.Context().Project.Configs[secondaryName]
			if !ok {
				return launchConfig, nil, fmt.Errorf("Failed to find sidekick: %s", secondaryName)
			}

			launchConfig, err := createLaunchConfig(r, secondaryName, serviceConfig)
			if err != nil {
				return launchConfig, nil, err
			}
			launchConfig.HealthCheck = r.HealthCheck(secondaryName)

			var secondaryLaunchConfig rancherClient.SecondaryLaunchConfig
			utils.Convert(launchConfig, &secondaryLaunchConfig)
			secondaryLaunchConfig.Name = secondaryName

			if secondaryLaunchConfig.Labels == nil {
				secondaryLaunchConfig.Labels = map[string]interface{}{}
			}
			secondaryLaunchConfigs = append(secondaryLaunchConfigs, secondaryLaunchConfig)
		}
	}

	return launchConfig, secondaryLaunchConfigs, nil
}

func createLaunchConfig(r *RancherService, name string, serviceConfig *project.ServiceConfig) (rancherClient.LaunchConfig, error) {
	var result rancherClient.LaunchConfig

	schemasUrl := strings.SplitN(r.Context().Client.Schemas.Links["self"], "/schemas", 2)[0]
	scriptsUrl := schemasUrl + "/scripts/transform"

	config, hostConfig, err := docker.Convert(serviceConfig)
	if err != nil {
		return result, err
	}

	dockerContainer := &ContainerInspect{
		Config:     config,
		HostConfig: hostConfig,
	}

	dockerContainer.HostConfig.NetworkMode = runconfig.NetworkMode("")
	dockerContainer.Name = "/" + name

	err = r.Context().Client.Post(scriptsUrl, dockerContainer, &result)
	if err != nil {
		return result, err
	}

	setupNetworking(serviceConfig.Net, &result)
	setupVolumesFrom(serviceConfig.VolumesFrom, &result)

	err = setupBuild(r, name, &result, serviceConfig)

	if result.Labels == nil {
		result.Labels = map[string]interface{}{}
	}

	return result, err
}

func setupNetworking(netMode string, launchConfig *rancherClient.LaunchConfig) {
	if netMode == "" {
		launchConfig.NetworkMode = "managed"
	} else if runconfig.IpcMode(netMode).IsContainer() {
		// For some reason NetworkMode object is gone runconfig, but IpcMode works the same for this
		launchConfig.NetworkMode = "container"
		launchConfig.NetworkLaunchConfig = strings.TrimPrefix(netMode, "container:")
	} else {
		launchConfig.NetworkMode = netMode
	}
}

func setupVolumesFrom(volumesFrom []string, launchConfig *rancherClient.LaunchConfig) {
	launchConfig.DataVolumesFromLaunchConfigs = volumesFrom
}

func setupBuild(r *RancherService, name string, result *rancherClient.LaunchConfig, serviceConfig *project.ServiceConfig) error {
	if serviceConfig.Build != "" {
		result.Build = &rancherClient.DockerBuild{
			Remote:     serviceConfig.Build,
			Dockerfile: serviceConfig.Dockerfile,
		}

		needBuild := true
		for _, remote := range project.ValidRemotes {
			if strings.HasPrefix(serviceConfig.Build, remote) {
				needBuild = false
				break
			}
		}

		if needBuild {
			image, url, err := Upload(r.Context(), name)
			if err != nil {
				return err
			}
			logrus.Infof("Build for %s available at %s", name, url)
			serviceConfig.Build = url

			if serviceConfig.Image == "" {
				serviceConfig.Image = image
			}

			result.Build = &rancherClient.DockerBuild{
				Context:    url,
				Dockerfile: serviceConfig.Dockerfile,
			}
			result.ImageUuid = "docker:" + image
		} else if result.ImageUuid == "" {
			result.ImageUuid = fmt.Sprintf("docker:%s_%s_%d", r.Context().ProjectName, name, time.Now().UnixNano()/int64(time.Millisecond))
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Package env provides a taskcluster-worker plugin that injects environment
// variables into the task environments.
//
// This plugin supports per-task environment variables specified in
// task.payload.env, but also globally configured environment variables, which
// can be used to inject information such as instance type.
//
// Finally, this plugin will inject TASK_ID and RUN_ID as environment variables.
package env

import (
	"strconv"

	schematypes "github.com/taskcluster/go-schematypes"
	"github.com/taskcluster/taskcluster-worker/engines"
	"github.com/taskcluster/taskcluster-worker/plugins"
	"github.com/taskcluster/taskcluster-worker/runtime"
	"github.com/taskcluster/taskcluster-worker/runtime/util"
)

type plugin struct {
	plugins.PluginBase
	extraVars map[string]string
}

type payload struct {
	Env map[string]string `json:"env"`
}

type config struct {
	Extra map[string]string `json:"extra"`
}

type provider struct {
	plugins.PluginProviderBase
}

type taskPlugin struct {
	plugins.TaskPluginBase
	variables map[string]string
}

func init() {
	plugins.Register("env", &provider{})
}

func (p *provider) ConfigSchema() schematypes.Schema {
	return schematypes.Object{
		Properties: schematypes.Properties{
			"extra": schematypes.Map{
				MetaData: schematypes.MetaData{
					Title: "Extra Environment Variables",
					Description: util.Markdown(`
						The 'extra' property holds a mapping from variable name to value.

						These _extra_ environment variables will be injected into all tasks,
						though they can be overwritten on per-task basis using the
						'task.payload.env' property.

						Notice that these overwrite built-in environment variables
						'TASK_ID' and 'RUN_ID' which is also supplied by this plugin.
					`),
				},
				Values: schematypes.String{},
			},
		},
	}
}

func (p *provider) NewPlugin(options plugins.PluginOptions) (plugins.Plugin, error) {
	var c config
	schematypes.MustValidateAndMap(p.ConfigSchema(), options.Config, &c)

	return &plugin{
		extraVars: c.Extra,
	}, nil
}

func (p *plugin) PayloadSchema() schematypes.Object {
	return schematypes.Object{
		Properties: schematypes.Properties{
			"env": schematypes.Map{
				MetaData: schematypes.MetaData{
					Title:       "Environment Variables",
					Description: "Mapping from environment variables to values",
				},
				Values: schematypes.String{},
			},
		},
	}
}

func (p *plugin) NewTaskPlugin(options plugins.TaskPluginOptions) (plugins.TaskPlugin, error) {
	var P payload
	schematypes.MustValidateAndMap(p.PayloadSchema(), options.Payload, &P)

	env := make(map[string]string)

	// Set built-in environment variables
	env["TASK_ID"] = options.TaskContext.TaskID
	env["RUN_ID"] = strconv.Itoa(options.TaskContext.RunID)

	// Set variables as configured globally
	for k, v := range p.extraVars {
		env[k] = v
	}
	// Set variables as configured per-task (overwriting globally config vars)
	for k, v := range P.Env {
		env[k] = v
	}

	return &taskPlugin{
		variables: env,
	}, nil
}

func (p *taskPlugin) BuildSandbox(sandboxBuilder engines.SandboxBuilder) error {
	for k, v := range p.variables {
		err := sandboxBuilder.SetEnvironmentVariable(k, v)

		// We can only return MalFormedPayloadError
		switch err {
		case engines.ErrNamingConflict:
			return runtime.NewMalformedPayloadError("Environment variable ", k, " has already been set.")
		case engines.ErrFeatureNotSupported:
			return runtime.NewMalformedPayloadError(
				"Custom environment variables is not supported in the current configuration of the engine")
		case nil:
			// break
		default:
			return err
		}
	}

	return nil
}
<commit_msg>Fixed grammar credits @walac<commit_after>// Package env provides a taskcluster-worker plugin that injects environment
// variables into the task environments.
//
// This plugin supports per-task environment variables specified in
// task.payload.env, but also globally configured environment variables, which
// can be used to inject information such as instance type.
//
// Finally, this plugin will inject TASK_ID and RUN_ID as environment variables.
package env

import (
	"strconv"

	schematypes "github.com/taskcluster/go-schematypes"
	"github.com/taskcluster/taskcluster-worker/engines"
	"github.com/taskcluster/taskcluster-worker/plugins"
	"github.com/taskcluster/taskcluster-worker/runtime"
	"github.com/taskcluster/taskcluster-worker/runtime/util"
)

type plugin struct {
	plugins.PluginBase
	extraVars map[string]string
}

type payload struct {
	Env map[string]string `json:"env"`
}

type config struct {
	Extra map[string]string `json:"extra"`
}

type provider struct {
	plugins.PluginProviderBase
}

type taskPlugin struct {
	plugins.TaskPluginBase
	variables map[string]string
}

func init() {
	plugins.Register("env", &provider{})
}

func (p *provider) ConfigSchema() schematypes.Schema {
	return schematypes.Object{
		Properties: schematypes.Properties{
			"extra": schematypes.Map{
				MetaData: schematypes.MetaData{
					Title: "Extra Environment Variables",
					Description: util.Markdown(`
						The 'extra' property holds a mapping from variable name to value.

						These _extra_ environment variables will be injected into all tasks,
						though they can be overwritten on per-task basis using the
						'task.payload.env' property.

						Notice that these overwrite built-in environment variables
						'TASK_ID' and 'RUN_ID' which is also supplied by this plugin.
					`),
				},
				Values: schematypes.String{},
			},
		},
	}
}

func (p *provider) NewPlugin(options plugins.PluginOptions) (plugins.Plugin, error) {
	var c config
	schematypes.MustValidateAndMap(p.ConfigSchema(), options.Config, &c)

	return &plugin{
		extraVars: c.Extra,
	}, nil
}

func (p *plugin) PayloadSchema() schematypes.Object {
	return schematypes.Object{
		Properties: schematypes.Properties{
			"env": schematypes.Map{
				MetaData: schematypes.MetaData{
					Title:       "Environment Variables",
					Description: "Mapping from environment variables to values",
				},
				Values: schematypes.String{},
			},
		},
	}
}

func (p *plugin) NewTaskPlugin(options plugins.TaskPluginOptions) (plugins.TaskPlugin, error) {
	var P payload
	schematypes.MustValidateAndMap(p.PayloadSchema(), options.Payload, &P)

	env := make(map[string]string)

	// Set built-in environment variables
	env["TASK_ID"] = options.TaskContext.TaskID
	env["RUN_ID"] = strconv.Itoa(options.TaskContext.RunID)

	// Set variables as configured globally
	for k, v := range p.extraVars {
		env[k] = v
	}
	// Set variables as configured per-task (overwriting globally config vars)
	for k, v := range P.Env {
		env[k] = v
	}

	return &taskPlugin{
		variables: env,
	}, nil
}

func (p *taskPlugin) BuildSandbox(sandboxBuilder engines.SandboxBuilder) error {
	for k, v := range p.variables {
		err := sandboxBuilder.SetEnvironmentVariable(k, v)

		// We can only return MalFormedPayloadError
		switch err {
		case engines.ErrNamingConflict:
			return runtime.NewMalformedPayloadError("Environment variable ", k, " has already been set.")
		case engines.ErrFeatureNotSupported:
			return runtime.NewMalformedPayloadError(
				"Custom environment variables are not supported in the current configuration of the engine")
		case nil:
			// break
		default:
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>/*
   Copyright (c) 2015, Mark Bucciarelli <mkbucc@gmail.com>
*/

package vufs

import (
	"fmt"
	. "github.com/smartystreets/goconvey/convey"
	"io/ioutil"
	"os"
	"testing"

	"9fans.net/go/plan9"
	"9fans.net/go/plan9/client"
)

const (
	port               = ":5000"
	messageSizeInBytes = 8192
)

// Initialize file system as:
//
//          /
//           |
//           +-- adm/            --rwx------ adm adm
//                   |
//                   +-- users     --rw------- adm adm
//
//         Notes:
//
//          a.    Users shown are virtual ones, not ones on disk.
//
//          b.    If no ownership specified (in .uidgid), it defaults to adm adm.
//
//
func initfs(rootdir string, mode os.FileMode, userdata string) {
	os.Mkdir(rootdir, mode)
	os.Mkdir(rootdir+"/adm", 0700)
	ioutil.WriteFile(rootdir+"/adm/users", []byte(userdata), 0600)
}

func runserver(rootdir, port string) *client.Conn {

	var err error
	fs := New(rootdir)
	fs.Id = "vufs"
	fs.Upool, err = NewVusers(rootdir)
	if err != nil {
		panic(err)
	}
	//fs.Debuglevel = 5

	fs.Start(fs)

	go func() {
		err = fs.StartNetListener("tcp", port)
		if err != nil {
			panic(err)
		}
	}()

	// Make sure runserver is listening before returning.
	var conn *client.Conn
	for i := 0; i < 16; i++ {
		if conn, err = client.Dial("tcp", port); err == nil {
			fmt.Printf("Server is up, got connnection %+v\n", conn)
			break
		}

	}

	if err != nil {
		panic("couldn't connect to runserver after 15 tries: " + err.Error())
	}

	return conn
}

func listDir(conn *client.Conn, path, user string) ([]*plan9.Dir, error) {

	fsys, err := conn.Attach(nil, user, "/")
	if err != nil {
		return nil, err
	}

	// file modes: ../../lionkov/go9p/p/p9.go:65,74
	fid, err := fsys.Open("/", plan9.OREAD)
	if err != nil {
		return nil, err
	}
	defer fid.Close()

	// returns an array of Dir instances: ../../lionkov/go9p/p/clnt/read.go:88
	d, err := fid.Dirreadall()
	if err != nil {
		return nil, err
	}

	return d, nil

}

func TestServer(t *testing.T) {

	rootdir := "./tmpfs"

	initfs(rootdir, 0755, "1:adm:adm\n2:mark:mark\n")

	conn := runserver(rootdir, port)

	Convey("Given a vufs rooted in a directory and a client", t, func() {

		var dirs []*plan9.Dir
		var err error

		Convey("/adm/users is 0600 adm, adm", func() {
			dirs, err = listDir(conn, "/", "adm")
			So(err, ShouldBeNil)
			So(len(dirs), ShouldEqual, 1)
		})

		Convey("A valid user can list the one file in a 0755 root directory", func() {
			os.Chmod(rootdir, 0755)
			dirs, err = listDir(conn, "/", "mark")
			So(err, ShouldBeNil)
			So(len(dirs), ShouldEqual, 1)
		})

		Convey("An invalid user cannot list root directory", func() {
			os.Chmod(rootdir, 0777)
			_, err = listDir(conn, ".", "hugo")
			So(err.Error(), ShouldEqual, "unknown user: 22")
		})

		Convey("A valid user without permissions cannot list files", func() {
			err = os.Chmod(rootdir, 0700)
			So(err, ShouldBeNil)
			dirs, err = listDir(conn, ".", "mark")
			So(err.Error(), ShouldEqual, "permission denied: 1")
		})

	})

	conn.Close()

	os.RemoveAll(rootdir)

}
<commit_msg>zap stale comments<commit_after>/*
   Copyright (c) 2015, Mark Bucciarelli <mkbucc@gmail.com>
*/

package vufs

import (
	"fmt"
	. "github.com/smartystreets/goconvey/convey"
	"io/ioutil"
	"os"
	"testing"

	"9fans.net/go/plan9"
	"9fans.net/go/plan9/client"
)

const (
	port               = ":5000"
	messageSizeInBytes = 8192
)

// Initialize file system as:
//
//          /
//           |
//           +-- adm/            --rwx------ adm adm
//                   |
//                   +-- users     --rw------- adm adm
//
//         Notes:
//
//          a.    Users shown are virtual ones, not ones on disk.
//
//          b.    If no ownership specified (in .uidgid), it defaults to adm adm.
//
//
func initfs(rootdir string, mode os.FileMode, userdata string) {
	os.Mkdir(rootdir, mode)
	os.Mkdir(rootdir+"/adm", 0700)
	ioutil.WriteFile(rootdir+"/adm/users", []byte(userdata), 0600)
}

func runserver(rootdir, port string) *client.Conn {

	var err error
	fs := New(rootdir)
	fs.Id = "vufs"
	fs.Upool, err = NewVusers(rootdir)
	if err != nil {
		panic(err)
	}
	//fs.Debuglevel = 5

	fs.Start(fs)

	go func() {
		err = fs.StartNetListener("tcp", port)
		if err != nil {
			panic(err)
		}
	}()

	// Make sure runserver is listening before returning.
	var conn *client.Conn
	for i := 0; i < 16; i++ {
		if conn, err = client.Dial("tcp", port); err == nil {
			fmt.Printf("Server is up, got connnection %+v\n", conn)
			break
		}

	}

	if err != nil {
		panic("couldn't connect to runserver after 15 tries: " + err.Error())
	}

	return conn
}

func listDir(conn *client.Conn, path, user string) ([]*plan9.Dir, error) {

	fsys, err := conn.Attach(nil, user, "/")
	if err != nil {
		return nil, err
	}

	fid, err := fsys.Open("/", plan9.OREAD)
	if err != nil {
		return nil, err
	}
	defer fid.Close()

	d, err := fid.Dirreadall()
	if err != nil {
		return nil, err
	}

	return d, nil

}

func TestServer(t *testing.T) {

	rootdir := "./tmpfs"

	initfs(rootdir, 0755, "1:adm:adm\n2:mark:mark\n")

	conn := runserver(rootdir, port)

	Convey("Given a vufs rooted in a directory and a client", t, func() {

		var dirs []*plan9.Dir
		var err error

		Convey("/adm/users is 0600 adm, adm", func() {
			dirs, err = listDir(conn, "/", "adm")
			So(err, ShouldBeNil)
			So(len(dirs), ShouldEqual, 1)
		})

		Convey("A valid user can list the one file in a 0755 root directory", func() {
			os.Chmod(rootdir, 0755)
			dirs, err = listDir(conn, "/", "mark")
			So(err, ShouldBeNil)
			So(len(dirs), ShouldEqual, 1)
		})

		Convey("An invalid user cannot list root directory", func() {
			os.Chmod(rootdir, 0777)
			_, err = listDir(conn, ".", "hugo")
			So(err.Error(), ShouldEqual, "unknown user: 22")
		})

		Convey("A valid user without permissions cannot list files", func() {
			err = os.Chmod(rootdir, 0700)
			So(err, ShouldBeNil)
			dirs, err = listDir(conn, ".", "mark")
			So(err.Error(), ShouldEqual, "permission denied: 1")
		})

	})

	conn.Close()

	os.RemoveAll(rootdir)

}
<|endoftext|>


<commit_before>package plugin

import (
	"os"
	"os/exec"

	hclog "github.com/hashicorp/go-hclog"
	plugin "github.com/hashicorp/go-plugin"
	"github.com/hashicorp/terraform/plugin/discovery"
)

// ClientConfig returns a configuration object that can be used to instantiate
// a client for the plugin described by the given metadata.
func ClientConfig(m discovery.PluginMeta) *plugin.ClientConfig {
	logger := hclog.New(&hclog.LoggerOptions{
		Name:   "plugin",
		Level:  hclog.Trace,
		Output: os.Stderr,
	})

	return &plugin.ClientConfig{
		Cmd:              exec.Command(m.Path),
		HandshakeConfig:  Handshake,
		VersionedPlugins: VersionedPlugins,
		Managed:          true,
		Logger:           logger,
		AllowedProtocols: []plugin.Protocol{plugin.ProtocolGRPC},
	}
}

// Client returns a plugin client for the plugin described by the given metadata.
func Client(m discovery.PluginMeta) *plugin.Client {
	return plugin.NewClient(ClientConfig(m))
}
<commit_msg>enable Auto mTLS in go-plugin<commit_after>package plugin

import (
	"os"
	"os/exec"

	hclog "github.com/hashicorp/go-hclog"
	plugin "github.com/hashicorp/go-plugin"
	"github.com/hashicorp/terraform/plugin/discovery"
)

// ClientConfig returns a configuration object that can be used to instantiate
// a client for the plugin described by the given metadata.
func ClientConfig(m discovery.PluginMeta) *plugin.ClientConfig {
	logger := hclog.New(&hclog.LoggerOptions{
		Name:   "plugin",
		Level:  hclog.Trace,
		Output: os.Stderr,
	})

	return &plugin.ClientConfig{
		Cmd:              exec.Command(m.Path),
		HandshakeConfig:  Handshake,
		VersionedPlugins: VersionedPlugins,
		Managed:          true,
		Logger:           logger,
		AllowedProtocols: []plugin.Protocol{plugin.ProtocolGRPC},
		AutoMTLS:         true,
	}
}

// Client returns a plugin client for the plugin described by the given metadata.
func Client(m discovery.PluginMeta) *plugin.Client {
	return plugin.NewClient(ClientConfig(m))
}
<|endoftext|>


<commit_before>package plugin

import (
	"fmt"
	"sync"

	"github.com/pkg/errors"
	"google.golang.org/grpc"
)

var (
	// ErrNoType is returned when no type is specified
	ErrNoType = errors.New("plugin: no type")
	// ErrNoPluginID is returned when no id is specified
	ErrNoPluginID = errors.New("plugin: no id")

	// ErrSkipPlugin is used when a plugin is not initialized and should not be loaded,
	// this allows the plugin loader differentiate between a plugin which is configured
	// not to load and one that fails to load.
	ErrSkipPlugin = errors.New("skip plugin")
)

// IsSkipPlugin returns true if the error is skipping the plugin
func IsSkipPlugin(err error) bool {
	if errors.Cause(err) == ErrSkipPlugin {
		return true
	}
	return false
}

// Type is the type of the plugin
type Type string

const (
	// RuntimePlugin implements a runtime
	RuntimePlugin Type = "io.containerd.runtime.v1"
	// GRPCPlugin implements a grpc service
	GRPCPlugin Type = "io.containerd.grpc.v1"
	// SnapshotPlugin implements a snapshotter
	SnapshotPlugin Type = "io.containerd.snapshotter.v1"
	// TaskMonitorPlugin implements a task monitor
	TaskMonitorPlugin Type = "io.containerd.monitor.v1"
	// DiffPlugin implements a differ
	DiffPlugin Type = "io.containerd.differ.v1"
	// MetadataPlugin implements a metadata store
	MetadataPlugin Type = "io.containerd.metadata.v1"
	// ContentPlugin implements a content store
	ContentPlugin Type = "io.containerd.content.v1"
)

// Registration contains information for registering a plugin
type Registration struct {
	Type     Type
	ID       string
	Config   interface{}
	Requires []Type
	Init     func(*InitContext) (interface{}, error)

	added bool
}

// URI returns the full plugin URI
func (r *Registration) URI() string {
	return fmt.Sprintf("%s.%s", r.Type, r.ID)
}

// Service allows GRPC services to be registered with the underlying server
type Service interface {
	Register(*grpc.Server) error
}

var register = struct {
	sync.Mutex
	r []*Registration
}{}

// Load loads all plugins at the provided path into containerd
func Load(path string) (err error) {
	defer func() {
		if v := recover(); v != nil {
			rerr, ok := v.(error)
			if !ok {
				rerr = fmt.Errorf("%s", v)
			}
			err = rerr
		}
	}()
	return loadPlugins(path)
}

// Register allows plugins to register
func Register(r *Registration) {
	register.Lock()
	defer register.Unlock()
	if r.Type == "" {
		panic(ErrNoType)
	}
	if r.ID == "" {
		panic(ErrNoPluginID)
	}
	register.r = append(register.r, r)
}

// Graph returns an ordered list of registered plugins for initialization
func Graph() (ordered []*Registration) {
	for _, r := range register.r {
		children(r.Requires, &ordered)
		if !r.added {
			ordered = append(ordered, r)
			r.added = true
		}
	}
	return ordered
}

func children(types []Type, ordered *[]*Registration) {
	for _, t := range types {
		for _, r := range register.r {
			if r.Type == t {
				children(r.Requires, ordered)
				if !r.added {
					*ordered = append(*ordered, r)
					r.added = true
				}
			}
		}
	}
}
<commit_msg>plugin: allow querying plugin graph to be re-entrant<commit_after>package plugin

import (
	"fmt"
	"sync"

	"github.com/pkg/errors"
	"google.golang.org/grpc"
)

var (
	// ErrNoType is returned when no type is specified
	ErrNoType = errors.New("plugin: no type")
	// ErrNoPluginID is returned when no id is specified
	ErrNoPluginID = errors.New("plugin: no id")

	// ErrSkipPlugin is used when a plugin is not initialized and should not be loaded,
	// this allows the plugin loader differentiate between a plugin which is configured
	// not to load and one that fails to load.
	ErrSkipPlugin = errors.New("skip plugin")
)

// IsSkipPlugin returns true if the error is skipping the plugin
func IsSkipPlugin(err error) bool {
	if errors.Cause(err) == ErrSkipPlugin {
		return true
	}
	return false
}

// Type is the type of the plugin
type Type string

const (
	// RuntimePlugin implements a runtime
	RuntimePlugin Type = "io.containerd.runtime.v1"
	// GRPCPlugin implements a grpc service
	GRPCPlugin Type = "io.containerd.grpc.v1"
	// SnapshotPlugin implements a snapshotter
	SnapshotPlugin Type = "io.containerd.snapshotter.v1"
	// TaskMonitorPlugin implements a task monitor
	TaskMonitorPlugin Type = "io.containerd.monitor.v1"
	// DiffPlugin implements a differ
	DiffPlugin Type = "io.containerd.differ.v1"
	// MetadataPlugin implements a metadata store
	MetadataPlugin Type = "io.containerd.metadata.v1"
	// ContentPlugin implements a content store
	ContentPlugin Type = "io.containerd.content.v1"
)

// Registration contains information for registering a plugin
type Registration struct {
	Type     Type
	ID       string
	Config   interface{}
	Requires []Type
	Init     func(*InitContext) (interface{}, error)

	added bool
}

// URI returns the full plugin URI
func (r *Registration) URI() string {
	return fmt.Sprintf("%s.%s", r.Type, r.ID)
}

// Service allows GRPC services to be registered with the underlying server
type Service interface {
	Register(*grpc.Server) error
}

var register = struct {
	sync.RWMutex
	r []*Registration
}{}

// Load loads all plugins at the provided path into containerd
func Load(path string) (err error) {
	defer func() {
		if v := recover(); v != nil {
			rerr, ok := v.(error)
			if !ok {
				rerr = fmt.Errorf("%s", v)
			}
			err = rerr
		}
	}()
	return loadPlugins(path)
}

// Register allows plugins to register
func Register(r *Registration) {
	register.Lock()
	defer register.Unlock()
	if r.Type == "" {
		panic(ErrNoType)
	}
	if r.ID == "" {
		panic(ErrNoPluginID)
	}
	register.r = append(register.r, r)
}

// Graph returns an ordered list of registered plugins for initialization
func Graph() (ordered []*Registration) {
	register.RLock()
	defer register.RUnlock()

	added := map[*Registration]bool{}
	for _, r := range register.r {
		children(r.Requires, added, &ordered)
		if !added[r] {
			ordered = append(ordered, r)
			added[r] = true
		}
	}
	return ordered
}

func children(types []Type, added map[*Registration]bool, ordered *[]*Registration) {
	for _, t := range types {
		for _, r := range register.r {
			if r.Type == t {
				children(r.Requires, added, ordered)
				if !added[r] {
					*ordered = append(*ordered, r)
					added[r] = true
				}
			}
		}
	}
}
<|endoftext|>