<commit_before>// Copyright 2014 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package app

import (
	"fmt"
	"io"
	"time"

	"github.com/tsuru/go-gandalfclient"
	"github.com/tsuru/tsuru/action"
	"github.com/tsuru/tsuru/db"
	"github.com/tsuru/tsuru/provision"
	"github.com/tsuru/tsuru/repository"
	"github.com/tsuru/tsuru/service"
	"gopkg.in/mgo.v2/bson"
)

type deploy struct {
	ID        bson.ObjectId `bson:"_id,omitempty"`
	App       string
	Timestamp time.Time
	Duration  time.Duration
	Commit    string
	Error     string
}

func (app *App) ListDeploys() ([]deploy, error) {
	return listDeploys(app, nil)
}

func ListDeploys(s *service.Service) ([]deploy, error) {
	return listDeploys(nil, s)
}

func listDeploys(app *App, s *service.Service) ([]deploy, error) {
	var list []deploy
	conn, err := db.Conn()
	if err != nil {
		return nil, err
	}
	defer conn.Close()
	var qr bson.M
	if app != nil {
		qr = bson.M{"app": app.Name}
	}
	if s != nil {
		var instances []service.ServiceInstance
		q := bson.M{"service_name": s.Name}
		err = conn.ServiceInstances().Find(q).All(&instances)
		if err != nil {
			return nil, err
		}
		var appNames []string
		for _, instance := range instances {
			for _, apps := range instance.Apps {
				appNames = append(appNames, apps)
			}
		}
		qr = bson.M{"app": bson.M{"$in": appNames}}
	}
	if err := conn.Deploys().Find(qr).Sort("-timestamp").All(&list); err != nil {
		return nil, err
	}
	return list, err
}

func GetDeploy(id string) (*deploy, error) {
	var dep deploy
	conn, err := db.Conn()
	if err != nil {
		return nil, err
	}
	defer conn.Close()
	if err := conn.Deploys().FindId(bson.ObjectIdHex(id)).One(&dep); err != nil {
		return nil, err
	}
	return &dep, nil
}

func GetDiffInDeploys(d *deploy) (string, error) {
	var list []deploy
	conn, err := db.Conn()
	if err != nil {
		return "", err
	}
	defer conn.Close()
	if err := conn.Deploys().Find(bson.M{"app": d.App, "_id": bson.M{"$lte": d.ID}}).Sort("-timestamp").All(&list); err != nil {
		return "", err
	}
	if len(list) < 2 {
		return "The deployment must have at least two commits for the diff.", nil
	}
	gandalfClient := gandalf.Client{Endpoint: repository.ServerURL()}
	diffOutput, err := gandalfClient.GetDiff(d.App, list[1].Commit, list[0].Commit)
	if err != nil {
		return "", fmt.Errorf("Caught error getting repository metadata: %s", err.Error())
	}
	return diffOutput, nil
}

type DeployOptions struct {
	App          *App
	Version      string
	Commit       string
	ArchiveURL   string
	File         io.ReadCloser
	OutputStream io.Writer
}

// Deploy runs a deployment of an application. It will first try to run an
// archive based deploy (if opts.ArchiveURL is not empty), and then fallback to
// the Git based deployment.
func Deploy(opts DeployOptions) error {
	var pipeline *action.Pipeline
	start := time.Now()
	if cprovisioner, ok := Provisioner.(provision.CustomizedDeployPipelineProvisioner); ok {
		pipeline = cprovisioner.DeployPipeline()
	} else {
		actions := []*action.Action{&ProvisionerDeploy, &IncrementDeploy}
		pipeline = action.NewPipeline(actions...)
	}
	logWriter := LogWriter{App: opts.App, Writer: opts.OutputStream}
	err := pipeline.Execute(opts, &logWriter)
	elapsed := time.Since(start)
	if err != nil {
		saveDeployData(opts.App.Name, opts.Commit, elapsed, err)
		return err
	}
	if opts.App.UpdatePlatform == true {
		opts.App.SetUpdatePlatform(false)
	}
	return saveDeployData(opts.App.Name, opts.Commit, elapsed, nil)
}

func saveDeployData(appName, commit string, duration time.Duration, deployError error) error {
	conn, err := db.Conn()
	if err != nil {
		return err
	}
	defer conn.Close()
	deploy := deploy{
		App:       appName,
		Timestamp: time.Now(),
		Duration:  duration,
		Commit:    commit,
	}
	if deployError != nil {
		deploy.Error = deployError.Error()
	}
	return conn.Deploys().Insert(deploy)
}

func incrementDeploy(app *App) error {
	conn, err := db.Conn()
	if err != nil {
		return err
	}
	defer conn.Close()
	return conn.Apps().Update(
		bson.M{"name": app.Name},
		bson.M{"$inc": bson.M{"deploys": 1}},
	)
}
<commit_msg>app/deploy: improving deploys query.<commit_after>// Copyright 2014 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package app

import (
	"fmt"
	"io"
	"time"

	"github.com/tsuru/go-gandalfclient"
	"github.com/tsuru/tsuru/action"
	"github.com/tsuru/tsuru/db"
	"github.com/tsuru/tsuru/provision"
	"github.com/tsuru/tsuru/repository"
	"github.com/tsuru/tsuru/service"
	"gopkg.in/mgo.v2/bson"
)

type deploy struct {
	ID        bson.ObjectId `bson:"_id,omitempty"`
	App       string
	Timestamp time.Time
	Duration  time.Duration
	Commit    string
	Error     string
}

func (app *App) ListDeploys() ([]deploy, error) {
	return listDeploys(app, nil)
}

func ListDeploys(s *service.Service) ([]deploy, error) {
	return listDeploys(nil, s)
}

func listDeploys(app *App, s *service.Service) ([]deploy, error) {
	var list []deploy
	conn, err := db.Conn()
	if err != nil {
		return nil, err
	}
	defer conn.Close()
	var qr bson.M
	if app != nil {
		qr = bson.M{"app": app.Name}
	}
	if s != nil {
		var instances []service.ServiceInstance
		q := bson.M{"service_name": s.Name}
		err = conn.ServiceInstances().Find(q).All(&instances)
		if err != nil {
			return nil, err
		}
		var appNames []string
		for _, instance := range instances {
			for _, apps := range instance.Apps {
				appNames = append(appNames, apps)
			}
		}
		qr = bson.M{"app": bson.M{"$in": appNames}}
	}
	if err := conn.Deploys().Find(qr).Sort("-timestamp").All(&list); err != nil {
		return nil, err
	}
	return list, err
}

func GetDeploy(id string) (*deploy, error) {
	var dep deploy
	conn, err := db.Conn()
	if err != nil {
		return nil, err
	}
	defer conn.Close()
	if err := conn.Deploys().FindId(bson.ObjectIdHex(id)).One(&dep); err != nil {
		return nil, err
	}
	return &dep, nil
}

func GetDiffInDeploys(d *deploy) (string, error) {
	var list []deploy
	conn, err := db.Conn()
	if err != nil {
		return "", err
	}
	defer conn.Close()
	if err := conn.Deploys().Find(bson.M{"app": d.App, "_id": bson.M{"$lte": d.ID}}).Sort("-timestamp").Limit(2).All(&list); err != nil {
		return "", err
	}
	if len(list) < 2 {
		return "The deployment must have at least two commits for the diff.", nil
	}
	gandalfClient := gandalf.Client{Endpoint: repository.ServerURL()}
	diffOutput, err := gandalfClient.GetDiff(d.App, list[1].Commit, list[0].Commit)
	if err != nil {
		return "", fmt.Errorf("Caught error getting repository metadata: %s", err.Error())
	}
	return diffOutput, nil
}

type DeployOptions struct {
	App          *App
	Version      string
	Commit       string
	ArchiveURL   string
	File         io.ReadCloser
	OutputStream io.Writer
}

// Deploy runs a deployment of an application. It will first try to run an
// archive based deploy (if opts.ArchiveURL is not empty), and then fallback to
// the Git based deployment.
func Deploy(opts DeployOptions) error {
	var pipeline *action.Pipeline
	start := time.Now()
	if cprovisioner, ok := Provisioner.(provision.CustomizedDeployPipelineProvisioner); ok {
		pipeline = cprovisioner.DeployPipeline()
	} else {
		actions := []*action.Action{&ProvisionerDeploy, &IncrementDeploy}
		pipeline = action.NewPipeline(actions...)
	}
	logWriter := LogWriter{App: opts.App, Writer: opts.OutputStream}
	err := pipeline.Execute(opts, &logWriter)
	elapsed := time.Since(start)
	if err != nil {
		saveDeployData(opts.App.Name, opts.Commit, elapsed, err)
		return err
	}
	if opts.App.UpdatePlatform == true {
		opts.App.SetUpdatePlatform(false)
	}
	return saveDeployData(opts.App.Name, opts.Commit, elapsed, nil)
}

func saveDeployData(appName, commit string, duration time.Duration, deployError error) error {
	conn, err := db.Conn()
	if err != nil {
		return err
	}
	defer conn.Close()
	deploy := deploy{
		App:       appName,
		Timestamp: time.Now(),
		Duration:  duration,
		Commit:    commit,
	}
	if deployError != nil {
		deploy.Error = deployError.Error()
	}
	return conn.Deploys().Insert(deploy)
}

func incrementDeploy(app *App) error {
	conn, err := db.Conn()
	if err != nil {
		return err
	}
	defer conn.Close()
	return conn.Apps().Update(
		bson.M{"name": app.Name},
		bson.M{"$inc": bson.M{"deploys": 1}},
	)
}
<|endoftext|>


<commit_before>package githop

import (
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/google/go-github/github"
)

const (
	CommitDisplayDateFormat        = "3:04pm"
	CommitDisplayDateTooltipFormat = "Monday January 2 3:04pm"
	DigestDisplayDateFormat        = "January 2, 2006 was a Monday"
)

type DigestCommit struct {
	DisplaySHA       string
	URL              string
	Title            string
	Message          string
	PushDate         time.Time
	CommitDate       time.Time
	RepositoryCommit *github.RepositoryCommit
}

func newDigestCommit(commit *github.RepositoryCommit, repo *github.Repository, location *time.Location) DigestCommit {
	messagePieces := strings.SplitN(*commit.Commit.Message, "\n", 2)
	title := messagePieces[0]
	message := ""
	if len(messagePieces) == 2 {
		message = messagePieces[1]
	}
	return DigestCommit{
		DisplaySHA:       (*commit.SHA)[:7],
		URL:              fmt.Sprintf("https://github.com/%s/commit/%s", *repo.FullName, *commit.SHA),
		Title:            title,
		Message:          message,
		PushDate:         commit.Commit.Committer.Date.In(location),
		CommitDate:       commit.Commit.Author.Date.In(location),
		RepositoryCommit: commit,
	}
}

func (commit DigestCommit) DisplayDate() string {
	// Prefer the date the comit was pushed, since that's what GitHub filters
	// and sorts by.
	return commit.PushDate.Format(CommitDisplayDateFormat)
}

func (commit DigestCommit) DisplayDateTooltip() string {
	// But show the full details in a tooltip
	return fmt.Sprintf(
		"Pushed at %s\nCommited at %s",
		commit.PushDate.Format(CommitDisplayDateTooltipFormat),
		commit.CommitDate.Format(CommitDisplayDateTooltipFormat))
}

type RepoDigest struct {
	Repo    *github.Repository
	Commits []DigestCommit
}

// sort.Interface implementation for sorting RepoDigests.
type ByRepoFullName []*RepoDigest

func (a ByRepoFullName) Len() int           { return len(a) }
func (a ByRepoFullName) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByRepoFullName) Less(i, j int) bool { return *a[i].Repo.FullName < *a[j].Repo.FullName }

type IntervalDigest struct {
	yearDelta   int
	StartTime   time.Time
	EndTime     time.Time
	RepoDigests []*RepoDigest
	repos       []github.Repository
}

func (digest *IntervalDigest) Empty() bool {
	for i := range digest.RepoDigests {
		if len(digest.RepoDigests[i].Commits) > 0 {
			return false
		}
	}
	return true
}

func (digest *IntervalDigest) Header() string {
	if digest.yearDelta == -1 {
		return "1 Year Ago"
	}
	return fmt.Sprintf("%d Years Ago", -digest.yearDelta)
}

func (digest *IntervalDigest) Description() string {
	commitCount := 0
	for i := range digest.RepoDigests {
		commitCount += len(digest.RepoDigests[i].Commits)
	}
	var formattedCommitCount string
	if commitCount == 0 {
		formattedCommitCount = "no commits"
	} else if commitCount == 1 {
		formattedCommitCount = "1 commit"
	} else {
		formattedCommitCount = fmt.Sprintf("%d commits", commitCount)
	}
	repoCount := len(digest.RepoDigests)
	var formattedRepoCount string
	if repoCount == 1 {
		formattedRepoCount = "1 repository"
	} else {
		formattedRepoCount = fmt.Sprintf("%d repositories", repoCount)
	}
	return fmt.Sprintf("%s. You had %s in %s that day.",
		digest.StartTime.Format(DigestDisplayDateFormat),
		formattedCommitCount,
		formattedRepoCount)
}

type Digest struct {
	User             *github.User
	TimezoneLocation *time.Location
	IntervalDigests  []*IntervalDigest
}

func newDigest(githubClient *github.Client, account *Account) (*Digest, error) {
	user, _, err := githubClient.Users.Get("")
	if err != nil {
		return nil, err
	}

	// The username parameter must be left blank so that we can get all of the
	// repositories the user has access to, not just ones that they own.
	repos, _, err := githubClient.Repositories.List("", nil)
	if err != nil {
		return nil, err
	}

	orgs, _, err := githubClient.Organizations.List("", nil)
	if err != nil {
		return nil, err
	}
	for _, org := range orgs {
		orgRepos, _, err := githubClient.Repositories.ListByOrg(*org.Login, nil)
		if err != nil {
			return nil, err
		}
		newRepos := make([]github.Repository, len(repos)+len(orgRepos))
		copy(newRepos, repos)
		copy(newRepos[len(repos):], orgRepos)
		repos = newRepos
	}

	oldestDigestTime := time.Now().In(account.TimezoneLocation)
	for _, repo := range repos {
		repoTime := repo.CreatedAt.In(account.TimezoneLocation)
		if repoTime.Before(oldestDigestTime) {
			oldestDigestTime = repoTime
		}
	}

	intervalDigests := make([]*IntervalDigest, 0)
	now := time.Now().In(account.TimezoneLocation)
	for yearDelta := -1; ; yearDelta-- {
		digestStartTime := time.Date(now.Year()+yearDelta, now.Month(), now.Day(), 0, 0, 0, 0, now.Location())
		if digestStartTime.Before(oldestDigestTime) {
			break
		}
		digestEndTime := digestStartTime.AddDate(0, 0, 1)

		// Only look at repos that may have activity in the digest interval.
		var intervalRepos []github.Repository
		for _, repo := range repos {
			if repo.CreatedAt.Before(digestEndTime) && repo.PushedAt != nil &&
				repo.PushedAt.After(digestStartTime) {
				intervalRepos = append(intervalRepos, repo)
			}
		}

		intervalDigests = append(intervalDigests, &IntervalDigest{
			yearDelta:   yearDelta,
			repos:       intervalRepos,
			RepoDigests: make([]*RepoDigest, 0, len(intervalRepos)),
			StartTime:   digestStartTime,
			EndTime:     digestEndTime,
		})
	}

	digest := &Digest{
		User:             user,
		TimezoneLocation: account.TimezoneLocation,
		IntervalDigests:  intervalDigests,
	}

	err = digest.fetch(githubClient)
	return digest, err
}

func (digest *Digest) fetch(githubClient *github.Client) error {
	type RepoDigestResponse struct {
		intervalDigest *IntervalDigest
		repoDigest     *RepoDigest
		err            error
	}
	fetchCount := 0
	ch := make(chan *RepoDigestResponse)
	for _, intervalDigest := range digest.IntervalDigests {
		for j := range intervalDigest.repos {
			repo := &intervalDigest.repos[j]
			go func(intervalDigest *IntervalDigest, repo *github.Repository) {
				commits, _, err := githubClient.Repositories.ListCommits(
					*repo.Owner.Login,
					*repo.Name,
					&github.CommitsListOptions{
						Author: *digest.User.Login,
						Since:  intervalDigest.StartTime.UTC(),
						Until:  intervalDigest.EndTime.UTC(),
					})
				if err != nil {
					ch <- &RepoDigestResponse{nil, nil, err}
				} else {
					digestCommits := make([]DigestCommit, len(commits))
					for i := range commits {
						digestCommits[len(commits)-i-1] = newDigestCommit(&commits[i], repo, digest.TimezoneLocation)
					}
					ch <- &RepoDigestResponse{intervalDigest, &RepoDigest{repo, digestCommits}, nil}
				}
			}(intervalDigest, repo)
			fetchCount++
		}
	}
	for i := 0; i < fetchCount; i++ {
		select {
		case r := <-ch:
			if r.err != nil {
				return r.err
			}
			if len(r.repoDigest.Commits) > 0 {
				r.intervalDigest.RepoDigests = append(r.intervalDigest.RepoDigests, r.repoDigest)
			}
		}
	}
	nonEmptyIntervalDigests := make([]*IntervalDigest, 0, len(digest.IntervalDigests))
	for _, intervalDigest := range digest.IntervalDigests {
		if !intervalDigest.Empty() {
			nonEmptyIntervalDigests = append(nonEmptyIntervalDigests, intervalDigest)
			sort.Sort(ByRepoFullName(intervalDigest.RepoDigests))
		}
	}
	digest.IntervalDigests = nonEmptyIntervalDigests
	return nil
}

func (digest *Digest) Empty() bool {
	return len(digest.IntervalDigests) == 0
}
<commit_msg>Store pointers to repository instances.<commit_after>package githop

import (
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/google/go-github/github"
)

const (
	CommitDisplayDateFormat        = "3:04pm"
	CommitDisplayDateTooltipFormat = "Monday January 2 3:04pm"
	DigestDisplayDateFormat        = "January 2, 2006 was a Monday"
)

type DigestCommit struct {
	DisplaySHA       string
	URL              string
	Title            string
	Message          string
	PushDate         time.Time
	CommitDate       time.Time
	RepositoryCommit *github.RepositoryCommit
}

func newDigestCommit(commit *github.RepositoryCommit, repo *github.Repository, location *time.Location) DigestCommit {
	messagePieces := strings.SplitN(*commit.Commit.Message, "\n", 2)
	title := messagePieces[0]
	message := ""
	if len(messagePieces) == 2 {
		message = messagePieces[1]
	}
	return DigestCommit{
		DisplaySHA:       (*commit.SHA)[:7],
		URL:              fmt.Sprintf("https://github.com/%s/commit/%s", *repo.FullName, *commit.SHA),
		Title:            title,
		Message:          message,
		PushDate:         commit.Commit.Committer.Date.In(location),
		CommitDate:       commit.Commit.Author.Date.In(location),
		RepositoryCommit: commit,
	}
}

func (commit DigestCommit) DisplayDate() string {
	// Prefer the date the comit was pushed, since that's what GitHub filters
	// and sorts by.
	return commit.PushDate.Format(CommitDisplayDateFormat)
}

func (commit DigestCommit) DisplayDateTooltip() string {
	// But show the full details in a tooltip
	return fmt.Sprintf(
		"Pushed at %s\nCommited at %s",
		commit.PushDate.Format(CommitDisplayDateTooltipFormat),
		commit.CommitDate.Format(CommitDisplayDateTooltipFormat))
}

type RepoDigest struct {
	Repo    *github.Repository
	Commits []DigestCommit
}

// sort.Interface implementation for sorting RepoDigests.
type ByRepoFullName []*RepoDigest

func (a ByRepoFullName) Len() int           { return len(a) }
func (a ByRepoFullName) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByRepoFullName) Less(i, j int) bool { return *a[i].Repo.FullName < *a[j].Repo.FullName }

type IntervalDigest struct {
	yearDelta   int
	StartTime   time.Time
	EndTime     time.Time
	RepoDigests []*RepoDigest
	repos       []*github.Repository
}

func (digest *IntervalDigest) Empty() bool {
	for i := range digest.RepoDigests {
		if len(digest.RepoDigests[i].Commits) > 0 {
			return false
		}
	}
	return true
}

func (digest *IntervalDigest) Header() string {
	if digest.yearDelta == -1 {
		return "1 Year Ago"
	}
	return fmt.Sprintf("%d Years Ago", -digest.yearDelta)
}

func (digest *IntervalDigest) Description() string {
	commitCount := 0
	for i := range digest.RepoDigests {
		commitCount += len(digest.RepoDigests[i].Commits)
	}
	var formattedCommitCount string
	if commitCount == 0 {
		formattedCommitCount = "no commits"
	} else if commitCount == 1 {
		formattedCommitCount = "1 commit"
	} else {
		formattedCommitCount = fmt.Sprintf("%d commits", commitCount)
	}
	repoCount := len(digest.RepoDigests)
	var formattedRepoCount string
	if repoCount == 1 {
		formattedRepoCount = "1 repository"
	} else {
		formattedRepoCount = fmt.Sprintf("%d repositories", repoCount)
	}
	return fmt.Sprintf("%s. You had %s in %s that day.",
		digest.StartTime.Format(DigestDisplayDateFormat),
		formattedCommitCount,
		formattedRepoCount)
}

type Digest struct {
	User             *github.User
	TimezoneLocation *time.Location
	IntervalDigests  []*IntervalDigest
}

func newDigest(githubClient *github.Client, account *Account) (*Digest, error) {
	user, _, err := githubClient.Users.Get("")
	if err != nil {
		return nil, err
	}

	// The username parameter must be left blank so that we can get all of the
	// repositories the user has access to, not just ones that they own.
	repos, _, err := githubClient.Repositories.List("", nil)
	if err != nil {
		return nil, err
	}

	orgs, _, err := githubClient.Organizations.List("", nil)
	if err != nil {
		return nil, err
	}
	for _, org := range orgs {
		orgRepos, _, err := githubClient.Repositories.ListByOrg(*org.Login, nil)
		if err != nil {
			return nil, err
		}
		newRepos := make([]github.Repository, len(repos)+len(orgRepos))
		copy(newRepos, repos)
		copy(newRepos[len(repos):], orgRepos)
		repos = newRepos
	}

	oldestDigestTime := time.Now().In(account.TimezoneLocation)
	for _, repo := range repos {
		repoTime := repo.CreatedAt.In(account.TimezoneLocation)
		if repoTime.Before(oldestDigestTime) {
			oldestDigestTime = repoTime
		}
	}

	intervalDigests := make([]*IntervalDigest, 0)
	now := time.Now().In(account.TimezoneLocation)
	for yearDelta := -1; ; yearDelta-- {
		digestStartTime := time.Date(now.Year()+yearDelta, now.Month(), now.Day(), 0, 0, 0, 0, now.Location())
		if digestStartTime.Before(oldestDigestTime) {
			break
		}
		digestEndTime := digestStartTime.AddDate(0, 0, 1)

		// Only look at repos that may have activity in the digest interval.
		var intervalRepos []*github.Repository
		for i := range repos {
			repo := &repos[i]
			if repo.CreatedAt.Before(digestEndTime) && repo.PushedAt != nil &&
				repo.PushedAt.After(digestStartTime) {
				intervalRepos = append(intervalRepos, repo)
			}
		}

		intervalDigests = append(intervalDigests, &IntervalDigest{
			yearDelta:   yearDelta,
			repos:       intervalRepos,
			RepoDigests: make([]*RepoDigest, 0, len(intervalRepos)),
			StartTime:   digestStartTime,
			EndTime:     digestEndTime,
		})
	}

	digest := &Digest{
		User:             user,
		TimezoneLocation: account.TimezoneLocation,
		IntervalDigests:  intervalDigests,
	}

	err = digest.fetch(githubClient)
	return digest, err
}

func (digest *Digest) fetch(githubClient *github.Client) error {
	type RepoDigestResponse struct {
		intervalDigest *IntervalDigest
		repoDigest     *RepoDigest
		err            error
	}
	fetchCount := 0
	ch := make(chan *RepoDigestResponse)
	for _, intervalDigest := range digest.IntervalDigests {
		for _, repo := range intervalDigest.repos {
			go func(intervalDigest *IntervalDigest, repo *github.Repository) {
				commits, _, err := githubClient.Repositories.ListCommits(
					*repo.Owner.Login,
					*repo.Name,
					&github.CommitsListOptions{
						Author: *digest.User.Login,
						Since:  intervalDigest.StartTime.UTC(),
						Until:  intervalDigest.EndTime.UTC(),
					})
				if err != nil {
					ch <- &RepoDigestResponse{nil, nil, err}
				} else {
					digestCommits := make([]DigestCommit, len(commits))
					for i := range commits {
						digestCommits[len(commits)-i-1] = newDigestCommit(&commits[i], repo, digest.TimezoneLocation)
					}
					ch <- &RepoDigestResponse{intervalDigest, &RepoDigest{repo, digestCommits}, nil}
				}
			}(intervalDigest, repo)
			fetchCount++
		}
	}
	for i := 0; i < fetchCount; i++ {
		select {
		case r := <-ch:
			if r.err != nil {
				return r.err
			}
			if len(r.repoDigest.Commits) > 0 {
				r.intervalDigest.RepoDigests = append(r.intervalDigest.RepoDigests, r.repoDigest)
			}
		}
	}
	nonEmptyIntervalDigests := make([]*IntervalDigest, 0, len(digest.IntervalDigests))
	for _, intervalDigest := range digest.IntervalDigests {
		if !intervalDigest.Empty() {
			nonEmptyIntervalDigests = append(nonEmptyIntervalDigests, intervalDigest)
			sort.Sort(ByRepoFullName(intervalDigest.RepoDigests))
		}
	}
	digest.IntervalDigests = nonEmptyIntervalDigests
	return nil
}

func (digest *Digest) Empty() bool {
	return len(digest.IntervalDigests) == 0
}
<|endoftext|>


<commit_before>package forecast

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"strconv"
)

// URL example:  "https://api.forecast.io/forecast/APIKEY/LATITUDE,LONGITUDE,TIME?units=ca"
const (
	BASEURL = "https://api.forecast.io/forecast"
)

type Flags struct {
	DarkSkyUnavailable string   `json:"darksky-unavailable"`
	DarkSkyStations    []string `json:"darksky-stations"`
	DataPointStations  []string `json:"datapoint-stations"`
	ISDStations        []string `json:"isds-stations"`
	LAMPStations       []string `json:"lamp-stations"`
	METARStations      []string `json:"metars-stations"`
	METNOLicense       string   `json:"metnol-license"`
	Sources            []string `json:"sources"`
	Units              string   `json:"units"`
}

type DataPoint struct {
	Time                   float64 `json:"time"`
	Summary                string  `json:"summary"`
	Icon                   string  `json:"icon"`
	SunriseTime            float64 `json:"sunriseTime"`
	SunsetTime             float64 `json:"sunsetTime"`
	PrecipIntensity        float64 `json:"precipIntensity"`
	PrecipIntensityMax     float64 `json:"precipIntensityMax"`
	PrecipIntensityMaxTime float64 `json:"precipIntensityMaxTime"`
	PrecipProbability      float64 `json:"precipProbability"`
	PrecipType             string  `json:"precipType"`
	PrecipAccumulation     float64 `json:"precipAccumulation"`
	Temperature            float64 `json:"temperature"`
	TemperatureMin         float64 `json:"temperatureMin"`
	TemperatureMinTime     float64 `json:"temperatureMinTime"`
	TemperatureMax         float64 `json:"temperatureMax"`
	TemperatureMaxTime     float64 `json:"temperatureMaxTime"`
	ApparentTemperature    float64 `json:"apparentTemperature"`
	DewPoint               float64 `json:"dewPoint"`
	WindSpeed              float64 `json:"windSpeed"`
	WindBearing            float64 `json:"windBearing"`
	CloudCover             float64 `json:"cloudCover"`
	Humidity               float64 `json:"humidity"`
	Pressure               float64 `json:"pressure"`
	Visibility             float64 `json:"visibility"`
	Ozone                  float64 `json:"ozone"`
	MoonPhase              float64 `json:"moonPhase"`
}

type DataBlock struct {
	Summary string      `json:"summary"`
	Icon    string      `json:"icon"`
	Data    []DataPoint `json:"data"`
}

type alert struct {
	Title       string  `json:"title"`
	Description string  `json:"description"`
	Time        float64 `json:"time"`
	Expires     float64 `json:"expires"`
	URI         string  `json:"uri"`
}

type Forecast struct {
	Latitude  float64   `json:"latitude"`
	Longitude float64   `json:"longitude"`
	Timezone  string    `json:"timezone"`
	Offset    float64   `json:"offset"`
	Currently DataPoint `json:"currently"`
	Minutely  DataBlock `json:"minutely"`
	Hourly    DataBlock `json:"hourly"`
	Daily     DataBlock `json:"daily"`
	Alerts    []alert   `json:"alerts"`
	Flags     Flags     `json:"flags"`
	APICalls  int       `json:"apicalls"`
	Code      int       `json:"code"`
}

type Units string

const (
	CA   Units = "ca"
	SI   Units = "si"
	US   Units = "us"
	UK   Units = "uk"
	AUTO Units = "auto"
)

func Get(key string, lat string, long string, time string, units Units) (*Forecast, error) {
	res, err := GetResponse(key, lat, long, time, units)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}

	f, err := FromJSON(body)
	if err != nil {
		return nil, err
	}

	calls, _ := strconv.Atoi(res.Header.Get("X-Forecast-API-Calls"))
	f.APICalls = calls

	return f, nil
}

func FromJSON(json_blob []byte) (*Forecast, error) {
	var f Forecast
	err := json.Unmarshal(json_blob, &f)
	if err != nil {
		return nil, err
	}

	return &f, nil
}

func GetResponse(key string, lat string, long string, time string, units Units) (*http.Response, error) {
	coord := lat + "," + long

	var url string
	if time == "now" {
		url = BASEURL + "/" + key + "/" + coord + "?units=" + string(units)
	} else {
		url = BASEURL + "/" + key + "/" + coord + "," + time + "?units=" + string(units)
	}

	res, err := http.Get(url)
	if err != nil {
		return res, err
	}

	return res, nil
}
<commit_msg>Change BASEURL to darksky.net<commit_after>package forecast

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"strconv"
)

// URL example:  "https://api.darksky.net/forecast/APIKEY/LATITUDE,LONGITUDE,TIME?units=ca"
const (
	BASEURL = "https://api.darksky.net/forecast"
)

type Flags struct {
	DarkSkyUnavailable string   `json:"darksky-unavailable"`
	DarkSkyStations    []string `json:"darksky-stations"`
	DataPointStations  []string `json:"datapoint-stations"`
	ISDStations        []string `json:"isds-stations"`
	LAMPStations       []string `json:"lamp-stations"`
	METARStations      []string `json:"metars-stations"`
	METNOLicense       string   `json:"metnol-license"`
	Sources            []string `json:"sources"`
	Units              string   `json:"units"`
}

type DataPoint struct {
	Time                   float64 `json:"time"`
	Summary                string  `json:"summary"`
	Icon                   string  `json:"icon"`
	SunriseTime            float64 `json:"sunriseTime"`
	SunsetTime             float64 `json:"sunsetTime"`
	PrecipIntensity        float64 `json:"precipIntensity"`
	PrecipIntensityMax     float64 `json:"precipIntensityMax"`
	PrecipIntensityMaxTime float64 `json:"precipIntensityMaxTime"`
	PrecipProbability      float64 `json:"precipProbability"`
	PrecipType             string  `json:"precipType"`
	PrecipAccumulation     float64 `json:"precipAccumulation"`
	Temperature            float64 `json:"temperature"`
	TemperatureMin         float64 `json:"temperatureMin"`
	TemperatureMinTime     float64 `json:"temperatureMinTime"`
	TemperatureMax         float64 `json:"temperatureMax"`
	TemperatureMaxTime     float64 `json:"temperatureMaxTime"`
	ApparentTemperature    float64 `json:"apparentTemperature"`
	DewPoint               float64 `json:"dewPoint"`
	WindSpeed              float64 `json:"windSpeed"`
	WindBearing            float64 `json:"windBearing"`
	CloudCover             float64 `json:"cloudCover"`
	Humidity               float64 `json:"humidity"`
	Pressure               float64 `json:"pressure"`
	Visibility             float64 `json:"visibility"`
	Ozone                  float64 `json:"ozone"`
	MoonPhase              float64 `json:"moonPhase"`
}

type DataBlock struct {
	Summary string      `json:"summary"`
	Icon    string      `json:"icon"`
	Data    []DataPoint `json:"data"`
}

type alert struct {
	Title       string  `json:"title"`
	Description string  `json:"description"`
	Time        float64 `json:"time"`
	Expires     float64 `json:"expires"`
	URI         string  `json:"uri"`
}

type Forecast struct {
	Latitude  float64   `json:"latitude"`
	Longitude float64   `json:"longitude"`
	Timezone  string    `json:"timezone"`
	Offset    float64   `json:"offset"`
	Currently DataPoint `json:"currently"`
	Minutely  DataBlock `json:"minutely"`
	Hourly    DataBlock `json:"hourly"`
	Daily     DataBlock `json:"daily"`
	Alerts    []alert   `json:"alerts"`
	Flags     Flags     `json:"flags"`
	APICalls  int       `json:"apicalls"`
	Code      int       `json:"code"`
}

type Units string

const (
	CA   Units = "ca"
	SI   Units = "si"
	US   Units = "us"
	UK   Units = "uk"
	AUTO Units = "auto"
)

func Get(key string, lat string, long string, time string, units Units) (*Forecast, error) {
	res, err := GetResponse(key, lat, long, time, units)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}

	f, err := FromJSON(body)
	if err != nil {
		return nil, err
	}

	calls, _ := strconv.Atoi(res.Header.Get("X-Forecast-API-Calls"))
	f.APICalls = calls

	return f, nil
}

func FromJSON(json_blob []byte) (*Forecast, error) {
	var f Forecast
	err := json.Unmarshal(json_blob, &f)
	if err != nil {
		return nil, err
	}

	return &f, nil
}

func GetResponse(key string, lat string, long string, time string, units Units) (*http.Response, error) {
	coord := lat + "," + long

	var url string
	if time == "now" {
		url = BASEURL + "/" + key + "/" + coord + "?units=" + string(units)
	} else {
		url = BASEURL + "/" + key + "/" + coord + "," + time + "?units=" + string(units)
	}

	res, err := http.Get(url)
	if err != nil {
		return res, err
	}

	return res, nil
}
<|endoftext|>


<commit_before>package main

func (t Type) MakeList() Type {
	t.List = true
	t.User = false
	return t
}

func (t Type) ListType() Type {
	t.List = false
	if t.Int > Something.Int {
		t.User = true
	}
	return t
}

//Add a value to a list of type t.
func (ic *Compiler) PutList(t Type, list string, value string) {
	if ic.ExpressionType.Push != "SHARE" {
		ic.RaiseError("Cannot put numeric value on list.")
	}

	if ic.ExpressionType.Name != t.Name {
		if t.Name == "Something" {
			var tmp = ic.Tmp("something")
			ic.Assembly("ARRAY ", tmp)
			ic.SetVariable(tmp, t)
			ic.AssignSomething(tmp, value)
			ic.SetVariable(tmp, Undefined)
			value = tmp
		} else {
		ic.RaiseError("Type mismatch! Cannot add a ", ic.ExpressionType.Name,
			 " to a List of ", t.Name)
		}
	}

	var tmp = ic.Tmp("index")
	ic.Assembly("SHARE ", value)
	ic.Assembly("PUSH 0")
	ic.Assembly("HEAP")
	ic.Assembly("PULL ", tmp)

	ic.Assembly("PLACE ", list)
	ic.Assembly("PUT ", tmp)
}
<commit_msg>Fix interface lists.<commit_after>package main

func (t Type) MakeList() Type {
	t.List = true
	t.User = false
	return t
}

func (t Type) ListType() Type {
	t.List = false
	if t.Int > Something.Int {
		t.User = true
	}
	return t
}

//Add a value to a list of type t.
func (ic *Compiler) PutList(t Type, list string, value string) {
		if ic.ExpressionType.Name != t.Name {
		if t.Name == "Something" {
			var tmp = ic.Tmp("something")
			ic.Assembly("ARRAY ", tmp)
			ic.SetVariable(tmp, t)
			ic.AssignSomething(tmp, value)
			ic.SetVariable(tmp, Undefined)
			value = tmp
		} else {
		ic.RaiseError("Type mismatch! Cannot add a ", ic.ExpressionType.Name,
			 " to a List of ", t.Name)
		}
	}

	var tmp = ic.Tmp("index")
	ic.Assembly("SHARE ", value)
	ic.Assembly("PUSH 0")
	ic.Assembly("HEAP")
	ic.Assembly("PULL ", tmp)

	ic.Assembly("PLACE ", list)
	ic.Assembly("PUT ", tmp)
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/coreos/fuze/third_party/github.com/coreos/ignition/config"
	"github.com/coreos/fuze/third_party/github.com/go-yaml/yaml"
)

var (
	flagHelp    = flag.Bool("help", false, "print help and exit")
	flagInFile  = flag.String("in-file", "/dev/stdin", "input file (YAML)")
	flagOutFile = flag.String("out-file", "/dev/stdout", "output file (JSON)")
)

func stderr(f string, a ...interface{}) {
	out := fmt.Sprintf(f, a...)
	fmt.Fprintln(os.Stderr, strings.TrimSuffix(out, "\n"))
}

func stdout(f string, a ...interface{}) {
	out := fmt.Sprintf(f, a...)
	fmt.Fprintln(os.Stdout, strings.TrimSuffix(out, "\n"))
}

func panicf(f string, a ...interface{}) {
	panic(fmt.Sprintf(f, a...))
}

func main() {
	flag.Parse()

	if *flagHelp {
		flag.Usage()
		os.Exit(1)
	}

	cfg := config.Config{}
	b, err := ioutil.ReadFile(*flagInFile)
	if err != nil {
		stderr("Failed to read: %v", err)
		os.Exit(2)
	}

	if err := yaml.Unmarshal(b, &cfg); err != nil {
		stderr("Failed to unmarshal input: %v", err)
		os.Exit(3)
	}

	b, err = json.Marshal(&cfg)
	if err != nil {
		stderr("Failed to marshal output: %v", err)
		os.Exit(4)
	}

	if err := ioutil.WriteFile(*flagOutFile, b, 0640); err != nil {
		stderr("Failed to write: %v", err)
		os.Exit(5)
	}

	os.Exit(0)
}
<commit_msg>main: clean up errors and remove unused code<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/coreos/fuze/third_party/github.com/coreos/ignition/config"
	"github.com/coreos/fuze/third_party/github.com/go-yaml/yaml"
)

var (
	flagHelp    = flag.Bool("help", false, "print help and exit")
	flagInFile  = flag.String("in-file", "/dev/stdin", "input file (YAML)")
	flagOutFile = flag.String("out-file", "/dev/stdout", "output file (JSON)")
)

func stderr(f string, a ...interface{}) {
	out := fmt.Sprintf(f, a...)
	fmt.Fprintln(os.Stderr, strings.TrimSuffix(out, "\n"))
}

func main() {
	flag.Parse()

	if *flagHelp {
		flag.Usage()
		return
	}

	cfg := config.Config{}
	dataIn, err := ioutil.ReadFile(*flagInFile)
	if err != nil {
		stderr("Failed to read: %v", err)
		os.Exit(1)
	}

	if err := yaml.Unmarshal(dataIn, &cfg); err != nil {
		stderr("Failed to unmarshal input: %v", err)
		os.Exit(1)
	}

	dataOut, err := json.Marshal(&cfg)
	if err != nil {
		stderr("Failed to marshal output: %v", err)
		os.Exit(1)
	}

	if err := ioutil.WriteFile(*flagOutFile, dataOut, 0640); err != nil {
		stderr("Failed to write: %v", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
)

func main() {
	fmt.Println("start")

	a:=new(int)
	defer func(){
		fmt.Println(*a)
	}()
	for i:=0;i<100;i++{
		*a++
	}
	// var a string
	// fmt.Scan(&a)
}
<commit_msg>update<commit_after><|endoftext|>


<commit_before>package main

import (
  "os"
  "fmt"
  "flag"
  "strings"
  "io/ioutil"
  "time"
  "github.com/vole/web"
  "github.com/nu7hatch/gouuid"
  "encoding/json"
)

/**
 *
 * import "vole/db"
 *
 * db.findPosts(group)
 * db.findUsers(group)
 *
 * db.findUser(group, KEY)
 * db.findPost(group, GUID)
 *
 * db.savePost(group, post)
 *
 */

func getAllPosts(ctx *web.Context) string {
  ctx.SetHeader("Content-Type", "application/json", true)

  posts := make([]string, 0)

  users_info, _ := ioutil.ReadDir("./data/users")

  for _, user_info := range users_info {
    fmt.Println(string(user_info.Name()))
    posts_dir := "./data/users/" + string(user_info.Name()) + "/v1/posts"
    posts_info, _ := ioutil.ReadDir(posts_dir)

    for _, post_info := range posts_info {
      post_path := posts_dir + "/" + string(post_info.Name())
      fmt.Println(post_path)

      post_data, err := ioutil.ReadFile(post_path)
      if err == nil {
        posts = append(posts, string(post_data))
      }
    }
  }

  out := `{ "posts": [`
  out += strings.Join(posts, ",")
  out += `] }`

  return out
}

func savePost(ctx *web.Context) string {
  data, err := ioutil.ReadFile("data/my_user")
  if err != nil {
    ctx.Abort(500, "Couldn't determine current user.")
  }
  //var jsonBlob = []byte(`{"post":{"title":"hello","user":"mark"}}`)

  body, err := ioutil.ReadAll(ctx.Request.Body);

  var post interface{}
  err = json.Unmarshal(body, &post)
  if err != nil {
    fmt.Println("error:", err)
  }

  user := strings.TrimSpace(string(data))

  ts := time.Now().UnixNano()
  uuid, _ := uuid.NewV4()
  filename := fmt.Sprintf("%d-post-%s", ts, uuid)

  file, err := os.Create("data/users/" + user + "/v1/posts/" + filename)
  if err != nil {
    ctx.Abort(500, "Unable to create file.")
  }

  m := post.(map[string]interface{})

  m["post"] = "dkjfbsdjf"

  ohnoes = m.(map[string]interface{})

  lol, _ := json.Marshal(post)
  fmt.Printf("%+v", string(lol))

  file.Write(lol)
  return string(lol)
}

func getMyUser(ctx *web.Context) string {
  ctx.SetHeader("Content-Type", "application/json", true)

  name := "";
  data, err := ioutil.ReadFile("data/my_user")
  if err == nil {
    name = strings.TrimSpace(string(data))
    fmt.Println("found my user: " + name)

    // Load the user's profile json.
    profile := ""
    profile_data, profile_err := ioutil.ReadFile("data/users/" + name + "/v1/user/" + name)

    if profile_err == nil {
      profile = strings.TrimSpace(string(profile_data))
    } else {
      fmt.Println(profile_err)
    }

    return `{ "users": [` + profile + `] }`
  }
  fmt.Println(err)
  return ""
}

func main() {
  var port = flag.String("port", "6789", "Port on which to run the web server.")
  flag.Parse()

  web.Get("/api/posts", getAllPosts)
  web.Get("/api/users", getMyUser)

  web.Post("/api/posts", savePost)

  web.Run("0.0.0.0:" + *port)
}
<commit_msg>Json improvements.<commit_after>package main

import (
  "os"
  "fmt"
  "flag"
  "strings"
  "io/ioutil"
  "time"
  "github.com/vole/web"
  "github.com/nu7hatch/gouuid"
  "encoding/json"
)

/**
 *
 * import "vole/db"
 *
 * db.findPosts(group)
 * db.findUsers(group)
 *
 * db.findUser(group, KEY)
 * db.findPost(group, GUID)
 *
 * db.savePost(group, post)
 *
 */

func getAllPosts(ctx *web.Context) string {
  ctx.SetHeader("Content-Type", "application/json", true)

  posts := make([]string, 0)

  users_info, _ := ioutil.ReadDir("./data/users")

  for _, user_info := range users_info {
    fmt.Println(string(user_info.Name()))
    posts_dir := "./data/users/" + string(user_info.Name()) + "/v1/posts"
    posts_info, _ := ioutil.ReadDir(posts_dir)

    for _, post_info := range posts_info {
      post_path := posts_dir + "/" + string(post_info.Name())
      fmt.Println(post_path)

      post_data, err := ioutil.ReadFile(post_path)
      if err == nil {
        posts = append(posts, string(post_data))
      }
    }
  }

  out := `{ "posts": [`
  out += strings.Join(posts, ",")
  out += `] }`

  return out
}

func savePost(ctx *web.Context) string {
  data, err := ioutil.ReadFile("data/my_user")
  if err != nil {
    ctx.Abort(500, "Couldn't determine current user.")
  }

  body, err := ioutil.ReadAll(ctx.Request.Body);

  var post Post
  json.Unmarshal(body, &post)

  user := strings.TrimSpace(string(data))
  ts := time.Now().UnixNano()
  uuid, _ := uuid.NewV4()

  filename := fmt.Sprintf("%d-post-%s", ts, uuid)
  file, err := os.Create("data/users/" + user + "/v1/posts/" + filename)
  if err != nil {
    ctx.Abort(500, "Unable to create file.")
  }

  post.Post.Uuid = fmt.Sprintf("%s", uuid)

  postBodyJson, _ := json.Marshal(post.Post)
  postJson, _ := json.Marshal(post)

  file.Write(postBodyJson)
  return string(postJson)
}

func getMyUser(ctx *web.Context) string {
  ctx.SetHeader("Content-Type", "application/json", true)

  name := "";
  data, err := ioutil.ReadFile("data/my_user")
  if err == nil {
    name = strings.TrimSpace(string(data))
    fmt.Println("found my user: " + name)

    // Load the user's profile json.
    profile := ""
    profile_data, profile_err := ioutil.ReadFile("data/users/" + name + "/v1/user/" + name)

    if profile_err == nil {
      profile = strings.TrimSpace(string(profile_data))
    } else {
      fmt.Println(profile_err)
    }

    return `{ "users": [` + profile + `] }`
  }
  fmt.Println(err)
  return ""
}

type Post struct {
  Post PostBody `json:"post"`
}

type PostBody struct {
  Uuid string `json:"uuid"`
  Title string `json:"title"`
  User string `json:"user"`
}

func main() {
  var port = flag.String("port", "6789", "Port on which to run the web server.")
  flag.Parse()

  web.Get("/api/posts", getAllPosts)
  web.Get("/api/users", getMyUser)

  web.Post("/api/posts", savePost)

  web.Run("0.0.0.0:" + *port)
}
<|endoftext|>


<commit_before>package integration

import (
	"runtime"
	"testing"

	"github.com/buildkite/bintest"
)

func argumentForCommand(cmd string) interface{} {
	// This is unpleasant, but we have to work around the fact that we generate
	// batch scripts for windows and plain commands for everything else
	if runtime.GOOS == `windows` {
		return bintest.MatchPattern(`buildkite-script-[\d+].bat$`)
	}
	return cmd
}

func TestRunningCommandWithDocker(t *testing.T) {
	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	// Mock out the meta-data calls to the agent after checkout
	agent := tester.MustMock(t, "buildkite-agent")
	agent.
		Expect("meta-data", "exists", "buildkite:git:commit").
		AndExitWith(0)

	env := []string{
		"BUILDKITE_DOCKER=llamas",
	}

	jobId := "1111-1111-1111-1111"
	imageId := "buildkite_" + jobId + "_image"
	containerId := "buildkite_" + jobId + "_container"

	docker := tester.MustMock(t, "docker")
	docker.ExpectAll([][]interface{}{
		{"build", "-f", "Dockerfile", "-t", imageId, "."},
		{"run", "--name", containerId, imageId, argumentForCommand("true")},
		{"rm", "-f", "-v", containerId},
	})

	expectCommandHooks("0", t, tester)

	tester.RunAndCheck(t, env...)
}

func TestRunningCommandWithDockerAndCustomDockerfile(t *testing.T) {
	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	// Mock out the meta-data calls to the agent after checkout
	agent := tester.MustMock(t, "buildkite-agent")
	agent.
		Expect("meta-data", "exists", "buildkite:git:commit").
		AndExitWith(0)

	env := []string{
		"BUILDKITE_DOCKER=llamas",
		"BUILDKITE_DOCKER_FILE=Dockerfile.llamas",
	}

	jobId := "1111-1111-1111-1111"
	imageId := "buildkite_" + jobId + "_image"
	containerId := "buildkite_" + jobId + "_container"

	docker := tester.MustMock(t, "docker")
	docker.ExpectAll([][]interface{}{
		{"build", "-f", "Dockerfile.llamas", "-t", imageId, "."},
		{"run", "--name", containerId, imageId, argumentForCommand("true")},
		{"rm", "-f", "-v", containerId},
	})

	expectCommandHooks("0", t, tester)

	tester.RunAndCheck(t, env...)
}

func TestRunningFailingCommandWithDocker(t *testing.T) {
	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	// Mock out the meta-data calls to the agent after checkout
	agent := tester.MustMock(t, "buildkite-agent")
	agent.
		Expect("meta-data", "exists", "buildkite:git:commit").
		AndExitWith(0)

	env := []string{
		"BUILDKITE_DOCKER=llamas",
	}

	jobId := "1111-1111-1111-1111"
	imageId := "buildkite_" + jobId + "_image"
	containerId := "buildkite_" + jobId + "_container"

	docker := tester.MustMock(t, "docker")
	docker.ExpectAll([][]interface{}{
		{"build", "-f", "Dockerfile", "-t", imageId, "."},
		{"rm", "-f", "-v", containerId},
	})

	docker.Expect("run", "--name", containerId, imageId, argumentForCommand("true")).
		AndExitWith(1)

	expectCommandHooks("1", t, tester)

	if err = tester.Run(t, env...); err == nil {
		t.Fatal("Expected bootstrap to fail")
	}

	tester.CheckMocks(t)
}

func TestRunningCommandWithDockerCompose(t *testing.T) {
	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	// Mock out the meta-data calls to the agent after checkout
	agent := tester.MustMock(t, "buildkite-agent")
	agent.
		Expect("meta-data", "exists", "buildkite:git:commit").
		AndExitWith(0)

	env := []string{
		"BUILDKITE_DOCKER_COMPOSE_CONTAINER=llamas",
	}

	projectName := "buildkite1111111111111111"

	dockerCompose := tester.MustMock(t, "docker-compose")
	dockerCompose.ExpectAll([][]interface{}{
		{"-f", "docker-compose.yml", "-p", projectName, "--verbose", "build", "--pull", "llamas"},
		{"-f", "docker-compose.yml", "-p", projectName, "--verbose", "run", "llamas", argumentForCommand("true")},
		{"-f", "docker-compose.yml", "-p", projectName, "--verbose", "kill"},
		{"-f", "docker-compose.yml", "-p", projectName, "--verbose", "rm", "--force", "--all", "-v"},
		{"-f", "docker-compose.yml", "-p", projectName, "--verbose", "down"},
	})

	expectCommandHooks("0", t, tester)

	tester.RunAndCheck(t, env...)
}

func TestRunningFailingCommandWithDockerCompose(t *testing.T) {
	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	// Mock out the meta-data calls to the agent after checkout
	agent := tester.MustMock(t, "buildkite-agent")
	agent.
		Expect("meta-data", "exists", "buildkite:git:commit").
		AndExitWith(0)

	env := []string{
		"BUILDKITE_DOCKER_COMPOSE_CONTAINER=llamas",
	}

	projectName := "buildkite1111111111111111"

	dockerCompose := tester.MustMock(t, "docker-compose")
	dockerCompose.ExpectAll([][]interface{}{
		{"-f", "docker-compose.yml", "-p", projectName, "--verbose", "build", "--pull", "llamas"},
		{"-f", "docker-compose.yml", "-p", projectName, "--verbose", "kill"},
		{"-f", "docker-compose.yml", "-p", projectName, "--verbose", "rm", "--force", "--all", "-v"},
		{"-f", "docker-compose.yml", "-p", projectName, "--verbose", "down"},
	})

	dockerCompose.Expect("-f", "docker-compose.yml", "-p", projectName, "--verbose", "run", "llamas", argumentForCommand("true")).
		AndWriteToStderr("Nope!").
		AndExitWith(1)

	expectCommandHooks("1", t, tester)

	if err = tester.Run(t, env...); err == nil {
		t.Fatal("Expected bootstrap to fail")
	}

	tester.CheckMocks(t)
}

func TestRunningCommandWithDockerComposeAndExtraConfig(t *testing.T) {
	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	// Mock out the meta-data calls to the agent after checkout
	agent := tester.MustMock(t, "buildkite-agent")
	agent.
		Expect("meta-data", "exists", "buildkite:git:commit").
		AndExitWith(0)

	env := []string{
		"BUILDKITE_DOCKER_COMPOSE_CONTAINER=llamas",
		"BUILDKITE_DOCKER_COMPOSE_FILE=dc1.yml:dc2.yml:dc3.yml",
	}

	projectName := "buildkite1111111111111111"

	dockerCompose := tester.MustMock(t, "docker-compose")
	dockerCompose.ExpectAll([][]interface{}{
		{"-f", "dc1.yml", "-f", "dc2.yml", "-f", "dc3.yml", "-p", projectName, "--verbose", "build", "--pull", "llamas"},
		{"-f", "dc1.yml", "-f", "dc2.yml", "-f", "dc3.yml", "-p", projectName, "--verbose", "run", "llamas", argumentForCommand("true")},
		{"-f", "dc1.yml", "-f", "dc2.yml", "-f", "dc3.yml", "-p", projectName, "--verbose", "kill"},
		{"-f", "dc1.yml", "-f", "dc2.yml", "-f", "dc3.yml", "-p", projectName, "--verbose", "rm", "--force", "--all", "-v"},
		{"-f", "dc1.yml", "-f", "dc2.yml", "-f", "dc3.yml", "-p", projectName, "--verbose", "down"},
	})

	expectCommandHooks("0", t, tester)

	tester.RunAndCheck(t, env...)
}

func TestRunningCommandWithDockerComposeAndBuildAll(t *testing.T) {
	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	// Mock out the meta-data calls to the agent after checkout
	agent := tester.MustMock(t, "buildkite-agent")
	agent.
		Expect("meta-data", "exists", "buildkite:git:commit").
		AndExitWith(0)

	env := []string{
		"BUILDKITE_DOCKER_COMPOSE_CONTAINER=llamas",
		"BUILDKITE_DOCKER_COMPOSE_BUILD_ALL=true",
	}

	dockerCompose := tester.MustMock(t, "docker-compose")
	dockerCompose.IgnoreUnexpectedInvocations()
	dockerCompose.Expect("-f", "docker-compose.yml", "-p", "buildkite1111111111111111", "--verbose", "build", "--pull").Once()

	tester.RunAndCheck(t, env...)
}

func expectCommandHooks(exitStatus string, t *testing.T, tester *BootstrapTester) {
	tester.ExpectGlobalHook("pre-command").Once()
	tester.ExpectLocalHook("pre-command").Once()
	tester.ExpectGlobalHook("post-command").Once()
	tester.ExpectLocalHook("post-command").Once()

	preExitFunc := func(c *bintest.Call) {
		cmdExitStatus := c.GetEnv(`BUILDKITE_COMMAND_EXIT_STATUS`)
		if cmdExitStatus != exitStatus {
			t.Errorf("Expected an exit status of %s, got %v", exitStatus, cmdExitStatus)
		}
		c.Exit(0)
	}

	tester.ExpectGlobalHook("pre-exit").Once().AndCallFunc(preExitFunc)
	tester.ExpectLocalHook("pre-exit").Once().AndCallFunc(preExitFunc)
}
<commit_msg>Fix pattern to match generated files<commit_after>package integration

import (
	"runtime"
	"testing"

	"github.com/buildkite/bintest"
)

func argumentForCommand(cmd string) interface{} {
	// This is unpleasant, but we have to work around the fact that we generate
	// batch scripts for windows and plain commands for everything else
	if runtime.GOOS == `windows` {
		return bintest.MatchPattern(`buildkite-script-.+.bat$`)
	}
	return cmd
}

func TestRunningCommandWithDocker(t *testing.T) {
	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	// Mock out the meta-data calls to the agent after checkout
	agent := tester.MustMock(t, "buildkite-agent")
	agent.
		Expect("meta-data", "exists", "buildkite:git:commit").
		AndExitWith(0)

	env := []string{
		"BUILDKITE_DOCKER=llamas",
	}

	jobId := "1111-1111-1111-1111"
	imageId := "buildkite_" + jobId + "_image"
	containerId := "buildkite_" + jobId + "_container"

	docker := tester.MustMock(t, "docker")
	docker.ExpectAll([][]interface{}{
		{"build", "-f", "Dockerfile", "-t", imageId, "."},
		{"run", "--name", containerId, imageId, argumentForCommand("true")},
		{"rm", "-f", "-v", containerId},
	})

	expectCommandHooks("0", t, tester)

	tester.RunAndCheck(t, env...)
}

func TestRunningCommandWithDockerAndCustomDockerfile(t *testing.T) {
	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	// Mock out the meta-data calls to the agent after checkout
	agent := tester.MustMock(t, "buildkite-agent")
	agent.
		Expect("meta-data", "exists", "buildkite:git:commit").
		AndExitWith(0)

	env := []string{
		"BUILDKITE_DOCKER=llamas",
		"BUILDKITE_DOCKER_FILE=Dockerfile.llamas",
	}

	jobId := "1111-1111-1111-1111"
	imageId := "buildkite_" + jobId + "_image"
	containerId := "buildkite_" + jobId + "_container"

	docker := tester.MustMock(t, "docker")
	docker.ExpectAll([][]interface{}{
		{"build", "-f", "Dockerfile.llamas", "-t", imageId, "."},
		{"run", "--name", containerId, imageId, argumentForCommand("true")},
		{"rm", "-f", "-v", containerId},
	})

	expectCommandHooks("0", t, tester)

	tester.RunAndCheck(t, env...)
}

func TestRunningFailingCommandWithDocker(t *testing.T) {
	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	// Mock out the meta-data calls to the agent after checkout
	agent := tester.MustMock(t, "buildkite-agent")
	agent.
		Expect("meta-data", "exists", "buildkite:git:commit").
		AndExitWith(0)

	env := []string{
		"BUILDKITE_DOCKER=llamas",
	}

	jobId := "1111-1111-1111-1111"
	imageId := "buildkite_" + jobId + "_image"
	containerId := "buildkite_" + jobId + "_container"

	docker := tester.MustMock(t, "docker")
	docker.ExpectAll([][]interface{}{
		{"build", "-f", "Dockerfile", "-t", imageId, "."},
		{"rm", "-f", "-v", containerId},
	})

	docker.Expect("run", "--name", containerId, imageId, argumentForCommand("true")).
		AndExitWith(1)

	expectCommandHooks("1", t, tester)

	if err = tester.Run(t, env...); err == nil {
		t.Fatal("Expected bootstrap to fail")
	}

	tester.CheckMocks(t)
}

func TestRunningCommandWithDockerCompose(t *testing.T) {
	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	// Mock out the meta-data calls to the agent after checkout
	agent := tester.MustMock(t, "buildkite-agent")
	agent.
		Expect("meta-data", "exists", "buildkite:git:commit").
		AndExitWith(0)

	env := []string{
		"BUILDKITE_DOCKER_COMPOSE_CONTAINER=llamas",
	}

	projectName := "buildkite1111111111111111"

	dockerCompose := tester.MustMock(t, "docker-compose")
	dockerCompose.ExpectAll([][]interface{}{
		{"-f", "docker-compose.yml", "-p", projectName, "--verbose", "build", "--pull", "llamas"},
		{"-f", "docker-compose.yml", "-p", projectName, "--verbose", "run", "llamas", argumentForCommand("true")},
		{"-f", "docker-compose.yml", "-p", projectName, "--verbose", "kill"},
		{"-f", "docker-compose.yml", "-p", projectName, "--verbose", "rm", "--force", "--all", "-v"},
		{"-f", "docker-compose.yml", "-p", projectName, "--verbose", "down"},
	})

	expectCommandHooks("0", t, tester)

	tester.RunAndCheck(t, env...)
}

func TestRunningFailingCommandWithDockerCompose(t *testing.T) {
	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	// Mock out the meta-data calls to the agent after checkout
	agent := tester.MustMock(t, "buildkite-agent")
	agent.
		Expect("meta-data", "exists", "buildkite:git:commit").
		AndExitWith(0)

	env := []string{
		"BUILDKITE_DOCKER_COMPOSE_CONTAINER=llamas",
	}

	projectName := "buildkite1111111111111111"

	dockerCompose := tester.MustMock(t, "docker-compose")
	dockerCompose.ExpectAll([][]interface{}{
		{"-f", "docker-compose.yml", "-p", projectName, "--verbose", "build", "--pull", "llamas"},
		{"-f", "docker-compose.yml", "-p", projectName, "--verbose", "kill"},
		{"-f", "docker-compose.yml", "-p", projectName, "--verbose", "rm", "--force", "--all", "-v"},
		{"-f", "docker-compose.yml", "-p", projectName, "--verbose", "down"},
	})

	dockerCompose.Expect("-f", "docker-compose.yml", "-p", projectName, "--verbose", "run", "llamas", argumentForCommand("true")).
		AndWriteToStderr("Nope!").
		AndExitWith(1)

	expectCommandHooks("1", t, tester)

	if err = tester.Run(t, env...); err == nil {
		t.Fatal("Expected bootstrap to fail")
	}

	tester.CheckMocks(t)
}

func TestRunningCommandWithDockerComposeAndExtraConfig(t *testing.T) {
	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	// Mock out the meta-data calls to the agent after checkout
	agent := tester.MustMock(t, "buildkite-agent")
	agent.
		Expect("meta-data", "exists", "buildkite:git:commit").
		AndExitWith(0)

	env := []string{
		"BUILDKITE_DOCKER_COMPOSE_CONTAINER=llamas",
		"BUILDKITE_DOCKER_COMPOSE_FILE=dc1.yml:dc2.yml:dc3.yml",
	}

	projectName := "buildkite1111111111111111"

	dockerCompose := tester.MustMock(t, "docker-compose")
	dockerCompose.ExpectAll([][]interface{}{
		{"-f", "dc1.yml", "-f", "dc2.yml", "-f", "dc3.yml", "-p", projectName, "--verbose", "build", "--pull", "llamas"},
		{"-f", "dc1.yml", "-f", "dc2.yml", "-f", "dc3.yml", "-p", projectName, "--verbose", "run", "llamas", argumentForCommand("true")},
		{"-f", "dc1.yml", "-f", "dc2.yml", "-f", "dc3.yml", "-p", projectName, "--verbose", "kill"},
		{"-f", "dc1.yml", "-f", "dc2.yml", "-f", "dc3.yml", "-p", projectName, "--verbose", "rm", "--force", "--all", "-v"},
		{"-f", "dc1.yml", "-f", "dc2.yml", "-f", "dc3.yml", "-p", projectName, "--verbose", "down"},
	})

	expectCommandHooks("0", t, tester)

	tester.RunAndCheck(t, env...)
}

func TestRunningCommandWithDockerComposeAndBuildAll(t *testing.T) {
	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	// Mock out the meta-data calls to the agent after checkout
	agent := tester.MustMock(t, "buildkite-agent")
	agent.
		Expect("meta-data", "exists", "buildkite:git:commit").
		AndExitWith(0)

	env := []string{
		"BUILDKITE_DOCKER_COMPOSE_CONTAINER=llamas",
		"BUILDKITE_DOCKER_COMPOSE_BUILD_ALL=true",
	}

	dockerCompose := tester.MustMock(t, "docker-compose")
	dockerCompose.IgnoreUnexpectedInvocations()
	dockerCompose.Expect("-f", "docker-compose.yml", "-p", "buildkite1111111111111111", "--verbose", "build", "--pull").Once()

	tester.RunAndCheck(t, env...)
}

func expectCommandHooks(exitStatus string, t *testing.T, tester *BootstrapTester) {
	tester.ExpectGlobalHook("pre-command").Once()
	tester.ExpectLocalHook("pre-command").Once()
	tester.ExpectGlobalHook("post-command").Once()
	tester.ExpectLocalHook("post-command").Once()

	preExitFunc := func(c *bintest.Call) {
		cmdExitStatus := c.GetEnv(`BUILDKITE_COMMAND_EXIT_STATUS`)
		if cmdExitStatus != exitStatus {
			t.Errorf("Expected an exit status of %s, got %v", exitStatus, cmdExitStatus)
		}
		c.Exit(0)
	}

	tester.ExpectGlobalHook("pre-exit").Once().AndCallFunc(preExitFunc)
	tester.ExpectLocalHook("pre-exit").Once().AndCallFunc(preExitFunc)
}
<|endoftext|>


<commit_before>package report

import (
	"fmt"
	"io"
	"io/ioutil"
	"path"
	"encoding/json"
	"log"
	"os"
	"os/user"
	"path/filepath"
	"talisman/detector"
)

var reportsFolder string
const htmlFileName string = "report.html"
const jsonFileName string = "report.json"

// GenerateReport generates a talisman scan report in html format
func GenerateReport(r *detector.DetectionResults, directory string) string {

	var path string
	var jsonFilePath string
	var home_dir string
	var base_report_dir_path string

	if directory == "talisman_html_report" {
			path = directory
			// base_report_dir_path = filepath.Join(home_dir, ".talisman/bin/talisman_html_report")
			// Dir(base_report_dir_path, "talisman_html_report")
	} else {
			path = filepath.Join(directory, "talisman_reports")
	}

	jsonFilePath = filepath.Join(path, "/data", jsonFileName)

	usr, err := user.Current()
	home_dir = usr.HomeDir

	base_report_dir_path = filepath.Join(home_dir, ".talisman/bin/talisman_html_report")

	os.MkdirAll(path, 0755)
	Dir(base_report_dir_path, "talisman_html_report")

	jsonFile, err := os.Create(jsonFilePath)
	if err != nil {
		log.Fatal("Cannot create report.json file", err)

	}
	//jsonResultSchema := detector.GetJsonSchema(r)
	jsonString, err := json.Marshal(r)
	if err != nil {
		log.Fatal("Unable to marshal JSON")
	}
	jsonFile.Write(jsonString)
	jsonFile.Close()
	return path
}

func File(src, dst string) error {
	var err error
	var srcfd *os.File
	var dstfd *os.File
	var srcinfo os.FileInfo

	if srcfd, err = os.Open(src); err != nil {
		return err
	}
	defer srcfd.Close()

	if dstfd, err = os.Create(dst); err != nil {
		return err
	}
	defer dstfd.Close()

	if _, err = io.Copy(dstfd, srcfd); err != nil {
		return err
	}
	if srcinfo, err = os.Stat(src); err != nil {
		return err
	}
	return os.Chmod(dst, srcinfo.Mode())
}

func Dir(src string, dst string) error {

	var err error
	var fds []os.FileInfo
	var srcinfo os.FileInfo

	if srcinfo, err = os.Stat(src); err != nil {
		return err
	}

	if err = os.MkdirAll(dst, srcinfo.Mode()); err != nil {
		return err
	}

	if fds, err = ioutil.ReadDir(src); err != nil {
		return err
	}
	for _, fd := range fds {
		srcfp := path.Join(src, fd.Name())
		dstfp := path.Join(dst, fd.Name())

		if fd.IsDir() {
			if err = Dir(srcfp, dstfp); err != nil {
				fmt.Println(err)
			}
		} else {
			if err = File(srcfp, dstfp); err != nil {
				fmt.Println(err)
			}
		}
	}
	return nil
}
<commit_msg>JD: Worked on review comments for #113: ** Changed the creation of report folder path through Join ** Extracted report folder name to a constant.<commit_after>package report

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path"
	"path/filepath"
	"talisman/detector"
)

var reportsFolder string

const htmlFileName string = "report.html"
const jsonFileName string = "report.json"

// GenerateReport generates a talisman scan report in html format
func GenerateReport(r *detector.DetectionResults, directory string) string {

	var path string
	var jsonFilePath string
	var home_dir string
	var base_report_dir_path string
	const htmlReportDir string = "talisman_html_report"

	if directory == htmlReportDir {
		path = directory
		// base_report_dir_path = filepath.Join(home_dir, ".talisman/bin/talisman_html_report")
		// Dir(base_report_dir_path, "talisman_html_report")
	} else {
		path = filepath.Join(directory, "talisman_reports")
	}

	jsonFilePath = filepath.Join(path, "/data", jsonFileName)

	usr, err := user.Current()
	home_dir = usr.HomeDir

	base_report_dir_path = filepath.Join(home_dir, ".talisman", "bin", htmlReportDir)

	os.MkdirAll(path, 0755)
	Dir(base_report_dir_path, htmlReportDir)

	jsonFile, err := os.Create(jsonFilePath)
	if err != nil {
		log.Fatal("Cannot create report.json file", err)

	}
	//jsonResultSchema := detector.GetJsonSchema(r)
	jsonString, err := json.Marshal(r)
	if err != nil {
		log.Fatal("Unable to marshal JSON")
	}
	jsonFile.Write(jsonString)
	jsonFile.Close()
	return path
}

func File(src, dst string) error {
	var err error
	var srcfd *os.File
	var dstfd *os.File
	var srcinfo os.FileInfo

	if srcfd, err = os.Open(src); err != nil {
		return err
	}
	defer srcfd.Close()

	if dstfd, err = os.Create(dst); err != nil {
		return err
	}
	defer dstfd.Close()

	if _, err = io.Copy(dstfd, srcfd); err != nil {
		return err
	}
	if srcinfo, err = os.Stat(src); err != nil {
		return err
	}
	return os.Chmod(dst, srcinfo.Mode())
}

func Dir(src string, dst string) error {

	var err error
	var fds []os.FileInfo
	var srcinfo os.FileInfo

	if srcinfo, err = os.Stat(src); err != nil {
		return err
	}

	if err = os.MkdirAll(dst, srcinfo.Mode()); err != nil {
		return err
	}

	if fds, err = ioutil.ReadDir(src); err != nil {
		return err
	}
	for _, fd := range fds {
		srcfp := path.Join(src, fd.Name())
		dstfp := path.Join(dst, fd.Name())

		if fd.IsDir() {
			if err = Dir(srcfp, dstfp); err != nil {
				fmt.Println(err)
			}
		} else {
			if err = File(srcfp, dstfp); err != nil {
				fmt.Println(err)
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package taskrunner

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"time"

	hclog "github.com/hashicorp/go-hclog"
	"github.com/hashicorp/nomad/client/allocrunner/interfaces"
	ti "github.com/hashicorp/nomad/client/allocrunner/taskrunner/interfaces"
	"github.com/hashicorp/nomad/client/dynamicplugins"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/nomad/plugins/csi"
	"github.com/hashicorp/nomad/plugins/drivers"
)

// csiPluginSupervisorHook manages supervising plugins that are running as Nomad
// tasks. These plugins will be fingerprinted and it will manage connecting them
// to their requisite plugin manager.
//
// It provides a couple of things to a task running inside Nomad. These are:
// * A mount to the `plugin_mount_dir`, that will then be used by Nomad
//   to connect to the nested plugin and handle volume mounts.
// * When the task has started, it starts a loop of attempting to connect to the
//   plugin, to perform initial fingerprinting of the plugins capabilities before
//   notifying the plugin manager of the plugin.
type csiPluginSupervisorHook struct {
	logger     hclog.Logger
	alloc      *structs.Allocation
	task       *structs.Task
	runner     *TaskRunner
	mountPoint string

	// eventEmitter is used to emit events to the task
	eventEmitter ti.EventEmitter

	shutdownCtx      context.Context
	shutdownCancelFn context.CancelFunc

	running     bool
	runningLock sync.Mutex

	// previousHealthstate is used by the supervisor goroutine to track historic
	// health states for gating task events.
	previousHealthState bool
}

// The plugin supervisor uses the PrestartHook mechanism to setup the requisite
// mount points and configuration for the task that exposes a CSI plugin.
var _ interfaces.TaskPrestartHook = &csiPluginSupervisorHook{}

// The plugin supervisor uses the PoststartHook mechanism to start polling the
// plugin for readiness and supported functionality before registering the
// plugin with the catalog.
var _ interfaces.TaskPoststartHook = &csiPluginSupervisorHook{}

// The plugin supervisor uses the StopHook mechanism to deregister the plugin
// with the catalog and to ensure any mounts are cleaned up.
var _ interfaces.TaskStopHook = &csiPluginSupervisorHook{}

func newCSIPluginSupervisorHook(csiRootDir string, eventEmitter ti.EventEmitter, runner *TaskRunner, logger hclog.Logger) *csiPluginSupervisorHook {
	task := runner.Task()

	// The Plugin directory will look something like this:
	// .
	// ..
	// csi.sock - A unix domain socket used to communicate with the CSI Plugin
	// staging/
	//  {volume-id}/{usage-mode-hash}/ - Intermediary mount point that will be used by plugins that support NODE_STAGE_UNSTAGE capabilities.
	// per-alloc/
	//  {alloc-id}/{volume-id}/{usage-mode-hash}/ - Mount Point that will be bind-mounted into tasks that utilise the volume
	pluginRoot := filepath.Join(csiRootDir, string(task.CSIPluginConfig.Type), task.CSIPluginConfig.ID)

	shutdownCtx, cancelFn := context.WithCancel(context.Background())

	hook := &csiPluginSupervisorHook{
		alloc:            runner.Alloc(),
		runner:           runner,
		logger:           logger,
		task:             task,
		mountPoint:       pluginRoot,
		shutdownCtx:      shutdownCtx,
		shutdownCancelFn: cancelFn,
		eventEmitter:     eventEmitter,
	}

	return hook
}

func (*csiPluginSupervisorHook) Name() string {
	return "csi_plugin_supervisor"
}

// Prestart is called before the task is started including after every
// restart. This requires that the mount paths for a plugin be idempotent,
// despite us not knowing the name of the plugin ahead of time.
// Because of this, we use the allocid_taskname as the unique identifier for a
// plugin on the filesystem.
func (h *csiPluginSupervisorHook) Prestart(ctx context.Context,
	req *interfaces.TaskPrestartRequest, resp *interfaces.TaskPrestartResponse) error {
	// Create the mount directory that the container will access if it doesn't
	// already exist. Default to only nomad user access.
	if err := os.MkdirAll(h.mountPoint, 0700); err != nil && !os.IsExist(err) {
		return fmt.Errorf("failed to create mount point: %v", err)
	}

	configMount := &drivers.MountConfig{
		TaskPath:        h.task.CSIPluginConfig.MountDir,
		HostPath:        h.mountPoint,
		Readonly:        false,
		PropagationMode: "bidirectional",
	}
	devMount := &drivers.MountConfig{
		TaskPath: "/dev",
		HostPath: "/dev",
		Readonly: false,
	}

	mounts := ensureMountpointInserted(h.runner.hookResources.getMounts(), configMount)
	mounts = ensureMountpointInserted(mounts, devMount)

	h.runner.hookResources.setMounts(mounts)

	resp.Done = true
	return nil
}

// Poststart is called after the task has started. Poststart is not
// called if the allocation is terminal.
//
// The context is cancelled if the task is killed.
func (h *csiPluginSupervisorHook) Poststart(_ context.Context, _ *interfaces.TaskPoststartRequest, _ *interfaces.TaskPoststartResponse) error {
	// If we're already running the supervisor routine, then we don't need to try
	// and restart it here as it only terminates on `Stop` hooks.
	h.runningLock.Lock()
	if h.running {
		h.runningLock.Unlock()
		return nil
	}
	h.runningLock.Unlock()

	go h.ensureSupervisorLoop(h.shutdownCtx)
	return nil
}

// ensureSupervisorLoop should be called in a goroutine. It will terminate when
// the passed in context is terminated.
//
// The supervisor works by:
// - Initially waiting for the plugin to become available. This loop is expensive
//   and may do things like create new gRPC Clients on every iteration.
// - After receiving an initial healthy status, it will inform the plugin catalog
//   of the plugin, registering it with the plugins fingerprinted capabilities.
// - We then perform a more lightweight check, simply probing the plugin on a less
//   frequent interval to ensure it is still alive, emitting task events when this
//   status changes.
//
// Deeper fingerprinting of the plugin is implemented by the csimanager.
func (h *csiPluginSupervisorHook) ensureSupervisorLoop(ctx context.Context) {
	h.runningLock.Lock()
	if h.running == true {
		h.runningLock.Unlock()
		return
	}
	h.running = true
	h.runningLock.Unlock()

	defer func() {
		h.runningLock.Lock()
		h.running = false
		h.runningLock.Unlock()
	}()

	socketPath := filepath.Join(h.mountPoint, structs.CSISocketName)
	t := time.NewTimer(0)

	// Step 1: Wait for the plugin to initially become available.
WAITFORREADY:
	for {
		select {
		case <-ctx.Done():
			return
		case <-t.C:
			pluginHealthy, err := h.supervisorLoopOnce(ctx, socketPath)
			if err != nil || !pluginHealthy {
				h.logger.Debug("CSI Plugin not ready", "error", err)

				// Plugin is not yet returning healthy, because we want to optimise for
				// quickly bringing a plugin online, we use a short timeout here.
				// TODO(dani): Test with more plugins and adjust.
				t.Reset(5 * time.Second)
				continue
			}

			// Mark the plugin as healthy in a task event
			h.previousHealthState = pluginHealthy
			event := structs.NewTaskEvent(structs.TaskPluginHealthy)
			event.SetMessage(fmt.Sprintf("plugin: %s", h.task.CSIPluginConfig.ID))
			h.eventEmitter.EmitEvent(event)

			break WAITFORREADY
		}
	}

	// Step 2: Register the plugin with the catalog.
	deregisterPluginFn, err := h.registerPlugin(socketPath)
	if err != nil {
		h.logger.Error("CSI Plugin registration failed", "error", err)
		event := structs.NewTaskEvent(structs.TaskPluginUnhealthy)
		event.SetMessage(fmt.Sprintf("failed to register plugin: %s, reason: %v", h.task.CSIPluginConfig.ID, err))
		h.eventEmitter.EmitEvent(event)
	}

	// Step 3: Start the lightweight supervisor loop.
	t.Reset(0)
	for {
		select {
		case <-ctx.Done():
			// De-register plugins on task shutdown
			deregisterPluginFn()
			return
		case <-t.C:
			pluginHealthy, err := h.supervisorLoopOnce(ctx, socketPath)
			if err != nil {
				h.logger.Error("CSI Plugin fingerprinting failed", "error", err)
			}

			// The plugin has transitioned to a healthy state. Emit an event.
			if !h.previousHealthState && pluginHealthy {
				event := structs.NewTaskEvent(structs.TaskPluginHealthy)
				event.SetMessage(fmt.Sprintf("plugin: %s", h.task.CSIPluginConfig.ID))
				h.eventEmitter.EmitEvent(event)
			}

			// The plugin has transitioned to an unhealthy state. Emit an event.
			if h.previousHealthState && !pluginHealthy {
				event := structs.NewTaskEvent(structs.TaskPluginUnhealthy)
				if err != nil {
					event.SetMessage(fmt.Sprintf("error: %v", err))
				} else {
					event.SetMessage("Unknown Reason")
				}
				h.eventEmitter.EmitEvent(event)
			}

			h.previousHealthState = pluginHealthy

			// This loop is informational and in some plugins this may be expensive to
			// validate. We use a longer timeout (30s) to avoid causing undue work.
			t.Reset(30 * time.Second)
		}
	}
}

func (h *csiPluginSupervisorHook) registerPlugin(socketPath string) (func(), error) {

	// At this point we know the plugin is ready and we can fingerprint it
	// to get its vendor name and version
	client, err := csi.NewClient(socketPath, h.logger.Named("csi_client").With("plugin.name", h.task.CSIPluginConfig.ID, "plugin.type", h.task.CSIPluginConfig.Type))
	defer client.Close()
	if err != nil {
		return nil, fmt.Errorf("failed to create csi client: %v", err)
	}

	info, err := client.PluginInfo()
	if err != nil {
		return nil, fmt.Errorf("failed to probe plugin: %v", err)
	}

	mkInfoFn := func(pluginType string) *dynamicplugins.PluginInfo {
		return &dynamicplugins.PluginInfo{
			Type:    pluginType,
			Name:    h.task.CSIPluginConfig.ID,
			Version: info.PluginVersion,
			ConnectionInfo: &dynamicplugins.PluginConnectionInfo{
				SocketPath: socketPath,
			},
			AllocID: h.alloc.ID,
			Options: map[string]string{
				"Provider":            info.Name, // vendor name
				"MountPoint":          h.mountPoint,
				"ContainerMountPoint": h.task.CSIPluginConfig.MountDir,
			},
		}
	}

	registrations := []*dynamicplugins.PluginInfo{}

	switch h.task.CSIPluginConfig.Type {
	case structs.CSIPluginTypeController:
		registrations = append(registrations, mkInfoFn(dynamicplugins.PluginTypeCSIController))
	case structs.CSIPluginTypeNode:
		registrations = append(registrations, mkInfoFn(dynamicplugins.PluginTypeCSINode))
	case structs.CSIPluginTypeMonolith:
		registrations = append(registrations, mkInfoFn(dynamicplugins.PluginTypeCSIController))
		registrations = append(registrations, mkInfoFn(dynamicplugins.PluginTypeCSINode))
	}

	deregistrationFns := []func(){}

	for _, reg := range registrations {
		if err := h.runner.dynamicRegistry.RegisterPlugin(reg); err != nil {
			for _, fn := range deregistrationFns {
				fn()
			}
			return nil, err
		}

		// need to rebind these so that each deregistration function
		// closes over its own registration
		rname := reg.Name
		rtype := reg.Type
		deregistrationFns = append(deregistrationFns, func() {
			err := h.runner.dynamicRegistry.DeregisterPlugin(rtype, rname)
			if err != nil {
				h.logger.Error("failed to deregister csi plugin", "name", rname, "type", rtype, "error", err)
			}
		})
	}

	return func() {
		for _, fn := range deregistrationFns {
			fn()
		}
	}, nil
}

func (h *csiPluginSupervisorHook) supervisorLoopOnce(ctx context.Context, socketPath string) (bool, error) {
	_, err := os.Stat(socketPath)
	if err != nil {
		return false, fmt.Errorf("failed to stat socket: %v", err)
	}

	client, err := csi.NewClient(socketPath, h.logger.Named("csi_client").With("plugin.name", h.task.CSIPluginConfig.ID, "plugin.type", h.task.CSIPluginConfig.Type))
	if err != nil {
		return false, fmt.Errorf("failed to create csi client: %v", err)
	}
	defer client.Close()

	healthy, err := client.PluginProbe(ctx)
	if err != nil {
		return false, fmt.Errorf("failed to probe plugin: %v", err)
	}

	return healthy, nil
}

// Stop is called after the task has exited and will not be started
// again. It is the only hook guaranteed to be executed whenever
// TaskRunner.Run is called (and not gracefully shutting down).
// Therefore it may be called even when prestart and the other hooks
// have not.
//
// Stop hooks must be idempotent. The context is cancelled prematurely if the
// task is killed.
func (h *csiPluginSupervisorHook) Stop(_ context.Context, req *interfaces.TaskStopRequest, _ *interfaces.TaskStopResponse) error {
	h.shutdownCancelFn()
	return nil
}

func ensureMountpointInserted(mounts []*drivers.MountConfig, mount *drivers.MountConfig) []*drivers.MountConfig {
	for _, mnt := range mounts {
		if mnt.IsEqual(mount) {
			return mounts
		}
	}

	mounts = append(mounts, mount)
	return mounts
}
<commit_msg>client/allocrunner/taskrunner: client.Close after err check (#8825)<commit_after>package taskrunner

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"time"

	hclog "github.com/hashicorp/go-hclog"
	"github.com/hashicorp/nomad/client/allocrunner/interfaces"
	ti "github.com/hashicorp/nomad/client/allocrunner/taskrunner/interfaces"
	"github.com/hashicorp/nomad/client/dynamicplugins"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/nomad/plugins/csi"
	"github.com/hashicorp/nomad/plugins/drivers"
)

// csiPluginSupervisorHook manages supervising plugins that are running as Nomad
// tasks. These plugins will be fingerprinted and it will manage connecting them
// to their requisite plugin manager.
//
// It provides a couple of things to a task running inside Nomad. These are:
// * A mount to the `plugin_mount_dir`, that will then be used by Nomad
//   to connect to the nested plugin and handle volume mounts.
// * When the task has started, it starts a loop of attempting to connect to the
//   plugin, to perform initial fingerprinting of the plugins capabilities before
//   notifying the plugin manager of the plugin.
type csiPluginSupervisorHook struct {
	logger     hclog.Logger
	alloc      *structs.Allocation
	task       *structs.Task
	runner     *TaskRunner
	mountPoint string

	// eventEmitter is used to emit events to the task
	eventEmitter ti.EventEmitter

	shutdownCtx      context.Context
	shutdownCancelFn context.CancelFunc

	running     bool
	runningLock sync.Mutex

	// previousHealthstate is used by the supervisor goroutine to track historic
	// health states for gating task events.
	previousHealthState bool
}

// The plugin supervisor uses the PrestartHook mechanism to setup the requisite
// mount points and configuration for the task that exposes a CSI plugin.
var _ interfaces.TaskPrestartHook = &csiPluginSupervisorHook{}

// The plugin supervisor uses the PoststartHook mechanism to start polling the
// plugin for readiness and supported functionality before registering the
// plugin with the catalog.
var _ interfaces.TaskPoststartHook = &csiPluginSupervisorHook{}

// The plugin supervisor uses the StopHook mechanism to deregister the plugin
// with the catalog and to ensure any mounts are cleaned up.
var _ interfaces.TaskStopHook = &csiPluginSupervisorHook{}

func newCSIPluginSupervisorHook(csiRootDir string, eventEmitter ti.EventEmitter, runner *TaskRunner, logger hclog.Logger) *csiPluginSupervisorHook {
	task := runner.Task()

	// The Plugin directory will look something like this:
	// .
	// ..
	// csi.sock - A unix domain socket used to communicate with the CSI Plugin
	// staging/
	//  {volume-id}/{usage-mode-hash}/ - Intermediary mount point that will be used by plugins that support NODE_STAGE_UNSTAGE capabilities.
	// per-alloc/
	//  {alloc-id}/{volume-id}/{usage-mode-hash}/ - Mount Point that will be bind-mounted into tasks that utilise the volume
	pluginRoot := filepath.Join(csiRootDir, string(task.CSIPluginConfig.Type), task.CSIPluginConfig.ID)

	shutdownCtx, cancelFn := context.WithCancel(context.Background())

	hook := &csiPluginSupervisorHook{
		alloc:            runner.Alloc(),
		runner:           runner,
		logger:           logger,
		task:             task,
		mountPoint:       pluginRoot,
		shutdownCtx:      shutdownCtx,
		shutdownCancelFn: cancelFn,
		eventEmitter:     eventEmitter,
	}

	return hook
}

func (*csiPluginSupervisorHook) Name() string {
	return "csi_plugin_supervisor"
}

// Prestart is called before the task is started including after every
// restart. This requires that the mount paths for a plugin be idempotent,
// despite us not knowing the name of the plugin ahead of time.
// Because of this, we use the allocid_taskname as the unique identifier for a
// plugin on the filesystem.
func (h *csiPluginSupervisorHook) Prestart(ctx context.Context,
	req *interfaces.TaskPrestartRequest, resp *interfaces.TaskPrestartResponse) error {
	// Create the mount directory that the container will access if it doesn't
	// already exist. Default to only nomad user access.
	if err := os.MkdirAll(h.mountPoint, 0700); err != nil && !os.IsExist(err) {
		return fmt.Errorf("failed to create mount point: %v", err)
	}

	configMount := &drivers.MountConfig{
		TaskPath:        h.task.CSIPluginConfig.MountDir,
		HostPath:        h.mountPoint,
		Readonly:        false,
		PropagationMode: "bidirectional",
	}
	devMount := &drivers.MountConfig{
		TaskPath: "/dev",
		HostPath: "/dev",
		Readonly: false,
	}

	mounts := ensureMountpointInserted(h.runner.hookResources.getMounts(), configMount)
	mounts = ensureMountpointInserted(mounts, devMount)

	h.runner.hookResources.setMounts(mounts)

	resp.Done = true
	return nil
}

// Poststart is called after the task has started. Poststart is not
// called if the allocation is terminal.
//
// The context is cancelled if the task is killed.
func (h *csiPluginSupervisorHook) Poststart(_ context.Context, _ *interfaces.TaskPoststartRequest, _ *interfaces.TaskPoststartResponse) error {
	// If we're already running the supervisor routine, then we don't need to try
	// and restart it here as it only terminates on `Stop` hooks.
	h.runningLock.Lock()
	if h.running {
		h.runningLock.Unlock()
		return nil
	}
	h.runningLock.Unlock()

	go h.ensureSupervisorLoop(h.shutdownCtx)
	return nil
}

// ensureSupervisorLoop should be called in a goroutine. It will terminate when
// the passed in context is terminated.
//
// The supervisor works by:
// - Initially waiting for the plugin to become available. This loop is expensive
//   and may do things like create new gRPC Clients on every iteration.
// - After receiving an initial healthy status, it will inform the plugin catalog
//   of the plugin, registering it with the plugins fingerprinted capabilities.
// - We then perform a more lightweight check, simply probing the plugin on a less
//   frequent interval to ensure it is still alive, emitting task events when this
//   status changes.
//
// Deeper fingerprinting of the plugin is implemented by the csimanager.
func (h *csiPluginSupervisorHook) ensureSupervisorLoop(ctx context.Context) {
	h.runningLock.Lock()
	if h.running == true {
		h.runningLock.Unlock()
		return
	}
	h.running = true
	h.runningLock.Unlock()

	defer func() {
		h.runningLock.Lock()
		h.running = false
		h.runningLock.Unlock()
	}()

	socketPath := filepath.Join(h.mountPoint, structs.CSISocketName)
	t := time.NewTimer(0)

	// Step 1: Wait for the plugin to initially become available.
WAITFORREADY:
	for {
		select {
		case <-ctx.Done():
			return
		case <-t.C:
			pluginHealthy, err := h.supervisorLoopOnce(ctx, socketPath)
			if err != nil || !pluginHealthy {
				h.logger.Debug("CSI Plugin not ready", "error", err)

				// Plugin is not yet returning healthy, because we want to optimise for
				// quickly bringing a plugin online, we use a short timeout here.
				// TODO(dani): Test with more plugins and adjust.
				t.Reset(5 * time.Second)
				continue
			}

			// Mark the plugin as healthy in a task event
			h.previousHealthState = pluginHealthy
			event := structs.NewTaskEvent(structs.TaskPluginHealthy)
			event.SetMessage(fmt.Sprintf("plugin: %s", h.task.CSIPluginConfig.ID))
			h.eventEmitter.EmitEvent(event)

			break WAITFORREADY
		}
	}

	// Step 2: Register the plugin with the catalog.
	deregisterPluginFn, err := h.registerPlugin(socketPath)
	if err != nil {
		h.logger.Error("CSI Plugin registration failed", "error", err)
		event := structs.NewTaskEvent(structs.TaskPluginUnhealthy)
		event.SetMessage(fmt.Sprintf("failed to register plugin: %s, reason: %v", h.task.CSIPluginConfig.ID, err))
		h.eventEmitter.EmitEvent(event)
	}

	// Step 3: Start the lightweight supervisor loop.
	t.Reset(0)
	for {
		select {
		case <-ctx.Done():
			// De-register plugins on task shutdown
			deregisterPluginFn()
			return
		case <-t.C:
			pluginHealthy, err := h.supervisorLoopOnce(ctx, socketPath)
			if err != nil {
				h.logger.Error("CSI Plugin fingerprinting failed", "error", err)
			}

			// The plugin has transitioned to a healthy state. Emit an event.
			if !h.previousHealthState && pluginHealthy {
				event := structs.NewTaskEvent(structs.TaskPluginHealthy)
				event.SetMessage(fmt.Sprintf("plugin: %s", h.task.CSIPluginConfig.ID))
				h.eventEmitter.EmitEvent(event)
			}

			// The plugin has transitioned to an unhealthy state. Emit an event.
			if h.previousHealthState && !pluginHealthy {
				event := structs.NewTaskEvent(structs.TaskPluginUnhealthy)
				if err != nil {
					event.SetMessage(fmt.Sprintf("error: %v", err))
				} else {
					event.SetMessage("Unknown Reason")
				}
				h.eventEmitter.EmitEvent(event)
			}

			h.previousHealthState = pluginHealthy

			// This loop is informational and in some plugins this may be expensive to
			// validate. We use a longer timeout (30s) to avoid causing undue work.
			t.Reset(30 * time.Second)
		}
	}
}

func (h *csiPluginSupervisorHook) registerPlugin(socketPath string) (func(), error) {

	// At this point we know the plugin is ready and we can fingerprint it
	// to get its vendor name and version
	client, err := csi.NewClient(socketPath, h.logger.Named("csi_client").With("plugin.name", h.task.CSIPluginConfig.ID, "plugin.type", h.task.CSIPluginConfig.Type))
	if err != nil {
		return nil, fmt.Errorf("failed to create csi client: %v", err)
	}
	defer client.Close()

	info, err := client.PluginInfo()
	if err != nil {
		return nil, fmt.Errorf("failed to probe plugin: %v", err)
	}

	mkInfoFn := func(pluginType string) *dynamicplugins.PluginInfo {
		return &dynamicplugins.PluginInfo{
			Type:    pluginType,
			Name:    h.task.CSIPluginConfig.ID,
			Version: info.PluginVersion,
			ConnectionInfo: &dynamicplugins.PluginConnectionInfo{
				SocketPath: socketPath,
			},
			AllocID: h.alloc.ID,
			Options: map[string]string{
				"Provider":            info.Name, // vendor name
				"MountPoint":          h.mountPoint,
				"ContainerMountPoint": h.task.CSIPluginConfig.MountDir,
			},
		}
	}

	registrations := []*dynamicplugins.PluginInfo{}

	switch h.task.CSIPluginConfig.Type {
	case structs.CSIPluginTypeController:
		registrations = append(registrations, mkInfoFn(dynamicplugins.PluginTypeCSIController))
	case structs.CSIPluginTypeNode:
		registrations = append(registrations, mkInfoFn(dynamicplugins.PluginTypeCSINode))
	case structs.CSIPluginTypeMonolith:
		registrations = append(registrations, mkInfoFn(dynamicplugins.PluginTypeCSIController))
		registrations = append(registrations, mkInfoFn(dynamicplugins.PluginTypeCSINode))
	}

	deregistrationFns := []func(){}

	for _, reg := range registrations {
		if err := h.runner.dynamicRegistry.RegisterPlugin(reg); err != nil {
			for _, fn := range deregistrationFns {
				fn()
			}
			return nil, err
		}

		// need to rebind these so that each deregistration function
		// closes over its own registration
		rname := reg.Name
		rtype := reg.Type
		deregistrationFns = append(deregistrationFns, func() {
			err := h.runner.dynamicRegistry.DeregisterPlugin(rtype, rname)
			if err != nil {
				h.logger.Error("failed to deregister csi plugin", "name", rname, "type", rtype, "error", err)
			}
		})
	}

	return func() {
		for _, fn := range deregistrationFns {
			fn()
		}
	}, nil
}

func (h *csiPluginSupervisorHook) supervisorLoopOnce(ctx context.Context, socketPath string) (bool, error) {
	_, err := os.Stat(socketPath)
	if err != nil {
		return false, fmt.Errorf("failed to stat socket: %v", err)
	}

	client, err := csi.NewClient(socketPath, h.logger.Named("csi_client").With("plugin.name", h.task.CSIPluginConfig.ID, "plugin.type", h.task.CSIPluginConfig.Type))
	if err != nil {
		return false, fmt.Errorf("failed to create csi client: %v", err)
	}
	defer client.Close()

	healthy, err := client.PluginProbe(ctx)
	if err != nil {
		return false, fmt.Errorf("failed to probe plugin: %v", err)
	}

	return healthy, nil
}

// Stop is called after the task has exited and will not be started
// again. It is the only hook guaranteed to be executed whenever
// TaskRunner.Run is called (and not gracefully shutting down).
// Therefore it may be called even when prestart and the other hooks
// have not.
//
// Stop hooks must be idempotent. The context is cancelled prematurely if the
// task is killed.
func (h *csiPluginSupervisorHook) Stop(_ context.Context, req *interfaces.TaskStopRequest, _ *interfaces.TaskStopResponse) error {
	h.shutdownCancelFn()
	return nil
}

func ensureMountpointInserted(mounts []*drivers.MountConfig, mount *drivers.MountConfig) []*drivers.MountConfig {
	for _, mnt := range mounts {
		if mnt.IsEqual(mount) {
			return mounts
		}
	}

	mounts = append(mounts, mount)
	return mounts
}
<|endoftext|>


<commit_before>package beep_test

import (
	"reflect"
	"testing"

	"github.com/faiface/beep"
)

func TestResample(t *testing.T) {
	for _, numSamples := range []int{8, 100, 500, 1000, 50000} {
		for _, old := range []beep.SampleRate{100, 2000, 44100, 48000} {
			for _, new := range []beep.SampleRate{100, 2000, 44100, 48000} {
				s, data := randomDataStreamer(numSamples)

				want := resampleCorrect(3, old, new, data)

				got := collect(beep.Resample(3, old, new, s))

				if !reflect.DeepEqual(want, got) {
					t.Fatal("Resample not working correctly")
				}
			}
		}
	}
}

func resampleCorrect(quality int, old, new beep.SampleRate, p [][2]float64) [][2]float64 {
	ratio := float64(old) / float64(new)
	pts := make([]point, quality*2)
	var resampled [][2]float64
	for i := 0; ; i++ {
		j := float64(i) * ratio
		if int(j) >= len(p) {
			break
		}
		var sample [2]float64
		for c := range sample {
			for k := range pts {
				l := int(j) + k - len(pts)/2 + 1
				if l >= 0 && l < len(p) {
					pts[k] = point{X: float64(l), Y: p[l][c]}
				} else {
					pts[k] = point{X: float64(l), Y: 0}
				}
			}
			y := lagrange(pts[:], j)
			sample[c] = y
		}
		resampled = append(resampled, sample)
	}
	return resampled
}

func lagrange(pts []point, x float64) (y float64) {
	y = 0.0
	for j := range pts {
		l := 1.0
		for m := range pts {
			if j == m {
				continue
			}
			l *= (x - pts[m].X) / (pts[j].X - pts[m].X)
		}
		y += pts[j].Y * l
	}
	return y
}

type point struct {
	X, Y float64
}
<commit_msg>resample_test: don't test too expensive cases<commit_after>package beep_test

import (
	"reflect"
	"testing"

	"github.com/faiface/beep"
)

func TestResample(t *testing.T) {
	for _, numSamples := range []int{8, 100, 500, 1000, 50000} {
		for _, old := range []beep.SampleRate{100, 2000, 44100, 48000} {
			for _, new := range []beep.SampleRate{100, 2000, 44100, 48000} {
				if numSamples/int(old)*int(new) > 1e6 {
					continue // skip too expensive combinations
				}

				s, data := randomDataStreamer(numSamples)

				want := resampleCorrect(3, old, new, data)

				got := collect(beep.Resample(3, old, new, s))

				if !reflect.DeepEqual(want, got) {
					t.Fatal("Resample not working correctly")
				}
			}
		}
	}
}

func resampleCorrect(quality int, old, new beep.SampleRate, p [][2]float64) [][2]float64 {
	ratio := float64(old) / float64(new)
	pts := make([]point, quality*2)
	var resampled [][2]float64
	for i := 0; ; i++ {
		j := float64(i) * ratio
		if int(j) >= len(p) {
			break
		}
		var sample [2]float64
		for c := range sample {
			for k := range pts {
				l := int(j) + k - len(pts)/2 + 1
				if l >= 0 && l < len(p) {
					pts[k] = point{X: float64(l), Y: p[l][c]}
				} else {
					pts[k] = point{X: float64(l), Y: 0}
				}
			}
			y := lagrange(pts[:], j)
			sample[c] = y
		}
		resampled = append(resampled, sample)
	}
	return resampled
}

func lagrange(pts []point, x float64) (y float64) {
	y = 0.0
	for j := range pts {
		l := 1.0
		for m := range pts {
			if j == m {
				continue
			}
			l *= (x - pts[m].X) / (pts[j].X - pts[m].X)
		}
		y += pts[j].Y * l
	}
	return y
}

type point struct {
	X, Y float64
}
<|endoftext|>


<commit_before>package rest

import (
	"bufio"
	"encoding/json"
	"net"
	"net/http"
)

// A ResponseWriter interface dedicated to JSON HTTP response.
// Note that the object instantiated by the ResourceHandler that implements this interface,
// also happens to implement http.ResponseWriter, http.Flusher and http.CloseNotifier.
type ResponseWriter interface {

	// Identical to the http.ResponseWriter interface
	Header() http.Header

	// Use EncodeJson to generate the payload, write the headers with http.StatusOK if
	// they are not already written, then write the payload.
	// The Content-Type header is set to "application/json", unless already specified.
	WriteJson(v interface{}) error

	// Encode the data structure to JSON, mainly used to wrap ResponseWriter in
	// middlewares.
	EncodeJson(v interface{}) ([]byte, error)

	// Similar to the http.ResponseWriter interface, with additional JSON related
	// headers set.
	WriteHeader(int)
}

// Error produces an error response in JSON with the following structure, '{"Error":"My error message"}'
// The standard plain text net/http Error helper can still be called like this:
// http.Error(w, "error message", code)
func Error(w ResponseWriter, error string, code int) {
	w.WriteHeader(code)
	err := w.WriteJson(map[string]string{"Error": error})
	if err != nil {
		panic(err)
	}
}

// NotFound produces a 404 response with the following JSON, '{"Error":"Resource not found"}'
// The standard plain text net/http NotFound helper can still be called like this:
// http.NotFound(w, r.Request)
func NotFound(w ResponseWriter, r *Request) {
	Error(w, "Resource not found", http.StatusNotFound)
}

// Private responseWriter intantiated by the resource handler.
// It implements the following interfaces:
// ResponseWriter
// http.ResponseWriter
// http.Flusher
// http.CloseNotifier
// http.Hijacker
type responseWriter struct {
	http.ResponseWriter
	wroteHeader bool
}

func (w *responseWriter) WriteHeader(code int) {
	if w.Header().Get("Content-Type") == "" {
		w.Header().Set("Content-Type", "application/json")
	}
	w.ResponseWriter.WriteHeader(code)
	w.wroteHeader = true
}

func (w *responseWriter) EncodeJson(v interface{}) ([]byte, error) {
	b, err := json.Marshal(v)
	if err != nil {
		return nil, err
	}
	return b, nil
}

// Encode the object in JSON and call Write.
func (w *responseWriter) WriteJson(v interface{}) error {
	b, err := w.EncodeJson(v)
	if err != nil {
		return err
	}
	_, err = w.Write(b)
	if err != nil {
		return err
	}
	return nil
}

// Provided in order to implement the http.ResponseWriter interface.
func (w *responseWriter) Write(b []byte) (int, error) {
	if !w.wroteHeader {
		w.WriteHeader(http.StatusOK)
	}
	return w.ResponseWriter.Write(b)
}

// Provided in order to implement the http.Flusher interface.
func (w *responseWriter) Flush() {
	if !w.wroteHeader {
		w.WriteHeader(http.StatusOK)
	}
	flusher := w.ResponseWriter.(http.Flusher)
	flusher.Flush()
}

// Provided in order to implement the http.CloseNotifier interface.
func (w *responseWriter) CloseNotify() <-chan bool {
	notifier := w.ResponseWriter.(http.CloseNotifier)
	return notifier.CloseNotify()
}

// Provided in order to implement the http.Hijacker interface.
func (w *responseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
	hijacker := w.ResponseWriter.(http.Hijacker)
	return hijacker.Hijack()
}
<commit_msg>Fix a docstring that was mentioning the ResourceHandler.<commit_after>package rest

import (
	"bufio"
	"encoding/json"
	"net"
	"net/http"
)

// A ResponseWriter interface dedicated to JSON HTTP response.
// Note, the responseWriter object instantiated by the framework also implements many other interfaces
// accessible by type assertion: http.ResponseWriter, http.Flusher, http.CloseNotifier, http.Hijacker.
type ResponseWriter interface {

	// Identical to the http.ResponseWriter interface
	Header() http.Header

	// Use EncodeJson to generate the payload, write the headers with http.StatusOK if
	// they are not already written, then write the payload.
	// The Content-Type header is set to "application/json", unless already specified.
	WriteJson(v interface{}) error

	// Encode the data structure to JSON, mainly used to wrap ResponseWriter in
	// middlewares.
	EncodeJson(v interface{}) ([]byte, error)

	// Similar to the http.ResponseWriter interface, with additional JSON related
	// headers set.
	WriteHeader(int)
}

// Error produces an error response in JSON with the following structure, '{"Error":"My error message"}'
// The standard plain text net/http Error helper can still be called like this:
// http.Error(w, "error message", code)
func Error(w ResponseWriter, error string, code int) {
	w.WriteHeader(code)
	err := w.WriteJson(map[string]string{"Error": error})
	if err != nil {
		panic(err)
	}
}

// NotFound produces a 404 response with the following JSON, '{"Error":"Resource not found"}'
// The standard plain text net/http NotFound helper can still be called like this:
// http.NotFound(w, r.Request)
func NotFound(w ResponseWriter, r *Request) {
	Error(w, "Resource not found", http.StatusNotFound)
}

// Private responseWriter intantiated by the resource handler.
// It implements the following interfaces:
// ResponseWriter
// http.ResponseWriter
// http.Flusher
// http.CloseNotifier
// http.Hijacker
type responseWriter struct {
	http.ResponseWriter
	wroteHeader bool
}

func (w *responseWriter) WriteHeader(code int) {
	if w.Header().Get("Content-Type") == "" {
		w.Header().Set("Content-Type", "application/json")
	}
	w.ResponseWriter.WriteHeader(code)
	w.wroteHeader = true
}

func (w *responseWriter) EncodeJson(v interface{}) ([]byte, error) {
	b, err := json.Marshal(v)
	if err != nil {
		return nil, err
	}
	return b, nil
}

// Encode the object in JSON and call Write.
func (w *responseWriter) WriteJson(v interface{}) error {
	b, err := w.EncodeJson(v)
	if err != nil {
		return err
	}
	_, err = w.Write(b)
	if err != nil {
		return err
	}
	return nil
}

// Provided in order to implement the http.ResponseWriter interface.
func (w *responseWriter) Write(b []byte) (int, error) {
	if !w.wroteHeader {
		w.WriteHeader(http.StatusOK)
	}
	return w.ResponseWriter.Write(b)
}

// Provided in order to implement the http.Flusher interface.
func (w *responseWriter) Flush() {
	if !w.wroteHeader {
		w.WriteHeader(http.StatusOK)
	}
	flusher := w.ResponseWriter.(http.Flusher)
	flusher.Flush()
}

// Provided in order to implement the http.CloseNotifier interface.
func (w *responseWriter) CloseNotify() <-chan bool {
	notifier := w.ResponseWriter.(http.CloseNotifier)
	return notifier.CloseNotify()
}

// Provided in order to implement the http.Hijacker interface.
func (w *responseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
	hijacker := w.ResponseWriter.(http.Hijacker)
	return hijacker.Hijack()
}
<|endoftext|>