<commit_before>// This tools automatically finds the latest CRI-O release branch and merges
// the latest master branch into it. This happens only if there is no
// tag present on the release branch.
package main

import (
	"flag"
	"strings"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"k8s.io/release/pkg/command"
	kgit "k8s.io/release/pkg/git"
	"k8s.io/release/pkg/util"
)

const (
	remote              = "https://github.com/cri-o/cri-o"
	git                 = "git"
	grep                = "grep"
	tail                = "tail"
	releaseBranchPrefix = "release-"
	dryRunEnv           = "DRY_RUN"
)

var dryRun bool

func main() {
	flag.BoolVar(
		&dryRun,
		"dry-run",
		!util.IsEnvSet(dryRunEnv),
		"do not really push, just do a dry run",
	)
	flag.Parse()

	logrus.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true})
	if err := run(); err != nil {
		logrus.Fatal(err)
	}
}

func run() error {
	if !command.Available(git, grep, tail) {
		return errors.Errorf(
			"please ensure that %s are available in $PATH",
			strings.Join([]string{git, grep, tail}, ", "),
		)
	}

	if dryRun {
		logrus.Warnf("Please note that this is only a dry-run and will not "+
			"result in any real git push to the remote location. "+
			"To enable a real git push, export the environment "+
			"variable %s=false",
			dryRunEnv,
		)
	}

	// Get the latest release branch
	lsRemoteHeads, err := command.
		New(git, "ls-remote", "--sort=v:refname", "--heads", remote).
		Pipe(grep, "-Eo", releaseBranchPrefix+".*").
		Pipe(tail, "-1").
		RunSilentSuccessOutput()
	if err != nil {
		return errors.Wrap(err, "unable to retrieve latest release branch")
	}
	latestReleaseBranch := lsRemoteHeads.OutputTrimNL()
	logrus.Infof("Latest release branch: %s", latestReleaseBranch)

	// Check if a release has been done on that branch
	tagPrefix := strings.TrimPrefix(latestReleaseBranch, releaseBranchPrefix)
	lsRemoteTags, err := command.
		New(git, "ls-remote", "--sort=v:refname", "--tags", remote).
		Pipe(grep, "-Eo", "v"+tagPrefix+".*").
		RunSilentSuccessOutput()
	if err == nil {
		logrus.Warnf(
			"Found existing tag(s) on release branch: %v",
			strings.Join(strings.Fields(lsRemoteTags.OutputTrimNL()), ", "),
		)
		logrus.Infof("We’re all set, doing nothing")
		return nil
	}

	// Checkout the release branch
	repo, err := kgit.OpenRepo(".")
	if err != nil {
		return errors.Wrap(err, "unable to open this repository")
	}
	if dryRun {
		logrus.Info("Setting repository to only do a dry-run")
		repo.SetDry()
	}
	currentBranch, err := repo.CurrentBranch()
	if err != nil {
		return errors.Wrap(err, "unable to get current branch")
	}
	logrus.Infof("Checking out branch: %s", latestReleaseBranch)
	if err := repo.Checkout(latestReleaseBranch); err != nil {
		return errors.Wrapf(err,
			"unable to checkout release branch %s", latestReleaseBranch,
		)
	}
	defer func() {
		logrus.Infof("Checking out branch: %s", currentBranch)
		err = repo.Checkout(currentBranch)
	}()

	// Merge the latest master
	mergeTarget := kgit.Remotify(kgit.Master)
	if err := repo.Merge(mergeTarget); err != nil {
		return errors.Wrapf(err,
			"unable to merge %s into release branch", mergeTarget,
		)
	}

	// Push the changes
	if err := repo.Push(latestReleaseBranch); err != nil {
		return errors.Wrap(err, "unable to push to remote branch")
	}

	return nil
}
<commit_msg>branch foward: skip release candidates<commit_after>// This tools automatically finds the latest CRI-O release branch and merges
// the latest master branch into it. This happens only if there is no
// tag present on the release branch.
package main

import (
	"flag"
	"strings"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"k8s.io/release/pkg/command"
	kgit "k8s.io/release/pkg/git"
	"k8s.io/release/pkg/util"
)

const (
	remote              = "https://github.com/cri-o/cri-o"
	git                 = "git"
	grep                = "grep"
	tail                = "tail"
	releaseBranchPrefix = "release-"
	dryRunEnv           = "DRY_RUN"
)

var dryRun bool

func main() {
	flag.BoolVar(
		&dryRun,
		"dry-run",
		!util.IsEnvSet(dryRunEnv),
		"do not really push, just do a dry run",
	)
	flag.Parse()

	logrus.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true})
	if err := run(); err != nil {
		logrus.Fatal(err)
	}
}

func run() error {
	if !command.Available(git, grep, tail) {
		return errors.Errorf(
			"please ensure that %s are available in $PATH",
			strings.Join([]string{git, grep, tail}, ", "),
		)
	}

	if dryRun {
		logrus.Warnf("Please note that this is only a dry-run and will not "+
			"result in any real git push to the remote location. "+
			"To enable a real git push, export the environment "+
			"variable %s=false",
			dryRunEnv,
		)
	}

	// Get the latest release branch
	lsRemoteHeads, err := command.
		New(git, "ls-remote", "--sort=v:refname", "--heads", remote).
		Pipe(grep, "-Eo", releaseBranchPrefix+".*").
		Pipe(tail, "-1").
		RunSilentSuccessOutput()
	if err != nil {
		return errors.Wrap(err, "unable to retrieve latest release branch")
	}
	latestReleaseBranch := lsRemoteHeads.OutputTrimNL()
	logrus.Infof("Latest release branch: %s", latestReleaseBranch)

	// Check if a release has been done on that branch
	tagPrefix := strings.TrimPrefix(latestReleaseBranch, releaseBranchPrefix)
	lsRemoteTags, err := command.
		New(git, "ls-remote", "--sort=v:refname", "--tags", remote).
		Pipe(grep, "-Eo", "v"+tagPrefix+".[0-9]*$").
		RunSilentSuccessOutput()
	if err == nil {
		logrus.Warnf(
			"Found existing tag(s) on release branch: %v",
			strings.Join(strings.Fields(lsRemoteTags.OutputTrimNL()), ", "),
		)
		logrus.Infof("We’re all set, doing nothing")
		return nil
	}

	// Checkout the release branch
	repo, err := kgit.OpenRepo(".")
	if err != nil {
		return errors.Wrap(err, "unable to open this repository")
	}
	if dryRun {
		logrus.Info("Setting repository to only do a dry-run")
		repo.SetDry()
	}
	currentBranch, err := repo.CurrentBranch()
	if err != nil {
		return errors.Wrap(err, "unable to get current branch")
	}
	logrus.Infof("Checking out branch: %s", latestReleaseBranch)
	if err := repo.Checkout(latestReleaseBranch); err != nil {
		return errors.Wrapf(err,
			"unable to checkout release branch %s", latestReleaseBranch,
		)
	}
	defer func() {
		logrus.Infof("Checking out branch: %s", currentBranch)
		err = repo.Checkout(currentBranch)
	}()

	// Merge the latest master
	mergeTarget := kgit.Remotify(kgit.Master)
	if err := repo.Merge(mergeTarget); err != nil {
		return errors.Wrapf(err,
			"unable to merge %s into release branch", mergeTarget,
		)
	}

	// Push the changes
	if err := repo.Push(latestReleaseBranch); err != nil {
		return errors.Wrap(err, "unable to push to remote branch")
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/mitchellh/packer/common"
	"github.com/mitchellh/packer/helper/config"
	"github.com/mitchellh/packer/packer"
	"github.com/mitchellh/packer/template/interpolate"
)

type Config struct {
	Region              string        `mapstructure:"region"`
	Bucket              string        `mapstructure:"bucket"`
	ManifestPath        string        `mapstructure:"manifest"`
	BoxName             string        `mapstructure:"box_name"`
	BoxDir              string        `mapstructure:"box_dir"`
	Version             string        `mapstructure:"version"`
	ACL                 string        `mapstructure:"acl"`
	AccessKey           string        `mapstructure:"access_key_id"`
	SecretKey           string        `mapstructure:"secret_key"`
	SignedExpiry        time.Duration `mapstructure:"signed_expiry"`
	common.PackerConfig `mapstructure:",squash"`

	ctx interpolate.Context
}

type PostProcessor struct {
	config  Config
	session *session.Session
	s3      *s3.S3
}

func (p *PostProcessor) Configure(raws ...interface{}) error {
	err := config.Decode(&p.config, &config.DecodeOpts{
		Interpolate:        true,
		InterpolateContext: &p.config.ctx,
		InterpolateFilter: &interpolate.RenderFilter{
			Exclude: []string{"output"},
		},
	}, raws...)
	if err != nil {
		return err
	}

	errs := new(packer.MultiError)
	// required configuration
	templates := map[string]*string{
		"region":   &p.config.Region,
		"bucket":   &p.config.Bucket,
		"manifest": &p.config.ManifestPath,
		"box_name": &p.config.BoxName,
		"box_dir":  &p.config.BoxDir,
	}

	for key, ptr := range templates {
		if *ptr == "" {
			errs = packer.MultiErrorAppend(errs, fmt.Errorf("vagrant-s3 %s must be set", key))
		}
	}

	// Template process
	for key, ptr := range templates {
		if err = interpolate.Validate(*ptr, &p.config.ctx); err != nil {
			errs = packer.MultiErrorAppend(
				errs, fmt.Errorf("Error parsing %s template: %s", key, err))
		}
	}

	// create a session and an S3 service
	accessKey := p.config.AccessKey
	secretKey := p.config.SecretKey

	p.session = session.New(&aws.Config{
		Region:      aws.String(p.config.Region),
		Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""),
	})

	p.s3 = s3.New(p.session)

	// check that we have permission to access the bucket
	_, err = p.s3.HeadBucket(&s3.HeadBucketInput{
		Bucket: aws.String(p.config.Bucket),
	})

	if err != nil {
		errs = packer.MultiErrorAppend(errs, fmt.Errorf("Unable to access the bucket %s, make sure your credentials are valid and have sufficient permissions", p.config.Bucket))
	}

	if p.config.ACL == "" {
		p.config.ACL = "public-read"
	}

	if len(errs.Errors) > 0 {
		return errs
	}

	return nil
}

func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) {
	// Only accept input from the vagrant post-processor
	if artifact.BuilderId() != "mitchellh.post-processor.vagrant" {
		return nil, false, fmt.Errorf("Unknown artifact type, requires box from vagrant post-processor: %s", artifact.BuilderId())
	}

	// Assume there is only one .box file to upload
	box := artifact.Files()[0]
	if !strings.HasSuffix(box, ".box") {
		return nil, false, fmt.Errorf("Unknown files in artifact from vagrant post-processor: %s", artifact.Files())
	}

	provider := providerFromBuilderName(artifact.Id())
	ui.Say(fmt.Sprintf("Preparing to upload box for '%s' provider to S3 bucket '%s'", provider, p.config.Bucket))

	// determine box size
	boxStat, err := os.Stat(box)
	if err != nil {
		return nil, false, err
	}
	ui.Message(fmt.Sprintf("Box to upload: %s (%d bytes)", box, boxStat.Size()))

	// determine version
	version, err := p.determineVersion(p.config.Version)
	if err != nil {
		return nil, false, err
	}
	ui.Message(fmt.Sprintf("No version defined, using %s as new version", version))

	// generate the path to store the box in S3
	boxPath := fmt.Sprintf("%s/%s/%s", p.config.BoxDir, version, path.Base(box))

	ui.Message("Generating checksum")
	checksum, err := sum256(box)
	if err != nil {
		return nil, false, err
	}
	ui.Message(fmt.Sprintf("Checksum is %s", checksum))

	// upload the box to S3
	ui.Message(fmt.Sprintf("Uploading box to S3: %s", boxPath))
	err = p.uploadBox(box, boxPath)

	if err != nil {
		return nil, false, err
	}

	// get the latest manifest so we can add to it
	ui.Message("Fetching latest manifest")
	manifest, err := p.getManifest()
	if err != nil {
		return nil, false, err
	}

	ui.Message(fmt.Sprintf("Adding %s %s box to manifest", provider, version))
	var url string
	if p.config.SignedExpiry == 0 {
		url = generateS3Url(p.config.Region, p.config.Bucket, boxPath)
	} else {
		// fetch the new object
		boxObject, _ := p.s3.GetObjectRequest(&s3.GetObjectInput{
			Bucket: aws.String(p.config.Bucket),
			Key:    aws.String(boxPath),
		})

		url, err = boxObject.Presign(p.config.SignedExpiry)

		if err != nil {
			return nil, false, err
		}
	}
	if err := manifest.add(version, &Provider{
		Name:         provider,
		Url:          url,
		ChecksumType: "sha256",
		Checksum:     checksum,
	}); err != nil {
		return nil, false, err
	}

	ui.Message(fmt.Sprintf("Uploading the manifest: %s", p.config.ManifestPath))
	if err := p.putManifest(manifest); err != nil {
		return nil, false, err
	}

	return &Artifact{generateS3Url(p.config.Region, p.config.Bucket, p.config.ManifestPath)}, true, nil
}

func (p *PostProcessor) determineVersion(configVersion string) (string, error) {
	version := configVersion

	if version == "" {
		// get the next version based on the existing manifest
		if manifest, err := p.getManifest(); err != nil {
			return "", err
		} else {
			version = manifest.getNextVersion()
		}
	}

	return version, nil
}

func (p *PostProcessor) uploadBox(box, boxPath string) error {
	// open the file for reading
	file, err := os.Open(box)
	if err != nil {
		return err
	}
	defer file.Close()

	// upload the file
	uploader := s3manager.NewUploader(p.session, func(u *s3manager.Uploader) {
		u.PartSize = 1024 * 1024 * 64
	})

	_, err = uploader.Upload(&s3manager.UploadInput{
		Body:   file,
		Bucket: aws.String(p.config.Bucket),
		Key:    aws.String(boxPath),
		ACL:    aws.String(p.config.ACL),
	})

	return err
}

func (p *PostProcessor) getManifest() (*Manifest, error) {
	result, err := p.s3.GetObject(&s3.GetObjectInput{
		Bucket: aws.String(p.config.Bucket),
		Key:    aws.String(p.config.ManifestPath),
	})

	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == "NoSuchKey" {
				return &Manifest{Name: p.config.BoxName}, nil
			}
		}
		return nil, err
	}

	defer result.Body.Close()

	manifest := &Manifest{}
	if err := json.NewDecoder(result.Body).Decode(manifest); err != nil {
		return nil, err
	}
	return manifest, nil
}

func (p *PostProcessor) putManifest(manifest *Manifest) error {
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(manifest); err != nil {
		return err
	}

	_, err := p.s3.PutObject(&s3.PutObjectInput{
		Body:        strings.NewReader(buf.String()),
		Bucket:      aws.String(p.config.Bucket),
		Key:         aws.String(p.config.ManifestPath),
		ContentType: aws.String("application/json"),
		ACL:         aws.String(p.config.ACL),
	})

	if err != nil {
		return err
	}

	return nil
}

func generateS3Url(region, bucket, key string) string {
	return fmt.Sprintf("https://s3-%s.amazonaws.com/%s/%s", region, bucket, key)
}

// calculates a sha256 checksum of the file
func sum256(filePath string) (string, error) {
	// open the file for reading
	file, err := os.Open(filePath)

	if err != nil {
		return "", err
	}

	defer file.Close()

	h := sha256.New()
	if _, err := io.Copy(h, file); err != nil {
		return "", err
	}
	return hex.EncodeToString(h.Sum(nil)), nil
}

// converts a packer builder name to the corresponding vagrant provider
func providerFromBuilderName(name string) string {
	switch name {
	case "aws":
		return "aws"
	case "digitalocean":
		return "digitalocean"
	case "virtualbox":
		return "virtualbox"
	case "vmware":
		return "vmware_desktop"
	case "parallels":
		return "parallels"
	default:
		return name
	}
}
<commit_msg>Don't print "No version defined" when a version was actually defined<commit_after>package main

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/mitchellh/packer/common"
	"github.com/mitchellh/packer/helper/config"
	"github.com/mitchellh/packer/packer"
	"github.com/mitchellh/packer/template/interpolate"
)

type Config struct {
	Region              string        `mapstructure:"region"`
	Bucket              string        `mapstructure:"bucket"`
	ManifestPath        string        `mapstructure:"manifest"`
	BoxName             string        `mapstructure:"box_name"`
	BoxDir              string        `mapstructure:"box_dir"`
	Version             string        `mapstructure:"version"`
	ACL                 string        `mapstructure:"acl"`
	AccessKey           string        `mapstructure:"access_key_id"`
	SecretKey           string        `mapstructure:"secret_key"`
	SignedExpiry        time.Duration `mapstructure:"signed_expiry"`
	common.PackerConfig `mapstructure:",squash"`

	ctx interpolate.Context
}

type PostProcessor struct {
	config  Config
	session *session.Session
	s3      *s3.S3
}

func (p *PostProcessor) Configure(raws ...interface{}) error {
	err := config.Decode(&p.config, &config.DecodeOpts{
		Interpolate:        true,
		InterpolateContext: &p.config.ctx,
		InterpolateFilter: &interpolate.RenderFilter{
			Exclude: []string{"output"},
		},
	}, raws...)
	if err != nil {
		return err
	}

	errs := new(packer.MultiError)
	// required configuration
	templates := map[string]*string{
		"region":   &p.config.Region,
		"bucket":   &p.config.Bucket,
		"manifest": &p.config.ManifestPath,
		"box_name": &p.config.BoxName,
		"box_dir":  &p.config.BoxDir,
	}

	for key, ptr := range templates {
		if *ptr == "" {
			errs = packer.MultiErrorAppend(errs, fmt.Errorf("vagrant-s3 %s must be set", key))
		}
	}

	// Template process
	for key, ptr := range templates {
		if err = interpolate.Validate(*ptr, &p.config.ctx); err != nil {
			errs = packer.MultiErrorAppend(
				errs, fmt.Errorf("Error parsing %s template: %s", key, err))
		}
	}

	// create a session and an S3 service
	accessKey := p.config.AccessKey
	secretKey := p.config.SecretKey

	p.session = session.New(&aws.Config{
		Region:      aws.String(p.config.Region),
		Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""),
	})

	p.s3 = s3.New(p.session)

	// check that we have permission to access the bucket
	_, err = p.s3.HeadBucket(&s3.HeadBucketInput{
		Bucket: aws.String(p.config.Bucket),
	})

	if err != nil {
		errs = packer.MultiErrorAppend(errs, fmt.Errorf("Unable to access the bucket %s, make sure your credentials are valid and have sufficient permissions", p.config.Bucket))
	}

	if p.config.ACL == "" {
		p.config.ACL = "public-read"
	}

	if len(errs.Errors) > 0 {
		return errs
	}

	return nil
}

func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) {
	// Only accept input from the vagrant post-processor
	if artifact.BuilderId() != "mitchellh.post-processor.vagrant" {
		return nil, false, fmt.Errorf("Unknown artifact type, requires box from vagrant post-processor: %s", artifact.BuilderId())
	}

	// Assume there is only one .box file to upload
	box := artifact.Files()[0]
	if !strings.HasSuffix(box, ".box") {
		return nil, false, fmt.Errorf("Unknown files in artifact from vagrant post-processor: %s", artifact.Files())
	}

	provider := providerFromBuilderName(artifact.Id())
	ui.Say(fmt.Sprintf("Preparing to upload box for '%s' provider to S3 bucket '%s'", provider, p.config.Bucket))

	// determine box size
	boxStat, err := os.Stat(box)
	if err != nil {
		return nil, false, err
	}
	ui.Message(fmt.Sprintf("Box to upload: %s (%d bytes)", box, boxStat.Size()))

	// determine version
	version := p.config.Version

	if version == "" {
		version, err = p.determineVersion()
		if err != nil {
			return nil, false, err
		}

		ui.Message(fmt.Sprintf("No version defined, using %s as new version", version))
	} else {
		ui.Message(fmt.Sprintf("Using %s as new version", version))
	}

	// generate the path to store the box in S3
	boxPath := fmt.Sprintf("%s/%s/%s", p.config.BoxDir, version, path.Base(box))

	ui.Message("Generating checksum")
	checksum, err := sum256(box)
	if err != nil {
		return nil, false, err
	}
	ui.Message(fmt.Sprintf("Checksum is %s", checksum))

	// upload the box to S3
	ui.Message(fmt.Sprintf("Uploading box to S3: %s", boxPath))
	err = p.uploadBox(box, boxPath)

	if err != nil {
		return nil, false, err
	}

	// get the latest manifest so we can add to it
	ui.Message("Fetching latest manifest")
	manifest, err := p.getManifest()
	if err != nil {
		return nil, false, err
	}

	ui.Message(fmt.Sprintf("Adding %s %s box to manifest", provider, version))
	var url string
	if p.config.SignedExpiry == 0 {
		url = generateS3Url(p.config.Region, p.config.Bucket, boxPath)
	} else {
		// fetch the new object
		boxObject, _ := p.s3.GetObjectRequest(&s3.GetObjectInput{
			Bucket: aws.String(p.config.Bucket),
			Key:    aws.String(boxPath),
		})

		url, err = boxObject.Presign(p.config.SignedExpiry)

		if err != nil {
			return nil, false, err
		}
	}
	if err := manifest.add(version, &Provider{
		Name:         provider,
		Url:          url,
		ChecksumType: "sha256",
		Checksum:     checksum,
	}); err != nil {
		return nil, false, err
	}

	ui.Message(fmt.Sprintf("Uploading the manifest: %s", p.config.ManifestPath))
	if err := p.putManifest(manifest); err != nil {
		return nil, false, err
	}

	return &Artifact{generateS3Url(p.config.Region, p.config.Bucket, p.config.ManifestPath)}, true, nil
}

func (p *PostProcessor) determineVersion() (string, error) {
	// get the next version based on the existing manifest
	if manifest, err := p.getManifest(); err != nil {
		return "", err
	} else {
		return manifest.getNextVersion(), nil
	}
}

func (p *PostProcessor) uploadBox(box, boxPath string) error {
	// open the file for reading
	file, err := os.Open(box)
	if err != nil {
		return err
	}
	defer file.Close()

	// upload the file
	uploader := s3manager.NewUploader(p.session, func(u *s3manager.Uploader) {
		u.PartSize = 1024 * 1024 * 64
	})

	_, err = uploader.Upload(&s3manager.UploadInput{
		Body:   file,
		Bucket: aws.String(p.config.Bucket),
		Key:    aws.String(boxPath),
		ACL:    aws.String(p.config.ACL),
	})

	return err
}

func (p *PostProcessor) getManifest() (*Manifest, error) {
	result, err := p.s3.GetObject(&s3.GetObjectInput{
		Bucket: aws.String(p.config.Bucket),
		Key:    aws.String(p.config.ManifestPath),
	})

	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == "NoSuchKey" {
				return &Manifest{Name: p.config.BoxName}, nil
			}
		}
		return nil, err
	}

	defer result.Body.Close()

	manifest := &Manifest{}
	if err := json.NewDecoder(result.Body).Decode(manifest); err != nil {
		return nil, err
	}
	return manifest, nil
}

func (p *PostProcessor) putManifest(manifest *Manifest) error {
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(manifest); err != nil {
		return err
	}

	_, err := p.s3.PutObject(&s3.PutObjectInput{
		Body:        strings.NewReader(buf.String()),
		Bucket:      aws.String(p.config.Bucket),
		Key:         aws.String(p.config.ManifestPath),
		ContentType: aws.String("application/json"),
		ACL:         aws.String(p.config.ACL),
	})

	if err != nil {
		return err
	}

	return nil
}

func generateS3Url(region, bucket, key string) string {
	return fmt.Sprintf("https://s3-%s.amazonaws.com/%s/%s", region, bucket, key)
}

// calculates a sha256 checksum of the file
func sum256(filePath string) (string, error) {
	// open the file for reading
	file, err := os.Open(filePath)

	if err != nil {
		return "", err
	}

	defer file.Close()

	h := sha256.New()
	if _, err := io.Copy(h, file); err != nil {
		return "", err
	}
	return hex.EncodeToString(h.Sum(nil)), nil
}

// converts a packer builder name to the corresponding vagrant provider
func providerFromBuilderName(name string) string {
	switch name {
	case "aws":
		return "aws"
	case "digitalocean":
		return "digitalocean"
	case "virtualbox":
		return "virtualbox"
	case "vmware":
		return "vmware_desktop"
	case "parallels":
		return "parallels"
	default:
		return name
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/hashicorp/packer/common"
	"github.com/hashicorp/packer/helper/config"
	"github.com/hashicorp/packer/packer"
	"github.com/hashicorp/packer/template/interpolate"
)

type Config struct {
	Region              string        `mapstructure:"region"`
	Bucket              string        `mapstructure:"bucket"`
	CloudFront          string        `mapstructure:"cloudfront"`
	ManifestPath        string        `mapstructure:"manifest"`
	BoxName             string        `mapstructure:"box_name"`
	BoxDir              string        `mapstructure:"box_dir"`
	Version             string        `mapstructure:"version"`
	ACL                 string        `mapstructure:"acl"`
	CredentialFile      string        `mapstructure:"credentials"`
	CredentialProfile   string        `mapstructure:"profile"`
	AccessKey           string        `mapstructure:"access_key_id"`
	SecretKey           string        `mapstructure:"secret_key"`
	SessionToken        string        `mapstructure:"session_token"`
	SignedExpiry        time.Duration `mapstructure:"signed_expiry"`
	StorageClass        string        `mapstructure:"storage_class"`
	common.PackerConfig `mapstructure:",squash"`

	ctx interpolate.Context
}

type PostProcessor struct {
	config  Config
	session *session.Session
	s3      *s3.S3
}

func (p *PostProcessor) Configure(raws ...interface{}) error {
	err := config.Decode(&p.config, &config.DecodeOpts{
		Interpolate:        true,
		InterpolateContext: &p.config.ctx,
		InterpolateFilter: &interpolate.RenderFilter{
			Exclude: []string{"output"},
		},
	}, raws...)
	if err != nil {
		return err
	}

	errs := new(packer.MultiError)
	// required configuration
	templates := map[string]*string{
		"region":   &p.config.Region,
		"bucket":   &p.config.Bucket,
		"manifest": &p.config.ManifestPath,
		"box_name": &p.config.BoxName,
		"box_dir":  &p.config.BoxDir,
	}

	for key, ptr := range templates {
		if *ptr == "" {
			errs = packer.MultiErrorAppend(errs, fmt.Errorf("vagrant-s3 %s must be set", key))
		}
	}

	// Template process
	for key, ptr := range templates {
		if err = interpolate.Validate(*ptr, &p.config.ctx); err != nil {
			errs = packer.MultiErrorAppend(
				errs, fmt.Errorf("Error parsing %s template: %s", key, err))
		}
	}

	var cred *credentials.Credentials = nil // nil credentials use the default aws sdk credential chain

	if p.config.AccessKey != "" && p.config.SecretKey != "" {
		// StaticProvider if both access id and secret are defined
		// Environmental variables used:
		// $AWS_SESSION_TOKEN
		cred = credentials.NewCredentials(&credentials.StaticProvider{
			Value: credentials.Value{
				AccessKeyID:     p.config.AccessKey,
				SecretAccessKey: p.config.SecretKey,
				SessionToken:    p.config.SessionToken,
			},
		})
	} else if p.config.CredentialFile != "" || p.config.CredentialProfile != "" {
		// SharedCredentialProvider if either credentials file or a profile is defined
		// Environmental variables used:
		// $AWS_SHARED_CREDENTIALS_FILE ("$HOME/.aws/credentials" if unset)
		// $AWS_PROFILE ("default" if unset)
		cred = credentials.NewCredentials(&credentials.SharedCredentialsProvider{
			Filename: p.config.CredentialFile,
			Profile:  p.config.CredentialProfile,
		})
	} else {
		// EnvProvider as fallback if none of the above matched
		// Environmental variables used:
		// $AWS_ACCESS_KEY_ID ($AWS_ACCESS_KEY if unset)
		// $AWS_SECRET_ACCESS_KEY ($AWS_SECRET_KEY if unset)
		// $AWS_SESSION_TOKEN
		cred = credentials.NewCredentials(&credentials.EnvProvider{})
	}

	p.session = session.New(&aws.Config{
		Region:      aws.String(p.config.Region),
		Credentials: cred,
	})

	p.s3 = s3.New(p.session)

	// check that we have permission to access the bucket
	_, err = p.s3.HeadBucket(&s3.HeadBucketInput{
		Bucket: aws.String(p.config.Bucket),
	})

	if err != nil {
		errs = packer.MultiErrorAppend(errs, fmt.Errorf("Unable to access the bucket %s:\n%s\nMake sure your credentials are valid and have sufficient permissions", p.config.Bucket, err))
	}

	if p.config.ACL == "" {
		p.config.ACL = "public-read"
	}

	// set default storage class
	if p.config.StorageClass == "" {
		p.config.StorageClass = "STANDARD"
	}

	if len(errs.Errors) > 0 {
		return errs
	}

	return nil
}

func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) {
	// Only accept input from the vagrant post-processor
	if artifact.BuilderId() != "mitchellh.post-processor.vagrant" {
		return nil, false, fmt.Errorf("Unknown artifact type, requires box from vagrant post-processor: %s", artifact.BuilderId())
	}

	// Assume there is only one .box file to upload
	box := artifact.Files()[0]
	if !strings.HasSuffix(box, ".box") {
		return nil, false, fmt.Errorf("Unknown files in artifact from vagrant post-processor: %s", artifact.Files())
	}

	provider := providerFromBuilderName(artifact.Id())
	ui.Say(fmt.Sprintf("Preparing to upload box for '%s' provider to S3 bucket '%s'", provider, p.config.Bucket))

	// determine box size
	boxStat, err := os.Stat(box)
	if err != nil {
		return nil, false, err
	}
	ui.Message(fmt.Sprintf("Box to upload: %s (%d bytes)", box, boxStat.Size()))

	// determine version
	version := p.config.Version

	if version == "" {
		version, err = p.determineVersion()
		if err != nil {
			return nil, false, err
		}

		ui.Message(fmt.Sprintf("No version defined, using %s as new version", version))
	} else {
		ui.Message(fmt.Sprintf("Using %s as new version", version))
	}

	// generate the path to store the box in S3
	boxPath := fmt.Sprintf("%s/%s/%s", p.config.BoxDir, version, path.Base(box))

	ui.Message("Generating checksum")
	checksum, err := sum256(box)
	if err != nil {
		return nil, false, err
	}
	ui.Message(fmt.Sprintf("Checksum is %s", checksum))

	// upload the box to S3
	ui.Message(fmt.Sprintf("Uploading box to S3: %s", boxPath))

	start := time.Now()
	err = p.uploadBox(box, boxPath)

	if err != nil {
		return nil, false, err
	} else {
		elapsed := time.Since(start)
		ui.Message(fmt.Sprintf("Box upload took: %s", elapsed))
	}

	// get the latest manifest so we can add to it
	ui.Message("Fetching latest manifest")
	manifest, err := p.getManifest()
	if err != nil {
		return nil, false, err
	}

	ui.Message(fmt.Sprintf("Adding %s %s box to manifest", provider, version))
	var url string
	if p.config.SignedExpiry == 0 {
		url = generateS3Url(p.config.Region, p.config.Bucket, p.config.CloudFront, boxPath)
	} else {
		// fetch the new object
		boxObject, _ := p.s3.GetObjectRequest(&s3.GetObjectInput{
			Bucket: aws.String(p.config.Bucket),
			Key:    aws.String(boxPath),
		})

		url, err = boxObject.Presign(p.config.SignedExpiry)

		if err != nil {
			return nil, false, err
		}
	}
	if err := manifest.add(version, &Provider{
		Name:         provider,
		Url:          url,
		ChecksumType: "sha256",
		Checksum:     checksum,
	}); err != nil {
		return nil, false, err
	}

	ui.Message(fmt.Sprintf("Uploading the manifest: %s", p.config.ManifestPath))
	if err := p.putManifest(manifest); err != nil {
		return nil, false, err
	}

	return &Artifact{generateS3Url(p.config.Region, p.config.Bucket, p.config.CloudFront, p.config.ManifestPath)}, true, nil
}

func (p *PostProcessor) determineVersion() (string, error) {
	// get the next version based on the existing manifest
	if manifest, err := p.getManifest(); err != nil {
		return "", err
	} else {
		return manifest.getNextVersion(), nil
	}
}

func (p *PostProcessor) uploadBox(box, boxPath string) error {
	// open the file for reading
	file, err := os.Open(box)
	if err != nil {
		return err
	}
	defer file.Close()

	// upload the file
	uploader := s3manager.NewUploader(p.session, func(u *s3manager.Uploader) {
		u.PartSize = 1024 * 1024 * 64
	})

	_, err = uploader.Upload(&s3manager.UploadInput{
		Body:         file,
		Bucket:       aws.String(p.config.Bucket),
		Key:          aws.String(boxPath),
		ACL:          aws.String(p.config.ACL),
		StorageClass: aws.String(p.config.StorageClass),
	})

	return err
}

func (p *PostProcessor) getManifest() (*Manifest, error) {
	result, err := p.s3.GetObject(&s3.GetObjectInput{
		Bucket: aws.String(p.config.Bucket),
		Key:    aws.String(p.config.ManifestPath),
	})

	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == "NoSuchKey" {
				return &Manifest{Name: p.config.BoxName}, nil
			}
		}
		return nil, err
	}

	defer result.Body.Close()

	manifest := &Manifest{}
	if err := json.NewDecoder(result.Body).Decode(manifest); err != nil {
		return nil, err
	}
	return manifest, nil
}

func (p *PostProcessor) putManifest(manifest *Manifest) error {
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(manifest); err != nil {
		return err
	}

	_, err := p.s3.PutObject(&s3.PutObjectInput{
		Body:        strings.NewReader(buf.String()),
		Bucket:      aws.String(p.config.Bucket),
		Key:         aws.String(p.config.ManifestPath),
		ContentType: aws.String("application/json"),
		ACL:         aws.String(p.config.ACL),
	})

	if err != nil {
		return err
	}

	return nil
}

func generateS3Url(region, bucket, cloudFront, key string) string {
	if cloudFront != "" {
		return fmt.Sprintf("https://%s/%s", cloudFront, key)
	}

	if region == "us-east-1" {
		return fmt.Sprintf("https://s3.amazonaws.com/%s/%s", bucket, key)
	}

	return fmt.Sprintf("https://s3-%s.amazonaws.com/%s/%s", region, bucket, key)
}

// calculates a sha256 checksum of the file
func sum256(filePath string) (string, error) {
	// open the file for reading
	file, err := os.Open(filePath)

	if err != nil {
		return "", err
	}

	defer file.Close()

	h := sha256.New()
	if _, err := io.Copy(h, file); err != nil {
		return "", err
	}
	return hex.EncodeToString(h.Sum(nil)), nil
}

// converts a packer builder name to the corresponding vagrant provider
func providerFromBuilderName(name string) string {
	switch name {
	case "aws":
		return "aws"
	case "digitalocean":
		return "digitalocean"
	case "virtualbox":
		return "virtualbox"
	case "vmware":
		return "vmware_desktop"
	case "parallels":
		return "parallels"
	default:
		return name
	}
}
<commit_msg>Make PartSize and Concurrency configurable<commit_after>package main

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/hashicorp/packer/common"
	"github.com/hashicorp/packer/helper/config"
	"github.com/hashicorp/packer/packer"
	"github.com/hashicorp/packer/template/interpolate"
)

type Config struct {
	Region              string        `mapstructure:"region"`
	Bucket              string        `mapstructure:"bucket"`
	CloudFront          string        `mapstructure:"cloudfront"`
	ManifestPath        string        `mapstructure:"manifest"`
	BoxName             string        `mapstructure:"box_name"`
	BoxDir              string        `mapstructure:"box_dir"`
	Version             string        `mapstructure:"version"`
	ACL                 string        `mapstructure:"acl"`
	CredentialFile      string        `mapstructure:"credentials"`
	CredentialProfile   string        `mapstructure:"profile"`
	AccessKey           string        `mapstructure:"access_key_id"`
	SecretKey           string        `mapstructure:"secret_key"`
	SessionToken        string        `mapstructure:"session_token"`
	SignedExpiry        time.Duration `mapstructure:"signed_expiry"`
	StorageClass        string        `mapstructure:"storage_class"`
	PartSize            int64         `mapstructure:"part_size"`
	Concurrency         int           `mapstructure:"concurrency"`
	common.PackerConfig `mapstructure:",squash"`

	ctx interpolate.Context
}

type PostProcessor struct {
	config  Config
	session *session.Session
	s3      *s3.S3
}

func (p *PostProcessor) Configure(raws ...interface{}) error {
	err := config.Decode(&p.config, &config.DecodeOpts{
		Interpolate:        true,
		InterpolateContext: &p.config.ctx,
		InterpolateFilter: &interpolate.RenderFilter{
			Exclude: []string{"output"},
		},
	}, raws...)
	if err != nil {
		return err
	}

	errs := new(packer.MultiError)
	// required configuration
	templates := map[string]*string{
		"region":   &p.config.Region,
		"bucket":   &p.config.Bucket,
		"manifest": &p.config.ManifestPath,
		"box_name": &p.config.BoxName,
		"box_dir":  &p.config.BoxDir,
	}

	for key, ptr := range templates {
		if *ptr == "" {
			errs = packer.MultiErrorAppend(errs, fmt.Errorf("vagrant-s3 %s must be set", key))
		}
	}

	// Template process
	for key, ptr := range templates {
		if err = interpolate.Validate(*ptr, &p.config.ctx); err != nil {
			errs = packer.MultiErrorAppend(
				errs, fmt.Errorf("Error parsing %s template: %s", key, err))
		}
	}

	var cred *credentials.Credentials = nil // nil credentials use the default aws sdk credential chain

	if p.config.AccessKey != "" && p.config.SecretKey != "" {
		// StaticProvider if both access id and secret are defined
		// Environmental variables used:
		// $AWS_SESSION_TOKEN
		cred = credentials.NewCredentials(&credentials.StaticProvider{
			Value: credentials.Value{
				AccessKeyID:     p.config.AccessKey,
				SecretAccessKey: p.config.SecretKey,
				SessionToken:    p.config.SessionToken,
			},
		})
	} else if p.config.CredentialFile != "" || p.config.CredentialProfile != "" {
		// SharedCredentialProvider if either credentials file or a profile is defined
		// Environmental variables used:
		// $AWS_SHARED_CREDENTIALS_FILE ("$HOME/.aws/credentials" if unset)
		// $AWS_PROFILE ("default" if unset)
		cred = credentials.NewCredentials(&credentials.SharedCredentialsProvider{
			Filename: p.config.CredentialFile,
			Profile:  p.config.CredentialProfile,
		})
	} else {
		// EnvProvider as fallback if none of the above matched
		// Environmental variables used:
		// $AWS_ACCESS_KEY_ID ($AWS_ACCESS_KEY if unset)
		// $AWS_SECRET_ACCESS_KEY ($AWS_SECRET_KEY if unset)
		// $AWS_SESSION_TOKEN
		cred = credentials.NewCredentials(&credentials.EnvProvider{})
	}

	p.session = session.New(&aws.Config{
		Region:      aws.String(p.config.Region),
		Credentials: cred,
	})

	p.s3 = s3.New(p.session)

	// check that we have permission to access the bucket
	_, err = p.s3.HeadBucket(&s3.HeadBucketInput{
		Bucket: aws.String(p.config.Bucket),
	})

	if err != nil {
		errs = packer.MultiErrorAppend(errs, fmt.Errorf("Unable to access the bucket %s:\n%s\nMake sure your credentials are valid and have sufficient permissions", p.config.Bucket, err))
	}

	if p.config.ACL == "" {
		p.config.ACL = "public-read"
	}

	// set default storage class
	if p.config.StorageClass == "" {
		p.config.StorageClass = "STANDARD"
	}

	if p.config.PartSize == 0 {
		p.config.PartSize = s3manager.DefaultUploadPartSize
	}

	if p.config.Concurrency == 0 {
		p.config.Concurrency = s3manager.DefaultUploadConcurrency
	}

	if len(errs.Errors) > 0 {
		return errs
	}

	return nil
}

func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) {
	// Only accept input from the vagrant post-processor
	if artifact.BuilderId() != "mitchellh.post-processor.vagrant" {
		return nil, false, fmt.Errorf("Unknown artifact type, requires box from vagrant post-processor: %s", artifact.BuilderId())
	}

	// Assume there is only one .box file to upload
	box := artifact.Files()[0]
	if !strings.HasSuffix(box, ".box") {
		return nil, false, fmt.Errorf("Unknown files in artifact from vagrant post-processor: %s", artifact.Files())
	}

	provider := providerFromBuilderName(artifact.Id())
	ui.Say(fmt.Sprintf("Preparing to upload box for '%s' provider to S3 bucket '%s'", provider, p.config.Bucket))

	// determine box size
	boxStat, err := os.Stat(box)
	if err != nil {
		return nil, false, err
	}
	ui.Message(fmt.Sprintf("Box to upload: %s (%d bytes)", box, boxStat.Size()))

	// determine version
	version := p.config.Version

	if version == "" {
		version, err = p.determineVersion()
		if err != nil {
			return nil, false, err
		}

		ui.Message(fmt.Sprintf("No version defined, using %s as new version", version))
	} else {
		ui.Message(fmt.Sprintf("Using %s as new version", version))
	}

	// generate the path to store the box in S3
	boxPath := fmt.Sprintf("%s/%s/%s", p.config.BoxDir, version, path.Base(box))

	ui.Message("Generating checksum")
	checksum, err := sum256(box)
	if err != nil {
		return nil, false, err
	}
	ui.Message(fmt.Sprintf("Checksum is %s", checksum))

	// upload the box to S3
	ui.Message(fmt.Sprintf("Uploading box to S3: %s, PartSize: %d, Concurrency: %d", boxPath, p.config.PartSize, p.config.Concurrency))

	start := time.Now()
	err = p.uploadBox(box, boxPath)

	if err != nil {
		return nil, false, err
	} else {
		elapsed := time.Since(start)
		ui.Message(fmt.Sprintf("Box upload took: %s", elapsed))
	}

	// get the latest manifest so we can add to it
	ui.Message("Fetching latest manifest")
	manifest, err := p.getManifest()
	if err != nil {
		return nil, false, err
	}

	ui.Message(fmt.Sprintf("Adding %s %s box to manifest", provider, version))
	var url string
	if p.config.SignedExpiry == 0 {
		url = generateS3Url(p.config.Region, p.config.Bucket, p.config.CloudFront, boxPath)
	} else {
		// fetch the new object
		boxObject, _ := p.s3.GetObjectRequest(&s3.GetObjectInput{
			Bucket: aws.String(p.config.Bucket),
			Key:    aws.String(boxPath),
		})

		url, err = boxObject.Presign(p.config.SignedExpiry)

		if err != nil {
			return nil, false, err
		}
	}
	if err := manifest.add(version, &Provider{
		Name:         provider,
		Url:          url,
		ChecksumType: "sha256",
		Checksum:     checksum,
	}); err != nil {
		return nil, false, err
	}

	ui.Message(fmt.Sprintf("Uploading the manifest: %s", p.config.ManifestPath))
	if err := p.putManifest(manifest); err != nil {
		return nil, false, err
	}

	return &Artifact{generateS3Url(p.config.Region, p.config.Bucket, p.config.CloudFront, p.config.ManifestPath)}, true, nil
}

func (p *PostProcessor) determineVersion() (string, error) {
	// get the next version based on the existing manifest
	if manifest, err := p.getManifest(); err != nil {
		return "", err
	} else {
		return manifest.getNextVersion(), nil
	}
}

func (p *PostProcessor) uploadBox(box, boxPath string) error {
	// open the file for reading
	file, err := os.Open(box)
	if err != nil {
		return err
	}
	defer file.Close()

	// upload the file
	uploader := s3manager.NewUploader(p.session, func(u *s3manager.Uploader) {
		u.PartSize = p.config.PartSize
		u.Concurrency = p.config.Concurrency
	})

	_, err = uploader.Upload(&s3manager.UploadInput{
		Body:         file,
		Bucket:       aws.String(p.config.Bucket),
		Key:          aws.String(boxPath),
		ACL:          aws.String(p.config.ACL),
		StorageClass: aws.String(p.config.StorageClass),
	})

	return err
}

func (p *PostProcessor) getManifest() (*Manifest, error) {
	result, err := p.s3.GetObject(&s3.GetObjectInput{
		Bucket: aws.String(p.config.Bucket),
		Key:    aws.String(p.config.ManifestPath),
	})

	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == "NoSuchKey" {
				return &Manifest{Name: p.config.BoxName}, nil
			}
		}
		return nil, err
	}

	defer result.Body.Close()

	manifest := &Manifest{}
	if err := json.NewDecoder(result.Body).Decode(manifest); err != nil {
		return nil, err
	}
	return manifest, nil
}

func (p *PostProcessor) putManifest(manifest *Manifest) error {
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(manifest); err != nil {
		return err
	}

	_, err := p.s3.PutObject(&s3.PutObjectInput{
		Body:        strings.NewReader(buf.String()),
		Bucket:      aws.String(p.config.Bucket),
		Key:         aws.String(p.config.ManifestPath),
		ContentType: aws.String("application/json"),
		ACL:         aws.String(p.config.ACL),
	})

	if err != nil {
		return err
	}

	return nil
}

func generateS3Url(region, bucket, cloudFront, key string) string {
	if cloudFront != "" {
		return fmt.Sprintf("https://%s/%s", cloudFront, key)
	}

	if region == "us-east-1" {
		return fmt.Sprintf("https://s3.amazonaws.com/%s/%s", bucket, key)
	}

	return fmt.Sprintf("https://s3-%s.amazonaws.com/%s/%s", region, bucket, key)
}

// calculates a sha256 checksum of the file
func sum256(filePath string) (string, error) {
	// open the file for reading
	file, err := os.Open(filePath)

	if err != nil {
		return "", err
	}

	defer file.Close()

	h := sha256.New()
	if _, err := io.Copy(h, file); err != nil {
		return "", err
	}
	return hex.EncodeToString(h.Sum(nil)), nil
}

// converts a packer builder name to the corresponding vagrant provider
func providerFromBuilderName(name string) string {
	switch name {
	case "aws":
		return "aws"
	case "digitalocean":
		return "digitalocean"
	case "virtualbox":
		return "virtualbox"
	case "vmware":
		return "vmware_desktop"
	case "parallels":
		return "parallels"
	default:
		return name
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package integration_test

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"testing"

	"github.com/googlecloudplatform/gcsfuse/internal/wiring"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
)

func TestGcsfuse(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type GcsfuseTest struct {
	// Path to the gcsfuse binary.
	gcsfusePath string

	// A temporary directory into which a file system may be mounted. Removed in
	// TearDown.
	dir string
}

var _ SetUpInterface = &GcsfuseTest{}
var _ TearDownInterface = &GcsfuseTest{}

func init() { RegisterTestSuite(&GcsfuseTest{}) }

func (t *GcsfuseTest) SetUp(_ *TestInfo) {
	var err error
	t.gcsfusePath = path.Join(gBuildDir, "bin/gcsfuse")

	// Set up the temporary directory.
	t.dir, err = ioutil.TempDir("", "gcsfuse_test")
	AssertEq(nil, err)
}

func (t *GcsfuseTest) TearDown() {
	err := os.Remove(t.dir)
	AssertEq(nil, err)
}

// Call gcsfuse with the supplied args, waiting for it to mount. Return nil
// only if it mounts successfully.
func (t *GcsfuseTest) mount(args []string) (err error) {
	// Set up a pipe that gcsfuse can write to to tell us when it has
	// successfully mounted.
	statusR, statusW, err := os.Pipe()
	if err != nil {
		err = fmt.Errorf("Pipe: %v", err)
		return
	}

	// Run gcsfuse, writing the result of waiting for it to a channel.
	gcsfuseErr := make(chan error, 1)
	go func() {
		gcsfuseErr <- runGcsfuse(args, statusW)
	}()

	// In the background, wait for something to be written to the pipe.
	pipeErr := make(chan error, 1)
	go func() {
		defer statusR.Close()
		n, err := statusR.Read(make([]byte, 1))
		if n == 1 {
			pipeErr <- nil
			return
		}

		pipeErr <- fmt.Errorf("statusR.Read: %v", err)
	}()

	// Watch for a result from one of them.
	select {
	case err = <-gcsfuseErr:
		err = fmt.Errorf("gcsfuse: %v", err)
		return

	case err = <-pipeErr:
		if err == nil {
			// All is good.
			return
		}

		err = <-gcsfuseErr
		err = fmt.Errorf("gcsfuse after pipe error: %v", err)
		return
	}
}

// Run gcsfuse and wait for it to return. Hand it the supplied pipe to write
// into when it successfully mounts. This function takes responsibility for
// closing the write end of the pipe locally.
func runGcsfuse(args []string, statusW *os.File) (err error) {
	err = errors.New("TODO")
	return
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *GcsfuseTest) BadUsage() {
	testCases := []struct {
		args           []string
		expectedOutput string
	}{
		// Too few args
		0: {
			[]string{wiring.FakeBucket},
			"exactly two arguments",
		},

		// Too many args
		1: {
			[]string{wiring.FakeBucket, "a", "b"},
			"exactly two arguments",
		},

		// Unknown flag
		2: {
			[]string{"--tweak_frobnicator", wiring.FakeBucket, "a"},
			"not defined.*tweak_frobnicator",
		},
	}

	// Run each test case.
	for i, tc := range testCases {
		cmd := exec.Command(t.gcsfusePath)
		cmd.Args = append(cmd.Args, tc.args...)

		output, err := cmd.CombinedOutput()
		ExpectThat(err, Error(HasSubstr("exit status")), "case %d", i)
		ExpectThat(string(output), MatchesRegexp(tc.expectedOutput), "case %d", i)
	}
}

func (t *GcsfuseTest) ReadOnlyMode() {
	var err error

	// Mount.
	args := []string{"-o", "ro", wiring.FakeBucket, t.dir}

	err = t.mount(args)
	AssertEq(nil, err)

	// Check that the expected file is there (cf. the documentation on
	// wiring.FakeBucket).
	contents, err := ioutil.ReadFile(path.Join(t.dir, "foo"))
	AssertEq(nil, err)
	ExpectEq("taco", string(contents))

	// The implicit directory shouldn't be visible, since we don't have implicit
	// directories enabled.
	_, err = os.Lstat(path.Join(t.dir, "bar"))
	ExpectTrue(os.IsNotExist(err), "err: %v", err)

	// Writing to the file system should ail.
	err = ioutil.WriteFile(path.Join(t.dir, "blah"), []byte{}, 0400)
	ExpectThat(err, Error(HasSubstr("TODO")))
}

func (t *GcsfuseTest) ReadWriteMode() {
	AssertTrue(false, "TODO")
}

func (t *GcsfuseTest) FileAndDirModeFlags() {
	AssertTrue(false, "TODO")
}

func (t *GcsfuseTest) UidAndGidFlags() {
	AssertTrue(false, "TODO")
}

func (t *GcsfuseTest) ImplicitDirs() {
	AssertTrue(false, "TODO")
}

func (t *GcsfuseTest) VersionFlags() {
	AssertTrue(false, "TODO")
}

func (t *GcsfuseTest) HelpFlags() {
	AssertTrue(false, "TODO")
}
<commit_msg>GcsfuseTest.runGcsfuse<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package integration_test

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"testing"

	"github.com/googlecloudplatform/gcsfuse/internal/wiring"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
)

func TestGcsfuse(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type GcsfuseTest struct {
	// Path to the gcsfuse binary.
	gcsfusePath string

	// A temporary directory into which a file system may be mounted. Removed in
	// TearDown.
	dir string
}

var _ SetUpInterface = &GcsfuseTest{}
var _ TearDownInterface = &GcsfuseTest{}

func init() { RegisterTestSuite(&GcsfuseTest{}) }

func (t *GcsfuseTest) SetUp(_ *TestInfo) {
	var err error
	t.gcsfusePath = path.Join(gBuildDir, "bin/gcsfuse")

	// Set up the temporary directory.
	t.dir, err = ioutil.TempDir("", "gcsfuse_test")
	AssertEq(nil, err)
}

func (t *GcsfuseTest) TearDown() {
	err := os.Remove(t.dir)
	AssertEq(nil, err)
}

// Call gcsfuse with the supplied args, waiting for it to mount. Return nil
// only if it mounts successfully.
func (t *GcsfuseTest) mount(args []string) (err error) {
	// Set up a pipe that gcsfuse can write to to tell us when it has
	// successfully mounted.
	statusR, statusW, err := os.Pipe()
	if err != nil {
		err = fmt.Errorf("Pipe: %v", err)
		return
	}

	// Run gcsfuse, writing the result of waiting for it to a channel.
	gcsfuseErr := make(chan error, 1)
	go func() {
		gcsfuseErr <- t.runGcsfuse(args, statusW)
	}()

	// In the background, wait for something to be written to the pipe.
	pipeErr := make(chan error, 1)
	go func() {
		defer statusR.Close()
		n, err := statusR.Read(make([]byte, 1))
		if n == 1 {
			pipeErr <- nil
			return
		}

		pipeErr <- fmt.Errorf("statusR.Read: %v", err)
	}()

	// Watch for a result from one of them.
	select {
	case err = <-gcsfuseErr:
		err = fmt.Errorf("gcsfuse: %v", err)
		return

	case err = <-pipeErr:
		if err == nil {
			// All is good.
			return
		}

		err = <-gcsfuseErr
		err = fmt.Errorf("gcsfuse after pipe error: %v", err)
		return
	}
}

// Run gcsfuse and wait for it to return. Hand it the supplied pipe to write
// into when it successfully mounts. This function takes responsibility for
// closing the write end of the pipe locally.
func (t *GcsfuseTest) runGcsfuse(args []string, statusW *os.File) (err error) {
	defer statusW.Close()

	cmd := exec.Command(t.gcsfusePath)
	cmd.Args = append(cmd.Args, args...)
	cmd.ExtraFiles = []*os.File{statusW}
	cmd.Env = []string{"STATUS_PIPE=3"}

	output, err := cmd.CombinedOutput()
	if err != nil {
		err = fmt.Errorf("%v\nOutput:\n%s", err, output)
		return
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *GcsfuseTest) BadUsage() {
	testCases := []struct {
		args           []string
		expectedOutput string
	}{
		// Too few args
		0: {
			[]string{wiring.FakeBucket},
			"exactly two arguments",
		},

		// Too many args
		1: {
			[]string{wiring.FakeBucket, "a", "b"},
			"exactly two arguments",
		},

		// Unknown flag
		2: {
			[]string{"--tweak_frobnicator", wiring.FakeBucket, "a"},
			"not defined.*tweak_frobnicator",
		},
	}

	// Run each test case.
	for i, tc := range testCases {
		cmd := exec.Command(t.gcsfusePath)
		cmd.Args = append(cmd.Args, tc.args...)

		output, err := cmd.CombinedOutput()
		ExpectThat(err, Error(HasSubstr("exit status")), "case %d", i)
		ExpectThat(string(output), MatchesRegexp(tc.expectedOutput), "case %d", i)
	}
}

func (t *GcsfuseTest) ReadOnlyMode() {
	var err error

	// Mount.
	args := []string{"-o", "ro", wiring.FakeBucket, t.dir}

	err = t.mount(args)
	AssertEq(nil, err)

	// Check that the expected file is there (cf. the documentation on
	// wiring.FakeBucket).
	contents, err := ioutil.ReadFile(path.Join(t.dir, "foo"))
	AssertEq(nil, err)
	ExpectEq("taco", string(contents))

	// The implicit directory shouldn't be visible, since we don't have implicit
	// directories enabled.
	_, err = os.Lstat(path.Join(t.dir, "bar"))
	ExpectTrue(os.IsNotExist(err), "err: %v", err)

	// Writing to the file system should ail.
	err = ioutil.WriteFile(path.Join(t.dir, "blah"), []byte{}, 0400)
	ExpectThat(err, Error(HasSubstr("TODO")))
}

func (t *GcsfuseTest) ReadWriteMode() {
	AssertTrue(false, "TODO")
}

func (t *GcsfuseTest) FileAndDirModeFlags() {
	AssertTrue(false, "TODO")
}

func (t *GcsfuseTest) UidAndGidFlags() {
	AssertTrue(false, "TODO")
}

func (t *GcsfuseTest) ImplicitDirs() {
	AssertTrue(false, "TODO")
}

func (t *GcsfuseTest) VersionFlags() {
	AssertTrue(false, "TODO")
}

func (t *GcsfuseTest) HelpFlags() {
	AssertTrue(false, "TODO")
}
<|endoftext|>


<commit_before>package golagraphite

import (
	"errors"
	"fmt"
	"github.com/lxn/win"
	"github.com/marpaia/graphite-golang"
	"log"
	"time"
	"unsafe"
)

func SendPerfCounters(c Config, metrics_channel chan []graphite.Metric) {

	if c.Performance_counters.Counters == nil {
		log.Println("Cannot find valid performance counters in config, will skip pc collection")
		return
	}

	for _, v := range c.Performance_counters.Counters {
		go func(counterName string) {
			for {	// Retry forever reading from conn counters
				// Handy when perfocunters reading is not yet available (soon after boot) or counter is temporarily missing
				ololosha, err := ReadPerformanceCounter(counterName, c.Performance_counters.Interval)
				if err == nil {
					for metrics := range ololosha {
						for i, v := range metrics {
							metrics[i].Name = fmt.Sprintf("%s.%s", c.Performance_counters.Metric_prefix, v.Name)
						}
						metrics_channel <- metrics
					}
				} else {
					log.Println(err)
				}
				time.Sleep(time.Duration(c.Performance_counters.Interval) * time.Second)
			}
		}(v)
	}
}

func ReadPerformanceCounter(counter string, sleepInterval int) (chan []graphite.Metric, error) {

	var queryHandle win.PDH_HQUERY
	var counterHandle win.PDH_HCOUNTER

	ret := win.PdhOpenQuery(0, 0, &queryHandle)
	if ret != win.ERROR_SUCCESS {
		return nil, errors.New("Unable to open query through DLL call")
	}

	// test path
	ret = win.PdhValidatePath(counter)
	if ret == win.PDH_CSTATUS_BAD_COUNTERNAME {
		return nil, errors.New("Unable to fetch counter (this is unexpected)")
	}

	ret = win.PdhAddEnglishCounter(queryHandle, counter, 0, &counterHandle)
	if ret != win.ERROR_SUCCESS {
		return nil, errors.New(fmt.Sprintf("Unable to add process counter. Error code is %x\n", ret))
	}

	ret = win.PdhCollectQueryData(queryHandle)
	if ret != win.ERROR_SUCCESS {
		return nil, errors.New(fmt.Sprintf("Got an error: 0x%x\n", ret))
	}

	out := make(chan []graphite.Metric)

	go func() {
		for {
			ret = win.PdhCollectQueryData(queryHandle)
			if ret == win.ERROR_SUCCESS {

				var metric []graphite.Metric

				var bufSize uint32
				var bufCount uint32
				var size uint32 = uint32(unsafe.Sizeof(win.PDH_FMT_COUNTERVALUE_ITEM_DOUBLE{}))
				var emptyBuf [1]win.PDH_FMT_COUNTERVALUE_ITEM_DOUBLE // need at least 1 addressable null ptr.

				ret = win.PdhGetFormattedCounterArrayDouble(counterHandle, &bufSize, &bufCount, &emptyBuf[0])
				if ret == win.PDH_MORE_DATA {
					filledBuf := make([]win.PDH_FMT_COUNTERVALUE_ITEM_DOUBLE, bufCount*size)
					ret = win.PdhGetFormattedCounterArrayDouble(counterHandle, &bufSize, &bufCount, &filledBuf[0])
					if ret == win.ERROR_SUCCESS {
						for i := 0; i < int(bufCount); i++ {
							c := filledBuf[i]
							s := win.UTF16PtrToString(c.SzName)

							metricName := NormalizeMetricName(counter)
							if len(s) > 0 {
								metricName = fmt.Sprintf("%s.%s", NormalizeMetricName(counter), NormalizeMetricName(s))
							}

							metric = append(metric, graphite.Metric{
								metricName,
								fmt.Sprintf("%v", c.FmtValue.DoubleValue),
								time.Now().Unix()})
						}
					}
				}
				out <- metric
			}

			time.Sleep(time.Duration(sleepInterval) * time.Second)
		}
	}()

	return out, nil

}
<commit_msg>rename channel variable<commit_after>package golagraphite

import (
	"errors"
	"fmt"
	"github.com/lxn/win"
	"github.com/marpaia/graphite-golang"
	"log"
	"time"
	"unsafe"
)

func SendPerfCounters(c Config, metrics_channel chan []graphite.Metric) {

	if c.Performance_counters.Counters == nil {
		log.Println("Cannot find valid performance counters in config, will skip pc collection")
		return
	}

	for _, v := range c.Performance_counters.Counters {
		go func(counterName string) {
			for {	// Retry forever reading from conn counters
				// Handy when perfocunters reading is not yet available (soon after boot) or counter is temporarily missing
				metric_chan, err := ReadPerformanceCounter(counterName, c.Performance_counters.Interval)
				if err == nil {
					for metrics := range metric_chan {
						for i, v := range metrics {
							metrics[i].Name = fmt.Sprintf("%s.%s", c.Performance_counters.Metric_prefix, v.Name)
						}
						metrics_channel <- metrics
					}
				} else {
					log.Println(err)
				}
				time.Sleep(time.Duration(c.Performance_counters.Interval) * time.Second)
			}
		}(v)
	}
}

func ReadPerformanceCounter(counter string, sleepInterval int) (chan []graphite.Metric, error) {

	var queryHandle win.PDH_HQUERY
	var counterHandle win.PDH_HCOUNTER

	ret := win.PdhOpenQuery(0, 0, &queryHandle)
	if ret != win.ERROR_SUCCESS {
		return nil, errors.New("Unable to open query through DLL call")
	}

	// test path
	ret = win.PdhValidatePath(counter)
	if ret == win.PDH_CSTATUS_BAD_COUNTERNAME {
		return nil, errors.New("Unable to fetch counter (this is unexpected)")
	}

	ret = win.PdhAddEnglishCounter(queryHandle, counter, 0, &counterHandle)
	if ret != win.ERROR_SUCCESS {
		return nil, errors.New(fmt.Sprintf("Unable to add process counter. Error code is %x\n", ret))
	}

	ret = win.PdhCollectQueryData(queryHandle)
	if ret != win.ERROR_SUCCESS {
		return nil, errors.New(fmt.Sprintf("Got an error: 0x%x\n", ret))
	}

	out := make(chan []graphite.Metric)

	go func() {
		for {
			ret = win.PdhCollectQueryData(queryHandle)
			if ret == win.ERROR_SUCCESS {

				var metric []graphite.Metric

				var bufSize uint32
				var bufCount uint32
				var size uint32 = uint32(unsafe.Sizeof(win.PDH_FMT_COUNTERVALUE_ITEM_DOUBLE{}))
				var emptyBuf [1]win.PDH_FMT_COUNTERVALUE_ITEM_DOUBLE // need at least 1 addressable null ptr.

				ret = win.PdhGetFormattedCounterArrayDouble(counterHandle, &bufSize, &bufCount, &emptyBuf[0])
				if ret == win.PDH_MORE_DATA {
					filledBuf := make([]win.PDH_FMT_COUNTERVALUE_ITEM_DOUBLE, bufCount*size)
					ret = win.PdhGetFormattedCounterArrayDouble(counterHandle, &bufSize, &bufCount, &filledBuf[0])
					if ret == win.ERROR_SUCCESS {
						for i := 0; i < int(bufCount); i++ {
							c := filledBuf[i]
							s := win.UTF16PtrToString(c.SzName)

							metricName := NormalizeMetricName(counter)
							if len(s) > 0 {
								metricName = fmt.Sprintf("%s.%s", NormalizeMetricName(counter), NormalizeMetricName(s))
							}

							metric = append(metric, graphite.Metric{
								metricName,
								fmt.Sprintf("%v", c.FmtValue.DoubleValue),
								time.Now().Unix()})
						}
					}
				}
				out <- metric
			}

			time.Sleep(time.Duration(sleepInterval) * time.Second)
		}
	}()

	return out, nil

}
<|endoftext|>


<commit_before>package forecasting

import (
	"github.com/fxsjy/RF.go/RF"
	"io"
	"draringi/codejam2013/src/data"
	"strconv"
)

func buildDataToLearn (data []data.Record) (inputs [][]interface{} , targets []string){
	for i := 0; i<len(data); i++ {
		if data[i].Null {
			break
		}
		row := make([]interface{},5)
		row[0]=data[i].Time
		row[1]=data[i].Radiation
		row[2]=data[i].Humidity
		row[3]=data[i].Temperature
		row[4]=data[i].Wind
		inputs = append(inputs,row)
		targets = append(targets,strconv.FormatFloat(data[i].Power,'f', -1, 64))
	}
	return
}

func learnCSV (file io.Reader, channel chan *data.CSVRequest) *RF.Forest {
	ret := make(chan (*data.CSVData), 1)
	request := new(data.CSVRequest)
	request.Return = ret
	request.Request = file
	channel <- request
	resp := new(data.CSVData)
	for {
		resp = <-ret
		if resp != nil {
			break
		}
	}
	inputs, targets := buildDataToLearn(resp.Data)
	forest := RF.BuildForest(inputs, targets, len(targets), len(inputs),1)
	return forest
}

func learnCSVSingle (file io.Reader) *RF.Forest {
	resp := data.CSVParse(file)
	inputs, targets := buildDataToLearn(resp.Data)
	forest := RF.BuildForest(inputs, targets, len(targets), len(inputs),1)
	return forest
}

<commit_msg>and more fixes<commit_after>package forecasting

import (
	"github.com/fxsjy/RF.go/RF"
	"io"
	"draringi/codejam2013/src/data"
	"strconv"
)

func buildDataToLearn (data []data.Record) (inputs [][]interface{} , targets []string){
	for i := 0; i<len(data); i++ {
		if data[i].Null {
			break
		}
		row := make([]interface{},5)
		row[0]=data[i].Time
		row[1]=data[i].Radiation
		row[2]=data[i].Humidity
		row[3]=data[i].Temperature
		row[4]=data[i].Wind
		inputs = append(inputs,row)
		targets = append(targets,strconv.FormatFloat(data[i].Power,'f', -1, 64))
	}
	return
}

func learnCSV (file io.Reader, channel chan *data.CSVRequest) *RF.Forest {
	ret := make(chan (*data.CSVData), 1)
	request := new(data.CSVRequest)
	request.Return = ret
	request.Request = file
	channel <- request
	resp := new(data.CSVData)
	for {
		resp = <-ret
		if resp != nil {
			break
		}
	}
	inputs, targets := buildDataToLearn(resp.Data)
	forest := RF.BuildForest(inputs, targets, len(targets), len(inputs),1)
	return forest
}

func learnCSVSingle (file io.Reader) *RF.Forest {
	resp := new(data.CSVData)
	resp.Labels, resp.Data = data.CSVParse(file)
	inputs, targets := buildDataToLearn(resp.Data)
	forest := RF.BuildForest(inputs, targets, len(targets), len(inputs),1)
	return forest
}

<|endoftext|>


<commit_before>// Copyright 2019 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package gomarshal implements the go_marshal code generator. See README.md.
package gomarshal

import (
	"bytes"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"os"
	"sort"
	"strings"

	"gvisor.dev/gvisor/tools/tags"
)

const (
	marshalImport  = "gvisor.dev/gvisor/tools/go_marshal/marshal"
	safecopyImport = "gvisor.dev/gvisor/pkg/safecopy"
	usermemImport  = "gvisor.dev/gvisor/pkg/usermem"
)

// List of identifiers we use in generated code that may conflict with a
// similarly-named source identifier. Abort gracefully when we see these to
// avoid potentially confusing compilation failures in generated code.
//
// This only applies to import aliases at the moment. All other identifiers
// are qualified by a receiver argument, since they're struct fields.
//
// All recievers are single letters, so we don't allow import aliases to be a
// single letter.
var badIdents = []string{
	"addr", "blk", "buf", "dst", "dsts", "err", "hdr", "len", "ptr", "src", "srcs", "task", "val",
	// All single-letter identifiers.
}

// Constructed fromt badIdents in init().
var badIdentsMap map[string]struct{}

func init() {
	badIdentsMap = make(map[string]struct{})
	for _, ident := range badIdents {
		badIdentsMap[ident] = struct{}{}
	}
}

// Generator drives code generation for a single invocation of the go_marshal
// utility.
//
// The Generator holds arguments passed to the tool, and drives parsing,
// processing and code Generator for all types marked with +marshal declared in
// the input files.
//
// See Generator.run() as the entry point.
type Generator struct {
	// Paths to input go source files.
	inputs []string
	// Output file to write generated go source.
	output *os.File
	// Output file to write generated tests.
	outputTest *os.File
	// Package name for the generated file.
	pkg string
	// Set of extra packages to import in the generated file.
	imports *importTable
}

// NewGenerator creates a new code Generator.
func NewGenerator(srcs []string, out, outTest, pkg string, imports []string) (*Generator, error) {
	f, err := os.OpenFile(out, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
	if err != nil {
		return nil, fmt.Errorf("Couldn't open output file %q: %v", out, err)
	}
	fTest, err := os.OpenFile(outTest, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
	if err != nil {
		return nil, fmt.Errorf("Couldn't open test output file %q: %v", out, err)
	}
	g := Generator{
		inputs:     srcs,
		output:     f,
		outputTest: fTest,
		pkg:        pkg,
		imports:    newImportTable(),
	}
	for _, i := range imports {
		// All imports on the extra imports list are unconditionally marked as
		// used, so that they're always added to the generated code.
		g.imports.add(i).markUsed()
	}

	// The following imports may or may not be used by the generated code,
	// depending on what's required for the target types. Don't mark these as
	// used by default.
	g.imports.add("io")
	g.imports.add("reflect")
	g.imports.add("runtime")
	g.imports.add("unsafe")
	g.imports.add(marshalImport)
	g.imports.add(safecopyImport)
	g.imports.add(usermemImport)

	return &g, nil
}

// writeHeader writes the header for the generated source file. The header
// includes the package name, package level comments and import statements.
func (g *Generator) writeHeader() error {
	var b sourceBuffer
	b.emit("// Automatically generated marshal implementation. See tools/go_marshal.\n\n")

	// Emit build tags.
	if t := tags.Aggregate(g.inputs); len(t) > 0 {
		b.emit(strings.Join(t.Lines(), "\n"))
		b.emit("\n\n")
	}

	// Package header.
	b.emit("package %s\n\n", g.pkg)
	if err := b.write(g.output); err != nil {
		return err
	}

	return g.imports.write(g.output)
}

// writeTypeChecks writes a statement to force the compiler to perform a type
// check for all Marshallable types referenced by the generated code.
func (g *Generator) writeTypeChecks(ms map[string]struct{}) error {
	if len(ms) == 0 {
		return nil
	}

	msl := make([]string, 0, len(ms))
	for m, _ := range ms {
		msl = append(msl, m)
	}
	sort.Strings(msl)

	var buf bytes.Buffer
	fmt.Fprint(&buf, "// Marshallable types used by this file.\n")

	for _, m := range msl {
		fmt.Fprintf(&buf, "var _ marshal.Marshallable = (*%s)(nil)\n", m)
	}
	fmt.Fprint(&buf, "\n")

	_, err := fmt.Fprint(g.output, buf.String())
	return err
}

// parse processes all input files passed this generator and produces a set of
// parsed go ASTs.
func (g *Generator) parse() ([]*ast.File, []*token.FileSet, error) {
	debugf("go_marshal invoked with %d input files:\n", len(g.inputs))
	for _, path := range g.inputs {
		debugf("  %s\n", path)
	}

	files := make([]*ast.File, 0, len(g.inputs))
	fsets := make([]*token.FileSet, 0, len(g.inputs))

	for _, path := range g.inputs {
		fset := token.NewFileSet()
		f, err := parser.ParseFile(fset, path, nil, parser.ParseComments)
		if err != nil {
			// Not a valid input file?
			return nil, nil, fmt.Errorf("Input %q can't be parsed: %v", path, err)
		}

		if debugEnabled() {
			debugf("AST for %q:\n", path)
			ast.Print(fset, f)
		}

		files = append(files, f)
		fsets = append(fsets, fset)
	}

	return files, fsets, nil
}

// collectMarshallabeTypes walks the parsed AST and collects a list of type
// declarations for which we need to generate the Marshallable interface.
func (g *Generator) collectMarshallabeTypes(a *ast.File, f *token.FileSet) []*ast.TypeSpec {
	var types []*ast.TypeSpec
	for _, decl := range a.Decls {
		gdecl, ok := decl.(*ast.GenDecl)
		// Type declaration?
		if !ok || gdecl.Tok != token.TYPE {
			debugfAt(f.Position(decl.Pos()), "Skipping declaration since it's not a type declaration.\n")
			continue
		}
		// Does it have a comment?
		if gdecl.Doc == nil {
			debugfAt(f.Position(gdecl.Pos()), "Skipping declaration since it doesn't have a comment.\n")
			continue
		}
		// Does the comment contain a "+marshal" line?
		marked := false
		for _, c := range gdecl.Doc.List {
			if c.Text == "// +marshal" {
				marked = true
				break
			}
		}
		if !marked {
			debugfAt(f.Position(gdecl.Pos()), "Skipping declaration since it doesn't have a comment containing +marshal line.\n")
			continue
		}
		for _, spec := range gdecl.Specs {
			// We already confirmed we're in a type declaration earlier.
			t := spec.(*ast.TypeSpec)
			if _, ok := t.Type.(*ast.StructType); ok {
				debugfAt(f.Position(t.Pos()), "Collected marshallable type %s.\n", t.Name.Name)
				types = append(types, t)
				continue
			}
			debugf("Skipping declaration %v since it's not a struct declaration.\n", gdecl)
		}
	}
	return types
}

// collectImports collects all imports from all input source files. Some of
// these imports are copied to the generated output, if they're referenced by
// the generated code.
//
// collectImports de-duplicates imports while building the list, and ensures
// identifiers in the generated code don't conflict with any imported package
// names.
func (g *Generator) collectImports(a *ast.File, f *token.FileSet) map[string]importStmt {
	is := make(map[string]importStmt)
	for _, decl := range a.Decls {
		gdecl, ok := decl.(*ast.GenDecl)
		// Import statement?
		if !ok || gdecl.Tok != token.IMPORT {
			continue
		}
		for _, spec := range gdecl.Specs {
			i := g.imports.addFromSpec(spec.(*ast.ImportSpec), f)
			debugf("Collected import '%s' as '%s'\n", i.path, i.name)

			// Make sure we have an import that doesn't use any local names that
			// would conflict with identifiers in the generated code.
			if len(i.name) == 1 {
				abortAt(f.Position(spec.Pos()), fmt.Sprintf("Import has a single character local name '%s'; this may conflict with code generated by go_marshal, use a multi-character import alias", i.name))
			}
			if _, ok := badIdentsMap[i.name]; ok {
				abortAt(f.Position(spec.Pos()), fmt.Sprintf("Import name '%s' is likely to conflict with code generated by go_marshal, use a different import alias", i.name))
			}
		}
	}
	return is

}

func (g *Generator) generateOne(t *ast.TypeSpec, fset *token.FileSet) *interfaceGenerator {
	// We're guaranteed to have only struct type specs by now. See
	// Generator.collectMarshallabeTypes.
	i := newInterfaceGenerator(t, fset)
	i.validate()
	i.emitMarshallable()
	return i
}

// generateOneTestSuite generates a test suite for the automatically generated
// implementations type t.
func (g *Generator) generateOneTestSuite(t *ast.TypeSpec) *testGenerator {
	i := newTestGenerator(t)
	i.emitTests()
	return i
}

// Run is the entry point to code generation using g.
//
// Run parses all input source files specified in g and emits generated code.
func (g *Generator) Run() error {
	// Parse our input source files into ASTs and token sets.
	asts, fsets, err := g.parse()
	if err != nil {
		return err
	}

	if len(asts) != len(fsets) {
		panic("ASTs and FileSets don't match")
	}

	// Map of imports in source files; key = local package name, value = import
	// path.
	is := make(map[string]importStmt)
	for i, a := range asts {
		// Collect all imports from the source files. We may need to copy some
		// of these to the generated code if they're referenced. This has to be
		// done before the loop below because we need to process all ASTs before
		// we start requesting imports to be copied one by one as we encounter
		// them in each generated source.
		for name, i := range g.collectImports(a, fsets[i]) {
			is[name] = i
		}
	}

	var impls []*interfaceGenerator
	var ts []*testGenerator
	// Set of Marshallable types referenced by generated code.
	ms := make(map[string]struct{})
	for i, a := range asts {
		// Collect type declarations marked for code generation and generate
		// Marshallable interfaces.
		for _, t := range g.collectMarshallabeTypes(a, fsets[i]) {
			impl := g.generateOne(t, fsets[i])
			// Collect Marshallable types referenced by the generated code.
			for ref, _ := range impl.ms {
				ms[ref] = struct{}{}
			}
			impls = append(impls, impl)
			// Collect imports referenced by the generated code and add them to
			// the list of imports we need to copy to the generated code.
			for name, _ := range impl.is {
				if !g.imports.markUsed(name) {
					panic(fmt.Sprintf("Generated code for '%s' referenced a non-existent import with local name '%s'", impl.typeName(), name))
				}
			}
			ts = append(ts, g.generateOneTestSuite(t))
		}
	}

	// Write output file header. These include things like package name and
	// import statements.
	if err := g.writeHeader(); err != nil {
		return err
	}

	// Write type checks for referenced marshallable types to output file.
	if err := g.writeTypeChecks(ms); err != nil {
		return err
	}

	// Write generated interfaces to output file.
	for _, i := range impls {
		if err := i.write(g.output); err != nil {
			return err
		}
	}

	// Write generated tests to test file.
	return g.writeTests(ts)
}

// writeTests outputs tests for the generated interface implementations to a go
// source file.
func (g *Generator) writeTests(ts []*testGenerator) error {
	var b sourceBuffer
	b.emit("package %s\n\n", g.pkg)
	if err := b.write(g.outputTest); err != nil {
		return err
	}

	// Collect and write test import statements.
	imports := newImportTable()
	for _, t := range ts {
		imports.merge(t.imports)
	}

	if err := imports.write(g.outputTest); err != nil {
		return err
	}

	// Write test functions.
	for _, t := range ts {
		if err := t.write(g.outputTest); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Handle situations where go-marshal generates an empty test file.<commit_after>// Copyright 2019 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package gomarshal implements the go_marshal code generator. See README.md.
package gomarshal

import (
	"bytes"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"os"
	"sort"
	"strings"

	"gvisor.dev/gvisor/tools/tags"
)

const (
	marshalImport  = "gvisor.dev/gvisor/tools/go_marshal/marshal"
	safecopyImport = "gvisor.dev/gvisor/pkg/safecopy"
	usermemImport  = "gvisor.dev/gvisor/pkg/usermem"
)

// List of identifiers we use in generated code that may conflict with a
// similarly-named source identifier. Abort gracefully when we see these to
// avoid potentially confusing compilation failures in generated code.
//
// This only applies to import aliases at the moment. All other identifiers
// are qualified by a receiver argument, since they're struct fields.
//
// All recievers are single letters, so we don't allow import aliases to be a
// single letter.
var badIdents = []string{
	"addr", "blk", "buf", "dst", "dsts", "err", "hdr", "len", "ptr", "src", "srcs", "task", "val",
	// All single-letter identifiers.
}

// Constructed fromt badIdents in init().
var badIdentsMap map[string]struct{}

func init() {
	badIdentsMap = make(map[string]struct{})
	for _, ident := range badIdents {
		badIdentsMap[ident] = struct{}{}
	}
}

// Generator drives code generation for a single invocation of the go_marshal
// utility.
//
// The Generator holds arguments passed to the tool, and drives parsing,
// processing and code Generator for all types marked with +marshal declared in
// the input files.
//
// See Generator.run() as the entry point.
type Generator struct {
	// Paths to input go source files.
	inputs []string
	// Output file to write generated go source.
	output *os.File
	// Output file to write generated tests.
	outputTest *os.File
	// Package name for the generated file.
	pkg string
	// Set of extra packages to import in the generated file.
	imports *importTable
}

// NewGenerator creates a new code Generator.
func NewGenerator(srcs []string, out, outTest, pkg string, imports []string) (*Generator, error) {
	f, err := os.OpenFile(out, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
	if err != nil {
		return nil, fmt.Errorf("Couldn't open output file %q: %v", out, err)
	}
	fTest, err := os.OpenFile(outTest, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
	if err != nil {
		return nil, fmt.Errorf("Couldn't open test output file %q: %v", out, err)
	}
	g := Generator{
		inputs:     srcs,
		output:     f,
		outputTest: fTest,
		pkg:        pkg,
		imports:    newImportTable(),
	}
	for _, i := range imports {
		// All imports on the extra imports list are unconditionally marked as
		// used, so that they're always added to the generated code.
		g.imports.add(i).markUsed()
	}

	// The following imports may or may not be used by the generated code,
	// depending on what's required for the target types. Don't mark these as
	// used by default.
	g.imports.add("io")
	g.imports.add("reflect")
	g.imports.add("runtime")
	g.imports.add("unsafe")
	g.imports.add(marshalImport)
	g.imports.add(safecopyImport)
	g.imports.add(usermemImport)

	return &g, nil
}

// writeHeader writes the header for the generated source file. The header
// includes the package name, package level comments and import statements.
func (g *Generator) writeHeader() error {
	var b sourceBuffer
	b.emit("// Automatically generated marshal implementation. See tools/go_marshal.\n\n")

	// Emit build tags.
	if t := tags.Aggregate(g.inputs); len(t) > 0 {
		b.emit(strings.Join(t.Lines(), "\n"))
		b.emit("\n\n")
	}

	// Package header.
	b.emit("package %s\n\n", g.pkg)
	if err := b.write(g.output); err != nil {
		return err
	}

	return g.imports.write(g.output)
}

// writeTypeChecks writes a statement to force the compiler to perform a type
// check for all Marshallable types referenced by the generated code.
func (g *Generator) writeTypeChecks(ms map[string]struct{}) error {
	if len(ms) == 0 {
		return nil
	}

	msl := make([]string, 0, len(ms))
	for m, _ := range ms {
		msl = append(msl, m)
	}
	sort.Strings(msl)

	var buf bytes.Buffer
	fmt.Fprint(&buf, "// Marshallable types used by this file.\n")

	for _, m := range msl {
		fmt.Fprintf(&buf, "var _ marshal.Marshallable = (*%s)(nil)\n", m)
	}
	fmt.Fprint(&buf, "\n")

	_, err := fmt.Fprint(g.output, buf.String())
	return err
}

// parse processes all input files passed this generator and produces a set of
// parsed go ASTs.
func (g *Generator) parse() ([]*ast.File, []*token.FileSet, error) {
	debugf("go_marshal invoked with %d input files:\n", len(g.inputs))
	for _, path := range g.inputs {
		debugf("  %s\n", path)
	}

	files := make([]*ast.File, 0, len(g.inputs))
	fsets := make([]*token.FileSet, 0, len(g.inputs))

	for _, path := range g.inputs {
		fset := token.NewFileSet()
		f, err := parser.ParseFile(fset, path, nil, parser.ParseComments)
		if err != nil {
			// Not a valid input file?
			return nil, nil, fmt.Errorf("Input %q can't be parsed: %v", path, err)
		}

		if debugEnabled() {
			debugf("AST for %q:\n", path)
			ast.Print(fset, f)
		}

		files = append(files, f)
		fsets = append(fsets, fset)
	}

	return files, fsets, nil
}

// collectMarshallabeTypes walks the parsed AST and collects a list of type
// declarations for which we need to generate the Marshallable interface.
func (g *Generator) collectMarshallabeTypes(a *ast.File, f *token.FileSet) []*ast.TypeSpec {
	var types []*ast.TypeSpec
	for _, decl := range a.Decls {
		gdecl, ok := decl.(*ast.GenDecl)
		// Type declaration?
		if !ok || gdecl.Tok != token.TYPE {
			debugfAt(f.Position(decl.Pos()), "Skipping declaration since it's not a type declaration.\n")
			continue
		}
		// Does it have a comment?
		if gdecl.Doc == nil {
			debugfAt(f.Position(gdecl.Pos()), "Skipping declaration since it doesn't have a comment.\n")
			continue
		}
		// Does the comment contain a "+marshal" line?
		marked := false
		for _, c := range gdecl.Doc.List {
			if c.Text == "// +marshal" {
				marked = true
				break
			}
		}
		if !marked {
			debugfAt(f.Position(gdecl.Pos()), "Skipping declaration since it doesn't have a comment containing +marshal line.\n")
			continue
		}
		for _, spec := range gdecl.Specs {
			// We already confirmed we're in a type declaration earlier.
			t := spec.(*ast.TypeSpec)
			if _, ok := t.Type.(*ast.StructType); ok {
				debugfAt(f.Position(t.Pos()), "Collected marshallable type %s.\n", t.Name.Name)
				types = append(types, t)
				continue
			}
			debugf("Skipping declaration %v since it's not a struct declaration.\n", gdecl)
		}
	}
	return types
}

// collectImports collects all imports from all input source files. Some of
// these imports are copied to the generated output, if they're referenced by
// the generated code.
//
// collectImports de-duplicates imports while building the list, and ensures
// identifiers in the generated code don't conflict with any imported package
// names.
func (g *Generator) collectImports(a *ast.File, f *token.FileSet) map[string]importStmt {
	is := make(map[string]importStmt)
	for _, decl := range a.Decls {
		gdecl, ok := decl.(*ast.GenDecl)
		// Import statement?
		if !ok || gdecl.Tok != token.IMPORT {
			continue
		}
		for _, spec := range gdecl.Specs {
			i := g.imports.addFromSpec(spec.(*ast.ImportSpec), f)
			debugf("Collected import '%s' as '%s'\n", i.path, i.name)

			// Make sure we have an import that doesn't use any local names that
			// would conflict with identifiers in the generated code.
			if len(i.name) == 1 {
				abortAt(f.Position(spec.Pos()), fmt.Sprintf("Import has a single character local name '%s'; this may conflict with code generated by go_marshal, use a multi-character import alias", i.name))
			}
			if _, ok := badIdentsMap[i.name]; ok {
				abortAt(f.Position(spec.Pos()), fmt.Sprintf("Import name '%s' is likely to conflict with code generated by go_marshal, use a different import alias", i.name))
			}
		}
	}
	return is

}

func (g *Generator) generateOne(t *ast.TypeSpec, fset *token.FileSet) *interfaceGenerator {
	// We're guaranteed to have only struct type specs by now. See
	// Generator.collectMarshallabeTypes.
	i := newInterfaceGenerator(t, fset)
	i.validate()
	i.emitMarshallable()
	return i
}

// generateOneTestSuite generates a test suite for the automatically generated
// implementations type t.
func (g *Generator) generateOneTestSuite(t *ast.TypeSpec) *testGenerator {
	i := newTestGenerator(t)
	i.emitTests()
	return i
}

// Run is the entry point to code generation using g.
//
// Run parses all input source files specified in g and emits generated code.
func (g *Generator) Run() error {
	// Parse our input source files into ASTs and token sets.
	asts, fsets, err := g.parse()
	if err != nil {
		return err
	}

	if len(asts) != len(fsets) {
		panic("ASTs and FileSets don't match")
	}

	// Map of imports in source files; key = local package name, value = import
	// path.
	is := make(map[string]importStmt)
	for i, a := range asts {
		// Collect all imports from the source files. We may need to copy some
		// of these to the generated code if they're referenced. This has to be
		// done before the loop below because we need to process all ASTs before
		// we start requesting imports to be copied one by one as we encounter
		// them in each generated source.
		for name, i := range g.collectImports(a, fsets[i]) {
			is[name] = i
		}
	}

	var impls []*interfaceGenerator
	var ts []*testGenerator
	// Set of Marshallable types referenced by generated code.
	ms := make(map[string]struct{})
	for i, a := range asts {
		// Collect type declarations marked for code generation and generate
		// Marshallable interfaces.
		for _, t := range g.collectMarshallabeTypes(a, fsets[i]) {
			impl := g.generateOne(t, fsets[i])
			// Collect Marshallable types referenced by the generated code.
			for ref, _ := range impl.ms {
				ms[ref] = struct{}{}
			}
			impls = append(impls, impl)
			// Collect imports referenced by the generated code and add them to
			// the list of imports we need to copy to the generated code.
			for name, _ := range impl.is {
				if !g.imports.markUsed(name) {
					panic(fmt.Sprintf("Generated code for '%s' referenced a non-existent import with local name '%s'", impl.typeName(), name))
				}
			}
			ts = append(ts, g.generateOneTestSuite(t))
		}
	}

	// Write output file header. These include things like package name and
	// import statements.
	if err := g.writeHeader(); err != nil {
		return err
	}

	// Write type checks for referenced marshallable types to output file.
	if err := g.writeTypeChecks(ms); err != nil {
		return err
	}

	// Write generated interfaces to output file.
	for _, i := range impls {
		if err := i.write(g.output); err != nil {
			return err
		}
	}

	// Write generated tests to test file.
	return g.writeTests(ts)
}

// writeTests outputs tests for the generated interface implementations to a go
// source file.
func (g *Generator) writeTests(ts []*testGenerator) error {
	var b sourceBuffer
	b.emit("package %s\n\n", g.pkg)
	if err := b.write(g.outputTest); err != nil {
		return err
	}

	// Collect and write test import statements.
	imports := newImportTable()
	for _, t := range ts {
		imports.merge(t.imports)
	}

	if err := imports.write(g.outputTest); err != nil {
		return err
	}

	// Write test functions.

	// If we didn't generate any Marshallable implementations, we can't just
	// emit an empty test file, since that causes the build to fail with "no
	// tests/benchmarks/examples found". Unfortunately we can't signal bazel to
	// omit the entire package since the outputs are already defined before
	// go-marshal is called. If we'd otherwise emit an empty test suite, emit an
	// empty example instead.
	if len(ts) == 0 {
		b.reset()
		b.emit("func ExampleEmptyTestSuite() {\n")
		b.inIndent(func() {
			b.emit("// This example is intentionally empty to ensure this file contains at least\n")
			b.emit("// one testable entity. go-marshal is forced to emit a test file if a package\n")
			b.emit("// is marked marshallable, but emitting a test file with no entities results\n")
			b.emit("// in a build failure.\n")
		})
		b.emit("}\n")
		return b.write(g.outputTest)
	}

	for _, t := range ts {
		if err := t.write(g.outputTest); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package string_array

import (
	"bufio"
	"io"
	"os"
)

func Contains(test string, set []string) bool {
	for _, member := range set {
		if test == member {
			return true
		}
	}

	return false
}

// Compares array1 against a2 and returns the values in array1 that are not present in a2.
func Diff(tests, set []string) []string {

	missing := make([]string, 0)

	for _, test := range tests {
		if Contains(test, set) {
			continue
		}

		missing = append(missing, test)
	}

	return missing
}

func WriteFile(filename string, string_sets ...[]string) error {

	var f *os.File

	if f, err := os.Create(filename); err != nil {
		return err
	}

	defer f.Close()

	for _, set := range string_sets {
		for _, lines := range set {
			if _, err := f.WriteString(line); err != nil {
				return err
			}
		}
	}

	return nil
}

func ReadFile(filename string) (lines []string) {
	var (
		f    *os.File
		line string
		err  error
	)

	if file, err := os.Open(filename); err != nil {
		panic(err)
	}

	defer file.Close()

	reader := bufio.NewReader(file)

	for {
		if line, err = reader.ReadString('\n'); err != nil {
			break
		}

		lines = append(lines, line)
	}

	if err != io.EOF {
		panic(err)
	}

	return lines
}
<commit_msg>Fix arg order on string_array.Contains(), add string_array.ContainsAll()<commit_after>package string_array

import (
	"bufio"
	"io"
	"os"
)

func Contains(set []string, test string) bool {
	for _, member := range set {
		if test == member {
			return true
		}
	}

	return false
}

func sum(values []int) (total int) {
	for _, v := range values {
		total += v
	}
	return
}

func ContainsAll(set []string, tests []string) bool {
	total := len(tests)
	found := make([]int, total)

	for _, s := range set {
		for idx, check := range tests {
			if s == check {
				found[idx] = 1
				if sum(found) == total {
					return true
				}
			}
		}
	}

	return false
}

// Compares array1 against a2 and returns the values in array1 that are not present in a2.
func Diff(tests, set []string) []string {

	missing := make([]string, 0)

	for _, test := range tests {
		if Contains(set, test) {
			continue
		}

		missing = append(missing, test)
	}

	return missing
}

func WriteFile(filename string, string_sets ...[]string) error {

	file, err := os.Create(filename)

	if err != nil {
		return err
	}

	defer file.Close()

	for _, set := range string_sets {
		for _, line := range set {
			if _, err := file.WriteString(line); err != nil {
				return err
			}
		}
	}

	return nil
}

func ReadFile(filename string) (lines []string) {

	file, err := os.Open(filename)

	if err != nil {
		panic(err)
	}

	defer file.Close()

	reader := bufio.NewReader(file)

	var line string

	for {
		if line, err = reader.ReadString('\n'); err != nil {
			break
		}

		lines = append(lines, line)
	}

	if err != io.EOF {
		panic(err)
	}

	return lines
}
<|endoftext|>