<commit_before>// Package snapcraft implements the Pipe interface providing Snapcraft bindings.
package snapcraft

import (
	"errors"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/apex/log"
	"github.com/goreleaser/goreleaser/context"
	"golang.org/x/sync/errgroup"
	yaml "gopkg.in/yaml.v2"
)

// ErrNoSnapcraft is shown when snapcraft cannot be found in $PATH
var ErrNoSnapcraft = errors.New("snapcraft not present in $PATH")

// SnapcraftMetadata to generate the snap package
type SnapcraftMetadata struct {
	Name          string
	Version       string
	Summary       string
	Description   string
	Grade         string `yaml:",omitempty"`
	Confinement   string `yaml:",omitempty"`
	Architectures []string
	Apps          map[string]AppsMetadata
}

// AppsMetadata for the binaries that will be in the snap package
type AppsMetadata struct {
	Command string
	//	Plugs []string
	//	Daemon string
}

// Pipe for snapcraft packaging
type Pipe struct{}

// Description of the pipe
func (Pipe) Description() string {
	return "Creating Linux packages with snapcraft"
}

// Run the pipe
func (Pipe) Run(ctx *context.Context) error {
	if ctx.Config.Snapcraft.Summary == "" {
		log.Error("no snapcraft summary defined, skipping")
		return nil
	}
	if ctx.Config.Snapcraft.Summary == "" {
		log.Error("no snapcraft description defined, skipping")
		return nil
	}
	_, err := exec.LookPath("snapcraft")
	if err != nil {
		return ErrNoSnapcraft
	}

	var g errgroup.Group
	for platform, groups := range ctx.Binaries {
		if !strings.Contains(platform, "linux") {
			log.WithField("platform", platform).Debug("skipped non-linux builds for snapcraft")
			continue
		}
		arch := archFor(platform)
		for folder, binaries := range groups {
			g.Go(func() error {
				return create(ctx, folder, arch, binaries)
			})
		}
	}
	return g.Wait()
}

func archFor(key string) string {
	switch {
	case strings.Contains(key, "amd64"):
		return "amd64"
	case strings.Contains(key, "386"):
		return "i386"
	case strings.Contains(key, "arm64"):
		return "arm64"
	case strings.Contains(key, "arm6"):
		return "armhf"
	}
	return key
}

func create(ctx *context.Context, folder, arch string, binaries []context.Binary) error {
	// prime is the directory that then will be compressed to make the .snap package.
	folderDir := filepath.Join(ctx.Config.Dist, folder)
	primeDir := filepath.Join(folderDir, "prime")
	metaDir := filepath.Join(primeDir, "meta")
	if err := os.MkdirAll(metaDir, 0755); err != nil {
		return err
	}

	var file = filepath.Join(primeDir, "meta", "snap.yaml")
	log.WithField("file", file).Info("creating snap metadata")

	var metadata = &SnapcraftMetadata{
		Name:          ctx.Config.ProjectName,
		Version:       ctx.Version,
		Summary:       ctx.Config.Snapcraft.Summary,
		Description:   ctx.Config.Snapcraft.Description,
		Grade:         ctx.Config.Snapcraft.Grade,
		Confinement:   ctx.Config.Snapcraft.Confinement,
		Architectures: []string{arch},
		Apps:          make(map[string]AppsMetadata),
	}

	for _, binary := range binaries {
		log.WithField("path", binary.Path).
			WithField("name", binary.Name).
			Info("passed binary to snapcraft")
		metadata.Apps[binary.Name] = AppsMetadata{Command: binary.Name}

		destBinaryPath := filepath.Join(primeDir, filepath.Base(binary.Path))
		os.Link(binary.Path, destBinaryPath)
	}
	out, err := yaml.Marshal(metadata)
	if err != nil {
		return err
	}

	if err = ioutil.WriteFile(file, out, 0644); err != nil {
		return err
	}

	snap := metadata.Name + "_" + metadata.Version + "_" + arch + ".snap"
	cmd := exec.Command("snapcraft", "snap", "prime", "--output", snap)
	cmd.Dir = folderDir
	if out, err = cmd.CombinedOutput(); err != nil {
		return errors.New(string(out))
	}
	ctx.AddArtifact(filepath.Join(folderDir, snap))
	return nil
}
<commit_msg>small improvements<commit_after>// Package snapcraft implements the Pipe interface providing Snapcraft bindings.
package snapcraft

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/apex/log"
	"github.com/goreleaser/goreleaser/context"
	"golang.org/x/sync/errgroup"
	yaml "gopkg.in/yaml.v2"
)

// ErrNoSnapcraft is shown when snapcraft cannot be found in $PATH
var ErrNoSnapcraft = errors.New("snapcraft not present in $PATH")

// SnapcraftMetadata to generate the snap package
type SnapcraftMetadata struct {
	Name          string
	Version       string
	Summary       string
	Description   string
	Grade         string `yaml:",omitempty"`
	Confinement   string `yaml:",omitempty"`
	Architectures []string
	Apps          map[string]AppsMetadata
}

// AppsMetadata for the binaries that will be in the snap package
type AppsMetadata struct {
	Command string
	//	Plugs []string
	//	Daemon string
}

// Pipe for snapcraft packaging
type Pipe struct{}

// Description of the pipe
func (Pipe) Description() string {
	return "Creating Linux packages with snapcraft"
}

// Run the pipe
func (Pipe) Run(ctx *context.Context) error {
	if ctx.Config.Snapcraft.Summary == "" {
		log.Error("no snapcraft summary defined, skipping")
		return nil
	}
	if ctx.Config.Snapcraft.Summary == "" {
		log.Error("no snapcraft description defined, skipping")
		return nil
	}
	_, err := exec.LookPath("snapcraft")
	if err != nil {
		return ErrNoSnapcraft
	}

	var g errgroup.Group
	for platform, groups := range ctx.Binaries {
		if !strings.Contains(platform, "linux") {
			log.WithField("platform", platform).Debug("skipped non-linux builds for snapcraft")
			continue
		}
		arch := archFor(platform)
		for folder, binaries := range groups {
			g.Go(func() error {
				return create(ctx, folder, arch, binaries)
			})
		}
	}
	return g.Wait()
}

func archFor(key string) string {
	switch {
	case strings.Contains(key, "amd64"):
		return "amd64"
	case strings.Contains(key, "386"):
		return "i386"
	case strings.Contains(key, "arm64"):
		return "arm64"
	case strings.Contains(key, "arm6"):
		return "armhf"
	}
	return key
}

func create(ctx *context.Context, folder, arch string, binaries []context.Binary) error {
	// prime is the directory that then will be compressed to make the .snap package.
	folderDir := filepath.Join(ctx.Config.Dist, folder)
	primeDir := filepath.Join(folderDir, "prime")
	metaDir := filepath.Join(primeDir, "meta")
	if err := os.MkdirAll(metaDir, 0755); err != nil {
		return err
	}

	var file = filepath.Join(primeDir, "meta", "snap.yaml")
	log.WithField("file", file).Info("creating snap metadata")

	var metadata = &SnapcraftMetadata{
		Name:          ctx.Config.ProjectName,
		Version:       ctx.Version,
		Summary:       ctx.Config.Snapcraft.Summary,
		Description:   ctx.Config.Snapcraft.Description,
		Grade:         ctx.Config.Snapcraft.Grade,
		Confinement:   ctx.Config.Snapcraft.Confinement,
		Architectures: []string{arch},
		Apps:          make(map[string]AppsMetadata),
	}

	for _, binary := range binaries {
		log.WithField("path", binary.Path).
			WithField("name", binary.Name).
			Info("passed binary to snapcraft")
		metadata.Apps[binary.Name] = AppsMetadata{Command: binary.Name}

		destBinaryPath := filepath.Join(primeDir, filepath.Base(binary.Path))
		if err := os.Link(binary.Path, destBinaryPath); err != nil {
			return err
		}
	}
	out, err := yaml.Marshal(metadata)
	if err != nil {
		return err
	}

	if err = ioutil.WriteFile(file, out, 0644); err != nil {
		return err
	}

	snap := filepath.Join(
		ctx.Config.Dist,
		metadata.Name+"_"+metadata.Version+"_"+arch+".snap",
	)
	cmd := exec.Command("snapcraft", "snap", "prime", "--output", snap)
	cmd.Dir = folderDir
	if out, err = cmd.CombinedOutput(); err != nil {
		return fmt.Errorf("failed to generate snap package: %s", string(out))
	}
	ctx.AddArtifact(filepath.Join(folderDir, snap))
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package fuzzer

import (
	fuzz "github.com/google/gofuzz"
	runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer"
	"k8s.io/kubernetes/pkg/apis/batch"
	"k8s.io/utils/pointer"
)

// Funcs returns the fuzzer functions for the batch api group.
var Funcs = func(codecs runtimeserializer.CodecFactory) []interface{} {
	return []interface{}{
		func(j *batch.Job, c fuzz.Continue) {
			c.FuzzNoCustom(j) // fuzz self without calling this function again

			// match defaulting
			if len(j.Labels) == 0 {
				j.Labels = j.Spec.Template.Labels
			}
		},
		func(j *batch.JobSpec, c fuzz.Continue) {
			c.FuzzNoCustom(j) // fuzz self without calling this function again
			completions := int32(c.Rand.Int31())
			parallelism := int32(c.Rand.Int31())
			backoffLimit := int32(c.Rand.Int31())
			j.Completions = &completions
			j.Parallelism = &parallelism
			j.BackoffLimit = &backoffLimit
			if c.Rand.Int31()%2 == 0 {
				j.ManualSelector = pointer.BoolPtr(true)
			} else {
				j.ManualSelector = nil
			}
			mode := batch.NonIndexedCompletion
			if c.RandBool() {
				mode = batch.IndexedCompletion
			}
			j.CompletionMode = &mode
			// We're fuzzing the internal JobSpec type, not the v1 type, so we don't
			// need to fuzz the nil value.
			j.Suspend = pointer.BoolPtr(c.RandBool())
		},
		func(sj *batch.CronJobSpec, c fuzz.Continue) {
			c.FuzzNoCustom(sj)
			suspend := c.RandBool()
			sj.Suspend = &suspend
			sds := int64(c.RandUint64())
			sj.StartingDeadlineSeconds = &sds
			sj.Schedule = c.RandString()
			successfulJobsHistoryLimit := int32(c.Rand.Int31())
			sj.SuccessfulJobsHistoryLimit = &successfulJobsHistoryLimit
			failedJobsHistoryLimit := int32(c.Rand.Int31())
			sj.FailedJobsHistoryLimit = &failedJobsHistoryLimit
		},
		func(cp *batch.ConcurrencyPolicy, c fuzz.Continue) {
			policies := []batch.ConcurrencyPolicy{batch.AllowConcurrent, batch.ForbidConcurrent, batch.ReplaceConcurrent}
			*cp = policies[c.Rand.Intn(len(policies))]
		},
	}
}
<commit_msg>Fix the TestRoundTripTypes by adding default to the fuzzer<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package fuzzer

import (
	fuzz "github.com/google/gofuzz"
	runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer"
	"k8s.io/kubernetes/pkg/apis/batch"
	api "k8s.io/kubernetes/pkg/apis/core"
	"k8s.io/utils/pointer"
)

// Funcs returns the fuzzer functions for the batch api group.
var Funcs = func(codecs runtimeserializer.CodecFactory) []interface{} {
	return []interface{}{
		func(j *batch.Job, c fuzz.Continue) {
			c.FuzzNoCustom(j) // fuzz self without calling this function again

			// match defaulting
			if len(j.Labels) == 0 {
				j.Labels = j.Spec.Template.Labels
			}
		},
		func(j *batch.JobSpec, c fuzz.Continue) {
			c.FuzzNoCustom(j) // fuzz self without calling this function again
			completions := int32(c.Rand.Int31())
			parallelism := int32(c.Rand.Int31())
			backoffLimit := int32(c.Rand.Int31())
			j.Completions = &completions
			j.Parallelism = &parallelism
			j.BackoffLimit = &backoffLimit
			if c.Rand.Int31()%2 == 0 {
				j.ManualSelector = pointer.BoolPtr(true)
			} else {
				j.ManualSelector = nil
			}
			mode := batch.NonIndexedCompletion
			if c.RandBool() {
				mode = batch.IndexedCompletion
			}
			j.CompletionMode = &mode
			// We're fuzzing the internal JobSpec type, not the v1 type, so we don't
			// need to fuzz the nil value.
			j.Suspend = pointer.BoolPtr(c.RandBool())
		},
		func(sj *batch.CronJobSpec, c fuzz.Continue) {
			c.FuzzNoCustom(sj)
			suspend := c.RandBool()
			sj.Suspend = &suspend
			sds := int64(c.RandUint64())
			sj.StartingDeadlineSeconds = &sds
			sj.Schedule = c.RandString()
			successfulJobsHistoryLimit := int32(c.Rand.Int31())
			sj.SuccessfulJobsHistoryLimit = &successfulJobsHistoryLimit
			failedJobsHistoryLimit := int32(c.Rand.Int31())
			sj.FailedJobsHistoryLimit = &failedJobsHistoryLimit
		},
		func(cp *batch.ConcurrencyPolicy, c fuzz.Continue) {
			policies := []batch.ConcurrencyPolicy{batch.AllowConcurrent, batch.ForbidConcurrent, batch.ReplaceConcurrent}
			*cp = policies[c.Rand.Intn(len(policies))]
		},
		func(p *batch.PodFailurePolicyOnPodConditionsPattern, c fuzz.Continue) {
			c.FuzzNoCustom(p)
			if p.Status == "" {
				p.Status = api.ConditionTrue
			}
		},
	}
}
<|endoftext|>


<commit_before>package goka

import (
	"fmt"
	"sync"

	"github.com/lovoo/goka/kafka"
)

type Emitter struct {
	codec    Codec
	producer kafka.Producer

	topic string

	wg sync.WaitGroup
}

// NewEmitter creates a new emitter using passed brokers, topic, codec and possibly options
func NewEmitter(brokers []string, topic Stream, codec Codec, options ...EmitterOption) (*Emitter, error) {
	options = append(
		// default options comes first
		[]EmitterOption{},

		// user-defined options (may overwrite default ones)
		options...,
	)

	opts := new(eoptions)

	err := opts.applyOptions(options...)
	if err != nil {
		return nil, fmt.Errorf(errApplyOptions, err)
	}

	prod, err := opts.builders.producer(brokers, opts.clientID, opts.hasher)
	if err != nil {
		return nil, fmt.Errorf(errBuildProducer, err)
	}

	return &Emitter{
		codec:    codec,
		producer: prod,
		topic:    string(topic),
	}, nil
}

// Emit sends a message for passed key using the emitter's codec.
func (e *Emitter) Emit(key string, msg interface{}) (*kafka.Promise, error) {
	var (
		err  error
		data []byte
	)

	if msg != nil {
		data, err = e.codec.Encode(msg)
		if err != nil {
			return nil, fmt.Errorf("Error encoding value for key %s in topic %s: %v", key, e.topic, err)
		}
	}
	e.wg.Add(1)
	return e.producer.Emit(e.topic, key, data).Then(func(err error) {
		e.wg.Done()
	}), nil
}

// EmitSync sends a message to passed topic and key
func (e *Emitter) EmitSync(key string, msg interface{}) error {
	promise, err := e.Emit(key, msg)

	if err != nil {
		return err
	}

	done := make(chan struct{})
	promise.Then(func(err error) {
		close(done)
	})
	<-done
	return nil
}

// Finish waits until the emitter is finished producing all pending messages
func (e *Emitter) Finish() {
	e.wg.Wait()
}
<commit_msg>bugfix #103: propagate error in Emitter.EmitSync<commit_after>package goka

import (
	"fmt"
	"sync"

	"github.com/lovoo/goka/kafka"
)

type Emitter struct {
	codec    Codec
	producer kafka.Producer

	topic string

	wg sync.WaitGroup
}

// NewEmitter creates a new emitter using passed brokers, topic, codec and possibly options
func NewEmitter(brokers []string, topic Stream, codec Codec, options ...EmitterOption) (*Emitter, error) {
	options = append(
		// default options comes first
		[]EmitterOption{},

		// user-defined options (may overwrite default ones)
		options...,
	)

	opts := new(eoptions)

	err := opts.applyOptions(options...)
	if err != nil {
		return nil, fmt.Errorf(errApplyOptions, err)
	}

	prod, err := opts.builders.producer(brokers, opts.clientID, opts.hasher)
	if err != nil {
		return nil, fmt.Errorf(errBuildProducer, err)
	}

	return &Emitter{
		codec:    codec,
		producer: prod,
		topic:    string(topic),
	}, nil
}

// Emit sends a message for passed key using the emitter's codec.
func (e *Emitter) Emit(key string, msg interface{}) (*kafka.Promise, error) {
	var (
		err  error
		data []byte
	)

	if msg != nil {
		data, err = e.codec.Encode(msg)
		if err != nil {
			return nil, fmt.Errorf("Error encoding value for key %s in topic %s: %v", key, e.topic, err)
		}
	}
	e.wg.Add(1)
	return e.producer.Emit(e.topic, key, data).Then(func(err error) {
		e.wg.Done()
	}), nil
}

// EmitSync sends a message to passed topic and key
func (e *Emitter) EmitSync(key string, msg interface{}) error {
	var (
		err     error
		promise *kafka.Promise
	)
	promise, err = e.Emit(key, msg)

	if err != nil {
		return err
	}

	done := make(chan struct{})
	promise.Then(func(asyncErr error) {
		err = asyncErr
		close(done)
	})
	<-done
	return err
}

// Finish waits until the emitter is finished producing all pending messages
func (e *Emitter) Finish() {
	e.wg.Wait()
}
<|endoftext|>


<commit_before>package aws

import (
	"regexp"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
)

func TestAccDataSourceAwsAcmpcaCertificateAuthority_Basic(t *testing.T) {
	resourceName := "aws_acmpca_certificate_authority.test"
	datasourceName := "data.aws_acmpca_certificate_authority.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			{
				Config:      testAccDataSourceAwsAcmpcaCertificateAuthorityConfig_NonExistent,
				ExpectError: regexp.MustCompile(`ResourceNotFoundException`),
			},
			{
				Config: testAccDataSourceAwsAcmpcaCertificateAuthorityConfig_ARN,
				Check: resource.ComposeTestCheckFunc(
					resource.TestCheckResourceAttrPair(datasourceName, "arn", resourceName, "arn"),
					resource.TestCheckResourceAttrPair(datasourceName, "certificate", resourceName, "certificate"),
					resource.TestCheckResourceAttrPair(datasourceName, "certificate_chain", resourceName, "certificate_chain"),
					resource.TestCheckResourceAttrPair(datasourceName, "certificate_signing_request", resourceName, "certificate_signing_request"),
					resource.TestCheckResourceAttrPair(datasourceName, "not_after", resourceName, "not_after"),
					resource.TestCheckResourceAttrPair(datasourceName, "not_before", resourceName, "not_before"),
					resource.TestCheckResourceAttrPair(datasourceName, "revocation_configuration.#", resourceName, "revocation_configuration.#"),
					resource.TestCheckResourceAttrPair(datasourceName, "revocation_configuration.0.crl_configuration.#", resourceName, "revocation_configuration.0.crl_configuration.#"),
					resource.TestCheckResourceAttrPair(datasourceName, "revocation_configuration.0.crl_configuration.0.enabled", resourceName, "revocation_configuration.0.crl_configuration.0.enabled"),
					resource.TestCheckResourceAttrPair(datasourceName, "serial", resourceName, "serial"),
					resource.TestCheckResourceAttrPair(datasourceName, "status", resourceName, "status"),
					resource.TestCheckResourceAttrPair(datasourceName, "tags.%", resourceName, "tags.%"),
					resource.TestCheckResourceAttrPair(datasourceName, "type", resourceName, "type"),
				),
			},
		},
	})
}

const testAccDataSourceAwsAcmpcaCertificateAuthorityConfig_ARN = `
resource "aws_acmpca_certificate_authority" "wrong" {
  certificate_authority_configuration {
    key_algorithm     = "RSA_4096"
    signing_algorithm = "SHA512WITHRSA"

    subject {
      common_name = "terraformtesting.com"
    }
  }
}

resource "aws_acmpca_certificate_authority" "test" {
  certificate_authority_configuration {
    key_algorithm     = "RSA_4096"
    signing_algorithm = "SHA512WITHRSA"

    subject {
      common_name = "terraformtesting.com"
    }
  }
}

data "aws_acmpca_certificate_authority" "test" {
  arn = "${aws_acmpca_certificate_authority.test.arn}"
}
`

const testAccDataSourceAwsAcmpcaCertificateAuthorityConfig_NonExistent = `
data "aws_acmpca_certificate_authority" "test" {
  arn = "arn:aws:acm-pca:us-east-1:123456789012:certificate-authority/tf-acc-test-does-not-exist"
}
`
<commit_msg>tests/data-source/aws_acmpca_certificate_authority: Add new permanent_deletion_time_in_days = 7 argument to test resources and handle additional error for non-existent testing<commit_after>package aws

import (
	"regexp"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
)

func TestAccDataSourceAwsAcmpcaCertificateAuthority_Basic(t *testing.T) {
	resourceName := "aws_acmpca_certificate_authority.test"
	datasourceName := "data.aws_acmpca_certificate_authority.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			{
				Config:      testAccDataSourceAwsAcmpcaCertificateAuthorityConfig_NonExistent,
				ExpectError: regexp.MustCompile(`(AccessDeniedException|ResourceNotFoundException)`),
			},
			{
				Config: testAccDataSourceAwsAcmpcaCertificateAuthorityConfig_ARN,
				Check: resource.ComposeTestCheckFunc(
					resource.TestCheckResourceAttrPair(datasourceName, "arn", resourceName, "arn"),
					resource.TestCheckResourceAttrPair(datasourceName, "certificate", resourceName, "certificate"),
					resource.TestCheckResourceAttrPair(datasourceName, "certificate_chain", resourceName, "certificate_chain"),
					resource.TestCheckResourceAttrPair(datasourceName, "certificate_signing_request", resourceName, "certificate_signing_request"),
					resource.TestCheckResourceAttrPair(datasourceName, "not_after", resourceName, "not_after"),
					resource.TestCheckResourceAttrPair(datasourceName, "not_before", resourceName, "not_before"),
					resource.TestCheckResourceAttrPair(datasourceName, "revocation_configuration.#", resourceName, "revocation_configuration.#"),
					resource.TestCheckResourceAttrPair(datasourceName, "revocation_configuration.0.crl_configuration.#", resourceName, "revocation_configuration.0.crl_configuration.#"),
					resource.TestCheckResourceAttrPair(datasourceName, "revocation_configuration.0.crl_configuration.0.enabled", resourceName, "revocation_configuration.0.crl_configuration.0.enabled"),
					resource.TestCheckResourceAttrPair(datasourceName, "serial", resourceName, "serial"),
					resource.TestCheckResourceAttrPair(datasourceName, "status", resourceName, "status"),
					resource.TestCheckResourceAttrPair(datasourceName, "tags.%", resourceName, "tags.%"),
					resource.TestCheckResourceAttrPair(datasourceName, "type", resourceName, "type"),
				),
			},
		},
	})
}

const testAccDataSourceAwsAcmpcaCertificateAuthorityConfig_ARN = `
resource "aws_acmpca_certificate_authority" "wrong" {
  permanent_deletion_time_in_days = 7

  certificate_authority_configuration {
    key_algorithm     = "RSA_4096"
    signing_algorithm = "SHA512WITHRSA"

    subject {
      common_name = "terraformtesting.com"
    }
  }
}

resource "aws_acmpca_certificate_authority" "test" {
  permanent_deletion_time_in_days = 7

  certificate_authority_configuration {
    key_algorithm     = "RSA_4096"
    signing_algorithm = "SHA512WITHRSA"

    subject {
      common_name = "terraformtesting.com"
    }
  }
}

data "aws_acmpca_certificate_authority" "test" {
  arn = "${aws_acmpca_certificate_authority.test.arn}"
}
`

const testAccDataSourceAwsAcmpcaCertificateAuthorityConfig_NonExistent = `
data "aws_acmpca_certificate_authority" "test" {
  arn = "arn:aws:acm-pca:us-east-1:123456789012:certificate-authority/tf-acc-test-does-not-exist"
}
`
<|endoftext|>


<commit_before>package auctionrunner_test

import (
	"errors"
	"net/http"
	"time"

	"code.cloudfoundry.org/auction/auctionrunner"
	"code.cloudfoundry.org/auction/auctiontypes/fakes"
	"code.cloudfoundry.org/lager/lagertest"
	"code.cloudfoundry.org/rep"
	"code.cloudfoundry.org/rep/repfakes"
	"code.cloudfoundry.org/workpool"

	"code.cloudfoundry.org/lager"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("ZoneBuilder", func() {
	var repA, repB, repC *repfakes.FakeSimClient
	var clients map[string]rep.Client
	var workPool *workpool.WorkPool
	var logger *lagertest.TestLogger
	var metricEmitter *fakes.FakeAuctionMetricEmitterDelegate

	BeforeEach(func() {
		logger = lagertest.NewTestLogger("test")

		var err error
		workPool, err = workpool.NewWorkPool(5)
		Expect(err).NotTo(HaveOccurred())

		repA = new(repfakes.FakeSimClient)
		repB = new(repfakes.FakeSimClient)
		repC = new(repfakes.FakeSimClient)

		clients = map[string]rep.Client{
			"A": repA,
			"B": repB,
			"C": repC,
		}

		repA.StateReturns(BuildCellState("A", "the-zone", 100, 200, 100, false, 0, linuxOnlyRootFSProviders, nil, []string{}, []string{}, []string{}), nil)
		repB.StateReturns(BuildCellState("B", "the-zone", 10, 10, 100, false, 0, linuxOnlyRootFSProviders, nil, []string{}, []string{}, []string{}), nil)
		repC.StateReturns(BuildCellState("C", "other-zone", 100, 10, 100, false, 0, linuxOnlyRootFSProviders, nil, []string{}, []string{}, []string{}), nil)

		metricEmitter = new(fakes.FakeAuctionMetricEmitterDelegate)
	})

	AfterEach(func() {
		workPool.Stop()
	})

	It("fetches state by calling each client", func() {
		zones := auctionrunner.FetchStateAndBuildZones(logger, workPool, clients, metricEmitter)
		Expect(zones).To(HaveLen(2))

		cells := map[string]*auctionrunner.Cell{}
		for _, cell := range zones["the-zone"] {
			cells[cell.Guid] = cell
		}
		Expect(cells).To(HaveLen(2))
		Expect(cells).To(HaveKey("A"))
		Expect(cells).To(HaveKey("B"))

		Expect(repA.StateCallCount()).To(Equal(1))
		Expect(repB.StateCallCount()).To(Equal(1))

		otherZone := zones["other-zone"]
		Expect(otherZone).To(HaveLen(1))
		Expect(otherZone[0].Guid).To(Equal("C"))

		Expect(repC.StateCallCount()).To(Equal(1))
	})

	Context("when cells are evacuating", func() {
		BeforeEach(func() {
			repB.StateReturns(BuildCellState("B", "the-zone", 10, 10, 100, true, 0, linuxOnlyRootFSProviders, nil, []string{}, []string{}, []string{}), nil)
		})

		It("does not include them in the map", func() {
			zones := auctionrunner.FetchStateAndBuildZones(logger, workPool, clients, metricEmitter)
			Expect(zones).To(HaveLen(2))

			cells := zones["the-zone"]
			Expect(cells).To(HaveLen(1))
			Expect(cells[0].Guid).To(Equal("A"))

			cells = zones["other-zone"]
			Expect(cells).To(HaveLen(1))
			Expect(cells[0].Guid).To(Equal("C"))
		})
	})

	Context("when a cell ID does not match the cell state ID", func() {
		BeforeEach(func() {
			repB.StateReturns(BuildCellState("badCellID", "the-zone", 10, 10, 100, false, 0, linuxOnlyRootFSProviders, nil, []string{}, []string{}, []string{}), nil)
		})

		It("does not include that cell in the map", func() {
			zones := auctionrunner.FetchStateAndBuildZones(logger, workPool, clients, metricEmitter)
			Expect(zones).To(HaveLen(2))

			cells := zones["the-zone"]
			Expect(cells).To(HaveLen(1))
			Expect(cells[0].Guid).To(Equal("A"))

			cells = zones["other-zone"]
			Expect(cells).To(HaveLen(1))
			Expect(cells[0].Guid).To(Equal("C"))
		})

		Context("when the cell id is empty", func() {
			BeforeEach(func() {
				repB.StateReturns(BuildCellState("", "the-zone", 10, 10, 100, false, 0, linuxOnlyRootFSProviders, nil, []string{}, []string{}, []string{}), nil)
			})

			It("includes that cell in the map", func() {
				zones := auctionrunner.FetchStateAndBuildZones(logger, workPool, clients, metricEmitter)
				Expect(zones).To(HaveLen(2))

				cells := zones["the-zone"]
				guids := []string{}
				for _, cell := range cells {
					guids = append(guids, cell.Guid)
				}
				Expect(guids).To(ConsistOf("A", "B"))

				cells = zones["other-zone"]
				Expect(cells).To(HaveLen(1))
				Expect(cells[0].Guid).To(Equal("C"))
			})
		})

		It("logs that there was a cell ID mismatch", func() {
			auctionrunner.FetchStateAndBuildZones(logger, workPool, clients, metricEmitter)

			getLogData := func(format lager.LogFormat) lager.Data { return format.Data }

			Expect(logger.LogMessages()).To(ContainElement("test.cell-id-mismatch"))
			Expect(logger.Logs()).To(ContainElement(WithTransform(getLogData, Equal(lager.Data{"cell-guid": "B", "cell-state-guid": "badCellID"}))))
		})
	})

	Context("when a client fails", func() {
		BeforeEach(func() {
			repB.StateReturns(BuildCellState("B", "the-zone", 10, 10, 100, false, 0, linuxOnlyRootFSProviders, nil, []string{}, []string{}, []string{}), errors.New("boom"))
		})

		It("does not include the client in the map", func() {
			zones := auctionrunner.FetchStateAndBuildZones(logger, workPool, clients, metricEmitter)
			Expect(zones).To(HaveLen(2))

			cells := zones["the-zone"]
			Expect(cells).To(HaveLen(1))
			Expect(cells[0].Guid).To(Equal("A"))

			cells = zones["other-zone"]
			Expect(cells).To(HaveLen(1))
			Expect(cells[0].Guid).To(Equal("C"))
		})

		It("it emits metrics for the failure", func() {
			zones := auctionrunner.FetchStateAndBuildZones(logger, workPool, clients, metricEmitter)
			Expect(zones).To(HaveLen(2))
			Expect(metricEmitter.FailedCellStateRequestCallCount()).To(Equal(1))
		})
	})

	Context("when clients are slow to respond", func() {
		BeforeEach(func() {
			repA.StateReturns(BuildCellState("A", "the-zone", 10, 10, 100, false, 0, linuxOnlyRootFSProviders, nil, []string{}, []string{}, []string{}), errors.New("timeout"))
			repA.StateClientTimeoutReturns(5 * time.Second)
			repA.SetStateClientStub = func(client *http.Client) {
				repA.StateClientTimeoutReturns(client.Timeout)
			}
			repB.StateReturns(BuildCellState("B", "the-zone", 10, 10, 100, false, 0, linuxOnlyRootFSProviders, nil, []string{}, []string{}, []string{}), errors.New("timeout"))
			repB.StateClientTimeoutReturns(2 * time.Second)
			repB.SetStateClientStub = func(client *http.Client) {
				repB.StateClientTimeoutReturns(client.Timeout)
			}
			repC.StateReturns(BuildCellState("C", "the-zone", 10, 10, 100, false, 0, linuxOnlyRootFSProviders, nil, []string{}, []string{}, []string{}), errors.New("timeout"))
			repC.StateClientTimeoutReturns(4 * time.Second)
			repC.SetStateClientStub = func(client *http.Client) {
				repC.StateClientTimeoutReturns(client.Timeout)
			}
		})
	})
})
<commit_msg>Backfill tests for logging<commit_after>package auctionrunner_test

import (
	"errors"
	"net/http"
	"time"

	"code.cloudfoundry.org/auction/auctionrunner"
	"code.cloudfoundry.org/auction/auctiontypes/fakes"
	"code.cloudfoundry.org/lager/lagertest"
	"code.cloudfoundry.org/rep"
	"code.cloudfoundry.org/rep/repfakes"
	"code.cloudfoundry.org/workpool"

	"code.cloudfoundry.org/lager"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("ZoneBuilder", func() {
	var repA, repB, repC *repfakes.FakeSimClient
	var clients map[string]rep.Client
	var workPool *workpool.WorkPool
	var logger *lagertest.TestLogger
	var metricEmitter *fakes.FakeAuctionMetricEmitterDelegate

	BeforeEach(func() {
		logger = lagertest.NewTestLogger("test")

		var err error
		workPool, err = workpool.NewWorkPool(5)
		Expect(err).NotTo(HaveOccurred())

		repA = new(repfakes.FakeSimClient)
		repB = new(repfakes.FakeSimClient)
		repC = new(repfakes.FakeSimClient)

		clients = map[string]rep.Client{
			"A": repA,
			"B": repB,
			"C": repC,
		}

		repA.StateReturns(BuildCellState("A", "the-zone", 100, 200, 100, false, 0, linuxOnlyRootFSProviders, nil, []string{}, []string{}, []string{}), nil)
		repB.StateReturns(BuildCellState("B", "the-zone", 10, 10, 100, false, 0, linuxOnlyRootFSProviders, nil, []string{}, []string{}, []string{}), nil)
		repC.StateReturns(BuildCellState("C", "other-zone", 100, 10, 100, false, 0, linuxOnlyRootFSProviders, nil, []string{}, []string{}, []string{}), nil)

		metricEmitter = new(fakes.FakeAuctionMetricEmitterDelegate)
	})

	AfterEach(func() {
		workPool.Stop()
	})

	It("fetches state by calling each client", func() {
		zones := auctionrunner.FetchStateAndBuildZones(logger, workPool, clients, metricEmitter)
		Expect(zones).To(HaveLen(2))

		cells := map[string]*auctionrunner.Cell{}
		for _, cell := range zones["the-zone"] {
			cells[cell.Guid] = cell
		}
		Expect(cells).To(HaveLen(2))
		Expect(cells).To(HaveKey("A"))
		Expect(cells).To(HaveKey("B"))

		Expect(repA.StateCallCount()).To(Equal(1))
		Expect(repB.StateCallCount()).To(Equal(1))

		otherZone := zones["other-zone"]
		Expect(otherZone).To(HaveLen(1))
		Expect(otherZone[0].Guid).To(Equal("C"))

		Expect(repC.StateCallCount()).To(Equal(1))
	})

	It("logs that it successfully fetched the state of the cells", func() {
		auctionrunner.FetchStateAndBuildZones(logger, workPool, clients, metricEmitter)

		getLogData := func(format lager.LogFormat) lager.Data { return format.Data }
		Expect(logger.LogMessages()).To(ContainElement("test.fetched-cell-state"))
		Expect(logger.Logs()).To(ContainElement(WithTransform(getLogData, Equal(lager.Data{"cell-guid": "A"}))))
		Expect(logger.Logs()).To(ContainElement(WithTransform(getLogData, Equal(lager.Data{"cell-guid": "B"}))))
	})

	Context("when cells are evacuating", func() {
		BeforeEach(func() {
			repB.StateReturns(BuildCellState("B", "the-zone", 10, 10, 100, true, 0, linuxOnlyRootFSProviders, nil, []string{}, []string{}, []string{}), nil)
		})

		It("does not include them in the map", func() {
			zones := auctionrunner.FetchStateAndBuildZones(logger, workPool, clients, metricEmitter)
			Expect(zones).To(HaveLen(2))

			cells := zones["the-zone"]
			Expect(cells).To(HaveLen(1))
			Expect(cells[0].Guid).To(Equal("A"))

			cells = zones["other-zone"]
			Expect(cells).To(HaveLen(1))
			Expect(cells[0].Guid).To(Equal("C"))
		})

		It("logs that it ignored the evacuating cell", func() {
			auctionrunner.FetchStateAndBuildZones(logger, workPool, clients, metricEmitter)

			getLogData := func(format lager.LogFormat) lager.Data { return format.Data }
			Expect(logger.LogMessages()).To(ContainElement("test.ignored-evacuating-cell"))
			Expect(logger.Logs()).To(ContainElement(WithTransform(getLogData, Equal(lager.Data{"cell-guid": "B"}))))
		})
	})

	Context("when a cell ID does not match the cell state ID", func() {
		BeforeEach(func() {
			repB.StateReturns(BuildCellState("badCellID", "the-zone", 10, 10, 100, false, 0, linuxOnlyRootFSProviders, nil, []string{}, []string{}, []string{}), nil)
		})

		It("does not include that cell in the map", func() {
			zones := auctionrunner.FetchStateAndBuildZones(logger, workPool, clients, metricEmitter)
			Expect(zones).To(HaveLen(2))

			cells := zones["the-zone"]
			Expect(cells).To(HaveLen(1))
			Expect(cells[0].Guid).To(Equal("A"))

			cells = zones["other-zone"]
			Expect(cells).To(HaveLen(1))
			Expect(cells[0].Guid).To(Equal("C"))
		})

		Context("when the cell id is empty", func() {
			BeforeEach(func() {
				repB.StateReturns(BuildCellState("", "the-zone", 10, 10, 100, false, 0, linuxOnlyRootFSProviders, nil, []string{}, []string{}, []string{}), nil)
			})

			It("includes that cell in the map", func() {
				zones := auctionrunner.FetchStateAndBuildZones(logger, workPool, clients, metricEmitter)
				Expect(zones).To(HaveLen(2))

				cells := zones["the-zone"]
				guids := []string{}
				for _, cell := range cells {
					guids = append(guids, cell.Guid)
				}
				Expect(guids).To(ConsistOf("A", "B"))

				cells = zones["other-zone"]
				Expect(cells).To(HaveLen(1))
				Expect(cells[0].Guid).To(Equal("C"))
			})
		})

		It("logs that there was a cell ID mismatch", func() {
			auctionrunner.FetchStateAndBuildZones(logger, workPool, clients, metricEmitter)

			getLogData := func(format lager.LogFormat) lager.Data { return format.Data }

			Expect(logger.LogMessages()).To(ContainElement("test.cell-id-mismatch"))
			Expect(logger.Logs()).To(ContainElement(WithTransform(getLogData, Equal(lager.Data{"cell-guid": "B", "cell-state-guid": "badCellID"}))))
		})
	})

	Context("when a client fails", func() {
		BeforeEach(func() {
			repB.StateReturns(BuildCellState("B", "the-zone", 10, 10, 100, false, 0, linuxOnlyRootFSProviders, nil, []string{}, []string{}, []string{}), errors.New("boom"))
		})

		It("does not include the client in the map", func() {
			zones := auctionrunner.FetchStateAndBuildZones(logger, workPool, clients, metricEmitter)
			Expect(zones).To(HaveLen(2))

			cells := zones["the-zone"]
			Expect(cells).To(HaveLen(1))
			Expect(cells[0].Guid).To(Equal("A"))

			cells = zones["other-zone"]
			Expect(cells).To(HaveLen(1))
			Expect(cells[0].Guid).To(Equal("C"))
		})

		It("it emits metrics for the failure", func() {
			zones := auctionrunner.FetchStateAndBuildZones(logger, workPool, clients, metricEmitter)
			Expect(zones).To(HaveLen(2))
			Expect(metricEmitter.FailedCellStateRequestCallCount()).To(Equal(1))
		})

		It("logs that it failed to fetch cell state", func() {
			auctionrunner.FetchStateAndBuildZones(logger, workPool, clients, metricEmitter)

			getLogData := func(format lager.LogFormat) lager.Data { return format.Data }
			Expect(logger.LogMessages()).To(ContainElement("test.failed-to-get-state"))
			Expect(logger.Logs()).To(ContainElement(WithTransform(getLogData, Equal(lager.Data{"cell-guid": "B", "error": "boom"}))))
		})
	})

	Context("when clients are slow to respond", func() {
		BeforeEach(func() {
			repA.StateReturns(BuildCellState("A", "the-zone", 10, 10, 100, false, 0, linuxOnlyRootFSProviders, nil, []string{}, []string{}, []string{}), errors.New("timeout"))
			repA.StateClientTimeoutReturns(5 * time.Second)
			repA.SetStateClientStub = func(client *http.Client) {
				repA.StateClientTimeoutReturns(client.Timeout)
			}
			repB.StateReturns(BuildCellState("B", "the-zone", 10, 10, 100, false, 0, linuxOnlyRootFSProviders, nil, []string{}, []string{}, []string{}), errors.New("timeout"))
			repB.StateClientTimeoutReturns(2 * time.Second)
			repB.SetStateClientStub = func(client *http.Client) {
				repB.StateClientTimeoutReturns(client.Timeout)
			}
			repC.StateReturns(BuildCellState("C", "the-zone", 10, 10, 100, false, 0, linuxOnlyRootFSProviders, nil, []string{}, []string{}, []string{}), errors.New("timeout"))
			repC.StateClientTimeoutReturns(4 * time.Second)
			repC.SetStateClientStub = func(client *http.Client) {
				repC.StateClientTimeoutReturns(client.Timeout)
			}
		})
	})
})
<|endoftext|>


<commit_before>package gocli

import (
	"errors"
	"go/build"
	"path/filepath"

	"github.com/jimmyfrasche/goutil"
)

//just a basic wrapper to give goutil.Import the same signature as ImportTree
func imports(ctx *build.Context, ip string) (pkgs goutil.Packages, err error) {
	pkg, err := goutil.Import(ctx, ip)
	if err != nil {
		return nil, err
	}
	return append(pkgs, pkg), nil
}

//Import is for importing command line arguments.
//It uses the following rules:
//	If args is len 0, try to import the current directory.
//	Otherwise, for each argument:
//		If it ends with ..., use goutil.ImportTree (unless notree is true)
//		Otherwise, use goutil.Import
//Regardless, the ctx is passed as is to the various importers.
//
//Import returns a list of any errors that resulted from attmepting to import.
//If you only care about the first error, wrap the call in FirstError.
func Import(notree bool, ctx *build.Context, args []string) (pkgs []goutil.Packages, errs []error) {
	push := func(ps goutil.Packages, err error) {
		pkgs = append(pkgs, ps)
		if err != nil {
			errs = append(errs, err)
		}
	}

	if len(args) == 0 {
		push(imports(ctx, ""))
		return
	}

	for _, arg := range args {
		if filepath.Base(arg) == "..." {
			if notree {
				push(nil, errors.New("cannot use ... imports"))
			} else {
				push(goutil.ImportTree(ctx, filepath.Dir(arg)))
			}
		} else {
			push(imports(ctx, arg))
		}
	}
	return
}

//FirstError is meant to wrap Import, it returns pkgs unchanged.
//If there were errors, only the first is returned.
func FirstError(pkgs []goutil.Packages, errs []error) ([]goutil.Packages, error) {
	if len(errs) == 0 {
		return pkgs, nil
	}
	return pkgs, errs[0]
}

//Flatten takes a slice of goutil.Packages and returns a single goutil.Packages
//containing only the unique packages from the slice.
func Flatten(pss []goutil.Packages) (out goutil.Packages) {
	for _, ps := range pss {
		out = append(out, ps...)
	}
	return out.Uniq()
}

//ImportOne only allows one argument to be specified.
func ImportOne(notree bool, ctx *build.Context, args []string) (goutil.Packages, error) {
	if len(args) > 1 {
		return nil, errors.New("only one package may be specified")
	}
	ps, err := FirstError(Import(notree, ctx, args))
	if err != nil {
		return nil, err
	}
	return ps[0], nil
}
<commit_msg>don't insert nil packages when Importing<commit_after>package gocli

import (
	"errors"
	"go/build"
	"path/filepath"

	"github.com/jimmyfrasche/goutil"
)

//just a basic wrapper to give goutil.Import the same signature as ImportTree
func imports(ctx *build.Context, ip string) (pkgs goutil.Packages, err error) {
	pkg, err := goutil.Import(ctx, ip)
	if err != nil {
		return nil, err
	}
	return append(pkgs, pkg), nil
}

//Import is for importing command line arguments.
//It uses the following rules:
//	If args is len 0, try to import the current directory.
//	Otherwise, for each argument:
//		If it ends with ..., use goutil.ImportTree (unless notree is true)
//		Otherwise, use goutil.Import
//Regardless, the ctx is passed as is to the various importers.
//
//Import returns a list of any errors that resulted from attmepting to import.
//If you only care about the first error, wrap the call in FirstError.
func Import(notree bool, ctx *build.Context, args []string) (pkgs []goutil.Packages, errs []error) {
	push := func(ps goutil.Packages, err error) {
		if len(ps) > 0 {
			pkgs = append(pkgs, ps)
		}
		if err != nil {
			errs = append(errs, err)
		}
	}

	if len(args) == 0 {
		push(imports(ctx, ""))
		return
	}

	for _, arg := range args {
		if filepath.Base(arg) == "..." {
			if notree {
				push(nil, errors.New("cannot use ... imports"))
			} else {
				push(goutil.ImportTree(ctx, filepath.Dir(arg)))
			}
		} else {
			push(imports(ctx, arg))
		}
	}
	return
}

//FirstError is meant to wrap Import, it returns pkgs unchanged.
//If there were errors, only the first is returned.
func FirstError(pkgs []goutil.Packages, errs []error) ([]goutil.Packages, error) {
	if len(errs) == 0 {
		return pkgs, nil
	}
	return pkgs, errs[0]
}

//Flatten takes a slice of goutil.Packages and returns a single goutil.Packages
//containing only the unique packages from the slice.
func Flatten(pss []goutil.Packages) (out goutil.Packages) {
	for _, ps := range pss {
		out = append(out, ps...)
	}
	return out.Uniq()
}

//ImportOne only allows one argument to be specified.
func ImportOne(notree bool, ctx *build.Context, args []string) (goutil.Packages, error) {
	if len(args) > 1 {
		return nil, errors.New("only one package may be specified")
	}
	ps, err := FirstError(Import(notree, ctx, args))
	if err != nil {
		return nil, err
	}
	return ps[0], nil
}
<|endoftext|>


<commit_before>package gcs

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"strconv"

	"cloud.google.com/go/storage"
	multierror "github.com/hashicorp/go-multierror"
	"github.com/hashicorp/terraform/state"
	"github.com/hashicorp/terraform/state/remote"
	"golang.org/x/net/context"
)

// remoteClient is used by "state/remote".State to read and write
// blobs representing state.
// Implements "state/remote".ClientLocker
type remoteClient struct {
	storageContext context.Context
	storageClient  *storage.Client
	bucketName     string
	stateFilePath  string
	lockFilePath   string
	encryptionKey  []byte
}

func (c *remoteClient) Get() (payload *remote.Payload, err error) {
	stateFileReader, err := c.stateFile().NewReader(c.storageContext)
	if err != nil {
		if err == storage.ErrObjectNotExist {
			return nil, nil
		} else {
			return nil, fmt.Errorf("Failed to open state file at %v: %v", c.stateFileURL(), err)
		}
	}
	defer stateFileReader.Close()

	stateFileContents, err := ioutil.ReadAll(stateFileReader)
	if err != nil {
		return nil, fmt.Errorf("Failed to read state file from %v: %v", c.stateFileURL(), err)
	}

	stateFileAttrs, err := c.stateFile().Attrs(c.storageContext)
	if err != nil {
		return nil, fmt.Errorf("Failed to read state file attrs from %v: %v", c.stateFileURL(), err)
	}

	result := &remote.Payload{
		Data: stateFileContents,
		MD5:  stateFileAttrs.MD5,
	}

	return result, nil
}

func (c *remoteClient) Put(data []byte) error {
	err := func() error {
		stateFileWriter := c.stateFile().NewWriter(c.storageContext)
		if _, err := stateFileWriter.Write(data); err != nil {
			return err
		}
		return stateFileWriter.Close()
	}()
	if err != nil {
		return fmt.Errorf("Failed to upload state to %v: %v", c.stateFileURL(), err)
	}

	return nil
}

func (c *remoteClient) Delete() error {
	if err := c.stateFile().Delete(c.storageContext); err != nil {
		return fmt.Errorf("Failed to delete state file %v: %v", c.stateFileURL(), err)
	}

	return nil
}

// Lock writes to a lock file, ensuring file creation. Returns the generation
// number, which must be passed to Unlock().
func (c *remoteClient) Lock(info *state.LockInfo) (string, error) {
	infoJson, err := json.Marshal(info)
	if err != nil {
		return "", err
	}

	lockFile := c.lockFile()
	w := lockFile.If(storage.Conditions{DoesNotExist: true}).NewWriter(c.storageContext)
	err = func() error {
		if _, err := w.Write(infoJson); err != nil {
			return err
		}
		return w.Close()
	}()
	if err != nil {
		return "", c.lockError(fmt.Errorf("writing %q failed: %v", c.lockFileURL(), err))
	}

	info.ID = strconv.FormatInt(w.Attrs().Generation, 10)
	info.Path = c.lockFileURL()

	return info.ID, nil
}

func (c *remoteClient) Unlock(id string) error {
	gen, err := strconv.ParseInt(id, 10, 64)
	if err != nil {
		return err
	}

	if err := c.lockFile().If(storage.Conditions{GenerationMatch: gen}).Delete(c.storageContext); err != nil {
		return c.lockError(err)
	}

	return nil
}

func (c *remoteClient) lockError(err error) *state.LockError {
	lockErr := &state.LockError{
		Err: err,
	}

	info, infoErr := c.lockInfo()
	if infoErr != nil {
		lockErr.Err = multierror.Append(lockErr.Err, infoErr)
	} else {
		lockErr.Info = info
	}
	return lockErr
}

// lockInfo reads the lock file, parses its contents and returns the parsed
// LockInfo struct.
func (c *remoteClient) lockInfo() (*state.LockInfo, error) {
	r, err := c.lockFile().NewReader(c.storageContext)
	if err != nil {
		return nil, err
	}
	defer r.Close()

	rawData, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, err
	}

	info := &state.LockInfo{}
	if err := json.Unmarshal(rawData, info); err != nil {
		return nil, err
	}

	return info, nil
}

func (c *remoteClient) stateFile() *storage.ObjectHandle {
	h := c.storageClient.Bucket(c.bucketName).Object(c.stateFilePath)
	if len(c.encryptionKey) > 0 {
		return h.Key(c.encryptionKey)
	}
	return h
}

func (c *remoteClient) stateFileURL() string {
	return fmt.Sprintf("gs://%v/%v", c.bucketName, c.stateFilePath)
}

func (c *remoteClient) lockFile() *storage.ObjectHandle {
	return c.storageClient.Bucket(c.bucketName).Object(c.lockFilePath)
}

func (c *remoteClient) lockFileURL() string {
	return fmt.Sprintf("gs://%v/%v", c.bucketName, c.lockFilePath)
}
<commit_msg>report the proper lock id from a state lock error<commit_after>package gcs

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"strconv"

	"cloud.google.com/go/storage"
	multierror "github.com/hashicorp/go-multierror"
	"github.com/hashicorp/terraform/state"
	"github.com/hashicorp/terraform/state/remote"
	"golang.org/x/net/context"
)

// remoteClient is used by "state/remote".State to read and write
// blobs representing state.
// Implements "state/remote".ClientLocker
type remoteClient struct {
	storageContext context.Context
	storageClient  *storage.Client
	bucketName     string
	stateFilePath  string
	lockFilePath   string
	encryptionKey  []byte
}

func (c *remoteClient) Get() (payload *remote.Payload, err error) {
	stateFileReader, err := c.stateFile().NewReader(c.storageContext)
	if err != nil {
		if err == storage.ErrObjectNotExist {
			return nil, nil
		} else {
			return nil, fmt.Errorf("Failed to open state file at %v: %v", c.stateFileURL(), err)
		}
	}
	defer stateFileReader.Close()

	stateFileContents, err := ioutil.ReadAll(stateFileReader)
	if err != nil {
		return nil, fmt.Errorf("Failed to read state file from %v: %v", c.stateFileURL(), err)
	}

	stateFileAttrs, err := c.stateFile().Attrs(c.storageContext)
	if err != nil {
		return nil, fmt.Errorf("Failed to read state file attrs from %v: %v", c.stateFileURL(), err)
	}

	result := &remote.Payload{
		Data: stateFileContents,
		MD5:  stateFileAttrs.MD5,
	}

	return result, nil
}

func (c *remoteClient) Put(data []byte) error {
	err := func() error {
		stateFileWriter := c.stateFile().NewWriter(c.storageContext)
		if _, err := stateFileWriter.Write(data); err != nil {
			return err
		}
		return stateFileWriter.Close()
	}()
	if err != nil {
		return fmt.Errorf("Failed to upload state to %v: %v", c.stateFileURL(), err)
	}

	return nil
}

func (c *remoteClient) Delete() error {
	if err := c.stateFile().Delete(c.storageContext); err != nil {
		return fmt.Errorf("Failed to delete state file %v: %v", c.stateFileURL(), err)
	}

	return nil
}

// Lock writes to a lock file, ensuring file creation. Returns the generation
// number, which must be passed to Unlock().
func (c *remoteClient) Lock(info *state.LockInfo) (string, error) {
	// update the path we're using
	// we can't set the ID until the info is written
	info.Path = c.lockFileURL()

	infoJson, err := json.Marshal(info)
	if err != nil {
		return "", err
	}

	lockFile := c.lockFile()
	w := lockFile.If(storage.Conditions{DoesNotExist: true}).NewWriter(c.storageContext)
	err = func() error {
		if _, err := w.Write(infoJson); err != nil {
			return err
		}
		return w.Close()
	}()

	if err != nil {
		return "", c.lockError(fmt.Errorf("writing %q failed: %v", c.lockFileURL(), err))
	}

	info.ID = strconv.FormatInt(w.Attrs().Generation, 10)

	return info.ID, nil
}

func (c *remoteClient) Unlock(id string) error {
	gen, err := strconv.ParseInt(id, 10, 64)
	if err != nil {
		return err
	}

	if err := c.lockFile().If(storage.Conditions{GenerationMatch: gen}).Delete(c.storageContext); err != nil {
		return c.lockError(err)
	}

	return nil
}

func (c *remoteClient) lockError(err error) *state.LockError {
	lockErr := &state.LockError{
		Err: err,
	}

	info, infoErr := c.lockInfo()
	if infoErr != nil {
		lockErr.Err = multierror.Append(lockErr.Err, infoErr)
	} else {
		lockErr.Info = info
	}
	return lockErr
}

// lockInfo reads the lock file, parses its contents and returns the parsed
// LockInfo struct.
func (c *remoteClient) lockInfo() (*state.LockInfo, error) {
	r, err := c.lockFile().NewReader(c.storageContext)
	if err != nil {
		return nil, err
	}
	defer r.Close()

	rawData, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, err
	}

	info := &state.LockInfo{}
	if err := json.Unmarshal(rawData, info); err != nil {
		return nil, err
	}

	// We use the Generation as the ID, so overwrite the ID in the json.
	// This can't be written into the Info, since the generation isn't known
	// until it's written.
	attrs, err := c.lockFile().Attrs(c.storageContext)
	if err != nil {
		return nil, err
	}
	info.ID = strconv.FormatInt(attrs.Generation, 10)

	return info, nil
}

func (c *remoteClient) stateFile() *storage.ObjectHandle {
	h := c.storageClient.Bucket(c.bucketName).Object(c.stateFilePath)
	if len(c.encryptionKey) > 0 {
		return h.Key(c.encryptionKey)
	}
	return h
}

func (c *remoteClient) stateFileURL() string {
	return fmt.Sprintf("gs://%v/%v", c.bucketName, c.stateFilePath)
}

func (c *remoteClient) lockFile() *storage.ObjectHandle {
	return c.storageClient.Bucket(c.bucketName).Object(c.lockFilePath)
}

func (c *remoteClient) lockFileURL() string {
	return fmt.Sprintf("gs://%v/%v", c.bucketName, c.lockFilePath)
}
<|endoftext|>


<commit_before>package velocity

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"sync"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/client"
	yaml "gopkg.in/yaml.v2"
)

type DockerCompose struct {
	BaseStep
	ComposeFile string `json:"composeFile" yaml:"composeFile"`
	Contents    dockerComposeYaml
}

func NewDockerCompose(y string) *DockerCompose {
	step := DockerCompose{
		BaseStep: BaseStep{
			Type: "compose",
		},
	}
	err := yaml.Unmarshal([]byte(y), &step)
	if err != nil {
		panic(err)
	}

	dir, _ := os.Getwd()
	dockerComposeYml, _ := ioutil.ReadFile(fmt.Sprintf("%s/%s", dir, step.ComposeFile))
	err = yaml.Unmarshal(dockerComposeYml, &step.Contents)
	if err != nil {
		panic(err)
	}

	services := make([]string, len(step.Contents.Services))
	i := 0
	for k := range step.Contents.Services {
		services[i] = k
		i++
	}
	step.OutputStreams = services

	return &step
}

func (dC DockerCompose) GetDetails() string {
	return fmt.Sprintf("composeFile: %s", dC.ComposeFile)
}

func (dC *DockerCompose) Validate(params map[string]Parameter) error {
	return nil
}

func (dC *DockerCompose) SetParams(params map[string]Parameter) error {
	return nil
}

func (dC *DockerCompose) Execute(emitter Emitter, params map[string]Parameter) error {
	serviceOrder := getServiceOrder(dC.Contents.Services, []string{})

	services := []*serviceRunner{}
	var wg sync.WaitGroup
	cli, _ := client.NewEnvClient()
	ctx := context.Background()

	networkResp, err := cli.NetworkCreate(ctx, fmt.Sprintf("vci-%s", dC.GetRunID()), types.NetworkCreate{
		Labels: map[string]string{"owner": "velocity-ci"},
	})
	if err != nil {
		log.Println(err)
	}
	log.Println(networkResp.ID)

	writers := map[string]StreamWriter{}
	// Create writers
	for _, serviceName := range serviceOrder {
		writers[serviceName] = emitter.NewStreamWriter(serviceName)
	}

	for _, serviceName := range serviceOrder {
		writer := writers[serviceName]
		writer.SetStatus(StateRunning)
		writer.Write([]byte(fmt.Sprintf("Configured %s", serviceName)))

		s := dC.Contents.Services[serviceName]

		// generate containerConfig + hostConfig
		containerConfig, hostConfig := generateContainerAndHostConfig(s)

		// Create service runners
		sR := newServiceRunner(
			cli,
			ctx,
			writer,
			&wg,
			params,
			fmt.Sprintf("%s-%s", dC.GetRunID(), serviceName),
			s.Image,
			&s.Build,
			containerConfig,
			hostConfig,
			networkResp.ID,
		)

		services = append(services, sR)
	}

	// Pull/Build images
	for _, serviceRunner := range services {
		serviceRunner.PullOrBuild()
	}

	// Create services
	for _, serviceRunner := range services {
		serviceRunner.Create()
	}
	stopServicesChannel := make(chan string, 32)
	// Start services
	for _, serviceRunner := range services {
		wg.Add(1)
		go serviceRunner.Run(stopServicesChannel)
	}

	_ = <-stopServicesChannel
	for _, s := range services {
		s.Stop()
	}
	wg.Wait()
	err = cli.NetworkRemove(ctx, networkResp.ID)
	if err != nil {
		log.Printf("network %s remove err: %s", networkResp.ID, err)
	}
	success := true
	for _, serviceRunner := range services {
		if serviceRunner.exitCode != 0 {
			success = false

			break
		}
	}

	if !success {
		for _, serviceName := range serviceOrder {
			writers[serviceName].SetStatus(StateFailed)
			writers[serviceName].Write([]byte(fmt.Sprintf("%s\n### FAILED \x1b[0m", errorANSI)))
		}
	} else {
		for _, serviceName := range serviceOrder {
			writers[serviceName].SetStatus(StateSuccess)
			writers[serviceName].Write([]byte(fmt.Sprintf("%s\n### SUCCESS \x1b[0m", successANSI)))
		}
	}

	return nil
}

func (dC *DockerCompose) String() string {
	j, _ := json.Marshal(dC)
	return string(j)
}

func generateContainerAndHostConfig(s dockerComposeService) (*container.Config, *container.HostConfig) {
	containerConfig := &container.Config{}
	if len(s.Command) > 0 {
		// containerConfig.Cmd = s.Command
	}
	return containerConfig, &container.HostConfig{}
}

func getServiceOrder(services map[string]dockerComposeService, serviceOrder []string) []string {
	for serviceName, serviceDef := range services {
		if isIn(serviceName, serviceOrder) {
			break
		}
		for _, linkedService := range serviceDef.Links {
			serviceOrder = getLinkedServiceOrder(linkedService, services, serviceOrder)
		}
		serviceOrder = append(serviceOrder, serviceName)
	}

	for len(services) != len(serviceOrder) {
		serviceOrder = getServiceOrder(services, serviceOrder)
	}

	return serviceOrder
}

func getLinkedServiceOrder(serviceName string, services map[string]dockerComposeService, serviceOrder []string) []string {
	if isIn(serviceName, serviceOrder) {
		return serviceOrder
	}
	for _, linkedService := range services[serviceName].Links {
		serviceOrder = getLinkedServiceOrder(linkedService, services, serviceOrder)
	}
	return append(serviceOrder, serviceName)
}

func isIn(needle string, haystack []string) bool {
	for _, v := range haystack {
		if needle == v {
			return true
		}
	}
	return false
}

type dockerComposeYaml struct {
	Services map[string]dockerComposeService `json:"services" yaml:"services"`
}

type dockerComposeService struct {
	Image       string                    `json:"image" yaml:"image"`
	Build       dockerComposeServiceBuild `json:"build" yaml:"build"`
	WorkingDir  string                    `json:"workingDir" yaml:"working_dir"`
	Command     []string                  `json:"command" yaml:"command"`
	Links       []string                  `json:"links" yaml:"links"`
	Environment map[string]string         `json:"environment" yaml:"environment"`
	Volumes     []string                  `json:"volumes" yaml:"volumes"`
	Expose      []string                  `json:"expose" yaml:"expose"`
}

// func (a *dockerComposeService) UnmarshalYAML(unmarshal func(interface{}) error) error {
// 	var multi []string
// 	err := unmarshal(&multi)
// 	if err != nil {
// 		var single string
// 		err := unmarshal(&single)
// 		if err != nil {
// 			return err
// 		}
// 		*a = []string{single}
// 	} else {
// 		*a = multi
// 	}
// 	return nil
// }

type dockerComposeServiceBuild struct {
	Context    string `json:"context" yaml:"context"`
	Dockerfile string `json:"dockerfile" yaml:"dockerfile"`
}
<commit_msg>[backend] Parsing docker-compose files. Need to resolve volume paths properly relative to docker-compose file being used.<commit_after>package velocity

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"strings"
	"sync"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/client"
	yaml "gopkg.in/yaml.v2"
)

type DockerCompose struct {
	BaseStep
	ComposeFile string `json:"composeFile" yaml:"composeFile"`
	Contents    dockerComposeYaml
}

func NewDockerCompose(y string) *DockerCompose {
	step := DockerCompose{
		BaseStep: BaseStep{
			Type: "compose",
		},
	}
	err := yaml.Unmarshal([]byte(y), &step)
	if err != nil {
		panic(err)
	}

	dir, _ := os.Getwd()
	dockerComposeYml, _ := ioutil.ReadFile(fmt.Sprintf("%s/%s", dir, step.ComposeFile))
	err = yaml.Unmarshal(dockerComposeYml, &step.Contents)
	if err != nil {
		panic(err)
	}

	services := make([]string, len(step.Contents.Services))
	i := 0
	for k := range step.Contents.Services {
		services[i] = k
		i++
	}
	step.OutputStreams = services

	return &step
}

func (dC DockerCompose) GetDetails() string {
	return fmt.Sprintf("composeFile: %s", dC.ComposeFile)
}

func (dC *DockerCompose) Validate(params map[string]Parameter) error {
	return nil
}

func (dC *DockerCompose) SetParams(params map[string]Parameter) error {
	return nil
}

func (dC *DockerCompose) Execute(emitter Emitter, params map[string]Parameter) error {
	serviceOrder := getServiceOrder(dC.Contents.Services, []string{})

	services := []*serviceRunner{}
	var wg sync.WaitGroup
	cli, _ := client.NewEnvClient()
	ctx := context.Background()

	networkResp, err := cli.NetworkCreate(ctx, fmt.Sprintf("vci-%s", dC.GetRunID()), types.NetworkCreate{
		Labels: map[string]string{"owner": "velocity-ci"},
	})
	if err != nil {
		log.Println(err)
	}
	log.Println(networkResp.ID)

	writers := map[string]StreamWriter{}
	// Create writers
	for _, serviceName := range serviceOrder {
		writers[serviceName] = emitter.NewStreamWriter(serviceName)
	}

	for _, serviceName := range serviceOrder {
		writer := writers[serviceName]
		writer.SetStatus(StateRunning)
		writer.Write([]byte(fmt.Sprintf("Configured %s", serviceName)))

		s := dC.Contents.Services[serviceName]

		// generate containerConfig + hostConfig
		containerConfig, hostConfig := generateContainerAndHostConfig(s)

		// Create service runners
		sR := newServiceRunner(
			cli,
			ctx,
			writer,
			&wg,
			params,
			fmt.Sprintf("%s-%s", dC.GetRunID(), serviceName),
			s.Image,
			&s.Build,
			containerConfig,
			hostConfig,
			networkResp.ID,
		)

		services = append(services, sR)
	}

	// Pull/Build images
	for _, serviceRunner := range services {
		serviceRunner.PullOrBuild()
	}

	// Create services
	for _, serviceRunner := range services {
		serviceRunner.Create()
	}
	stopServicesChannel := make(chan string, 32)
	// Start services
	for _, serviceRunner := range services {
		wg.Add(1)
		go serviceRunner.Run(stopServicesChannel)
	}

	_ = <-stopServicesChannel
	for _, s := range services {
		s.Stop()
	}
	wg.Wait()
	err = cli.NetworkRemove(ctx, networkResp.ID)
	if err != nil {
		log.Printf("network %s remove err: %s", networkResp.ID, err)
	}
	success := true
	for _, serviceRunner := range services {
		if serviceRunner.exitCode != 0 {
			success = false

			break
		}
	}

	if !success {
		for _, serviceName := range serviceOrder {
			writers[serviceName].SetStatus(StateFailed)
			writers[serviceName].Write([]byte(fmt.Sprintf("%s\n### FAILED \x1b[0m", errorANSI)))
		}
	} else {
		for _, serviceName := range serviceOrder {
			writers[serviceName].SetStatus(StateSuccess)
			writers[serviceName].Write([]byte(fmt.Sprintf("%s\n### SUCCESS \x1b[0m", successANSI)))
		}
	}

	return nil
}

func (dC *DockerCompose) String() string {
	j, _ := json.Marshal(dC)
	return string(j)
}

func generateContainerAndHostConfig(s dockerComposeService) (*container.Config, *container.HostConfig) {
	containerConfig := &container.Config{}
	if len(s.Command) > 0 {
		// containerConfig.Cmd = s.Command
	}
	return containerConfig, &container.HostConfig{}
}

func getServiceOrder(services map[string]dockerComposeService, serviceOrder []string) []string {
	for serviceName, serviceDef := range services {
		if isIn(serviceName, serviceOrder) {
			break
		}
		for _, linkedService := range serviceDef.Links {
			serviceOrder = getLinkedServiceOrder(linkedService, services, serviceOrder)
		}
		serviceOrder = append(serviceOrder, serviceName)
	}

	for len(services) != len(serviceOrder) {
		serviceOrder = getServiceOrder(services, serviceOrder)
	}

	return serviceOrder
}

func getLinkedServiceOrder(serviceName string, services map[string]dockerComposeService, serviceOrder []string) []string {
	if isIn(serviceName, serviceOrder) {
		return serviceOrder
	}
	for _, linkedService := range services[serviceName].Links {
		serviceOrder = getLinkedServiceOrder(linkedService, services, serviceOrder)
	}
	return append(serviceOrder, serviceName)
}

func isIn(needle string, haystack []string) bool {
	for _, v := range haystack {
		if needle == v {
			return true
		}
	}
	return false
}

type dockerComposeYaml struct {
	Services map[string]dockerComposeService `json:"services" yaml:"services"`
}

type dockerComposeService struct {
	Image       string                    `json:"image" yaml:"image"`
	Build       dockerComposeServiceBuild `json:"build" yaml:"build"`
	WorkingDir  string                    `json:"workingDir" yaml:"working_dir"`
	Command     []string                  `json:"command" yaml:"command"`
	Links       []string                  `json:"links" yaml:"links"`
	Environment map[string]string         `json:"environment" yaml:"environment"`
	Volumes     []string                  `json:"volumes" yaml:"volumes"`
	Expose      []string                  `json:"expose" yaml:"expose"`
}

func (a *dockerComposeService) UnmarshalYAML(unmarshal func(interface{}) error) error {
	var serviceMap map[string]interface{}
	err := unmarshal(&serviceMap)
	if err != nil {
		log.Printf("unable to unmarshal service")
		return err
	}
	// log.Printf("serviceMap: %+v\n", serviceMap)

	// image
	switch x := serviceMap["image"].(type) {
	case interface{}:
		a.Image = x.(string)
		break
	default:
		break
	}

	// build
	switch x := serviceMap["build"].(type) {
	case string:
		// use string as context path. Dockerfile in root of that path
		a.Build = dockerComposeServiceBuild{
			Context:    x, // get path of docker-compose file
			Dockerfile: "Dockerfile",
		}
		// a.Image = x.(string)
		break
	case map[interface{}]interface{}:
		a.Build = dockerComposeServiceBuild{
			Context:    x["context"].(string), // get path of docker-compose file
			Dockerfile: x["dockerfile"].(string),
		}
		break
	default:
		break
	}

	// command
	switch x := serviceMap["command"].(type) {
	case []interface{}:
		for _, p := range x {
			a.Command = append(a.Command, p.(string))
		}
		break
	case interface{}:
		// TODO: handle /bin/sh -c "sleep 3"; should be: ["/bin/sh", "-c", "\"sleep 3\""]
		a.Command = strings.Split(x.(string), " ")
		break
	default:
		break
	}

	// environment
	a.Environment = map[string]string{}
	switch x := serviceMap["environment"].(type) {
	case []interface{}:
		for _, e := range x {
			parts := strings.Split(e.(string), "=")
			key := parts[0]
			val := parts[1]
			a.Environment[key] = val
		}
		break
	case map[interface{}]interface{}:
		for k, v := range x {
			if num, ok := v.(int); ok {
				v = strconv.Itoa(num)
			}
			a.Environment[k.(string)] = v.(string)
		}
		break
	default:
		log.Println("no environment specified")
		break
	}

	// volumes
	switch x := serviceMap["volumes"].(type) {
	case []interface{}:
		for _, v := range x {
			a.Volumes = append(a.Volumes, v.(string))
		}
		break
	}

	// links
	switch x := serviceMap["links"].(type) {
	case []interface{}:
		for _, v := range x {
			a.Links = append(a.Links, v.(string))
		}
		break
	}

	// expose
	switch x := serviceMap["expose"].(type) {
	case []interface{}:
		for _, v := range x {
			a.Expose = append(a.Expose, v.(string))
		}
		break
	}

	return nil
}

type dockerComposeServiceBuild struct {
	Context    string `json:"context" yaml:"context"`
	Dockerfile string `json:"dockerfile" yaml:"dockerfile"`
}
<|endoftext|>


<commit_before>package benchmark

import (
	"fmt"
	"testing"
)

func BenchmarkFoo(b *testing.B) {
	b.Run("Bar", func(b *testing.B) {
		for i := 0; i < b.N; i++ {
			_ = fmt.Sprint("%d", i)
		}
	})
}
<commit_msg>Add benchmark<commit_after>package benchmark

import (
	"reflect"
	"testing"
	"time"
)

func Test_Divide(t *testing.T) {
	const N = 1000000
	tmp := float64(1) / float64(time.Millisecond)
	for j := 0; j < N; j++ {
		f1 := int64(j) / int64(time.Millisecond)
		f2 := int64(float64(j) * tmp)
		if !reflect.DeepEqual(f1, f2) {
			t.Fatal("Should be the same values")
		}
	}
}

var f int64

func Benchmark_Divide(b *testing.B) {
	const N = 1000000
	b.Run("Divide", func(b *testing.B) {
		for i := 0; i < b.N; i++ {
			for j := 0; j < N; j++ {
				f = int64(j) / int64(time.Millisecond)
			}
		}
	})

	b.Run("Multiple", func(b *testing.B) {
		t := float64(1) / float64(time.Millisecond)
		for i := 0; i < b.N; i++ {
			for j := 0; j < N; j++ {
				f = int64(float64(j) * t)
			}
		}
	})
}
<|endoftext|>


<commit_before>// Copyright © 2017 Aaron Donovan <amdonov@gmail.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package idp

import (
	"bytes"
	"compress/flate"
	"crypto"
	"crypto/rsa"
	"crypto/sha1"
	"encoding/base64"
	"encoding/xml"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strings"

	"github.com/amdonov/lite-idp/model"
	"github.com/amdonov/lite-idp/saml"
	"github.com/golang/protobuf/proto"
	"github.com/google/uuid"
	log "github.com/sirupsen/logrus"
)

func (i *IDP) validateRequest(request *saml.AuthnRequest, r *http.Request) error {
	// Only accept requests from registered service providers
	if request.Issuer == "" {
		return errors.New("request does not contain an issuer")
	}
	log.Infof("received authentication request from %s", request.Issuer)
	sp, ok := i.sps[request.Issuer]
	if !ok {
		return errors.New("request from an unregistered issuer")
	}
	// Determine the right assertion consumer service
	var acs *AssertionConsumerService
	for i, a := range sp.AssertionConsumerServices {
		// Find either the matching service or the default
		if a.Index == request.AssertionConsumerServiceIndex {
			acs = &sp.AssertionConsumerServices[i]
			break
		}
		if a.Location == request.AssertionConsumerServiceURL {
			acs = &sp.AssertionConsumerServices[i]
			break
		}
		if a.IsDefault {
			acs = &sp.AssertionConsumerServices[i]
		}
	}
	if acs == nil {
		return errors.New("unable to determine assertion consumer service")
	}
	// Don't allow a different URL than specified in the metadata
	if request.AssertionConsumerServiceURL == "" {
		request.AssertionConsumerServiceURL = acs.Location
	} else if request.AssertionConsumerServiceURL != acs.Location {
		return errors.New("assertion consumer location in request does not match metadata")
	}
	// At this point, we're OK with the request
	// Need to validate the signature
	// Have to use the raw query as pointed out in the spec.
	// https://docs.oasis-open.org/security/saml/v2.0/saml-bindings-2.0-os.pdf
	// Line 621

	// Split up the parts
	params := strings.Split(r.URL.RawQuery, "&")
	pMap := make(map[string]string, len(params))
	for i := range params {
		parts := strings.Split(params[i], "=")
		if len(parts) != 2 {
			return errors.New("trouble validating signature on request")
		}
		pMap[parts[0]] = parts[1]
	}
	// Order them
	sigparts := []string{fmt.Sprintf("SAMLRequest=%s", pMap["SAMLRequest"])}
	if state, ok := pMap["RelayState"]; ok {
		sigparts = append(sigparts, fmt.Sprintf("RelayState=%s", state))
	}
	sigparts = append(sigparts, fmt.Sprintf("SigAlg=%s", pMap["SigAlg"]))
	sig := []byte(strings.Join(sigparts, "&"))
	fmt.Println("REQUEST TO SIGN =======")
	fmt.Println(strings.Join(sigparts, "&"))
	fmt.Println("REQUEST TO SIGN =======")
	// Validate the signature
	switch r.Form.Get("SigAlg") {
	case "http://www.w3.org/2000/09/xmldsig#dsa-sha1":
		//return dsa.VerifyPKCS1v15(sp.publicKey.(*dsa.PublicKey), crypto.SHA1, sig, signature)
		return nil
	case "http://www.w3.org/2000/09/xmldsig#rsa-sha1":
		signature, err := base64.StdEncoding.DecodeString(r.Form.Get("Signature"))
		if err != nil {
			return err
		}
		h := sha1.New()
		h.Write(sig)
		sum := h.Sum(nil)
		return rsa.VerifyPKCS1v15(sp.publicKey.(*rsa.PublicKey), crypto.SHA1, sum, signature)
	default:
		return fmt.Errorf("unsupported signature algorithm, %s", r.Form.Get("SigAlg"))
	}
}

func (i *IDP) DefaultRedirectSSOHandler() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		err := func() error {
			err := r.ParseForm()
			if err != nil {
				return err
			}
			relayState := r.Form.Get("RelayState")
			if len(relayState) > 80 {
				return errors.New("RelayState cannot be longer than 80 characters")
			}

			samlReq := r.Form.Get("SAMLRequest")
			// URL decoding is already performed
			// remove base64 encoding
			reqBytes, err := base64.StdEncoding.DecodeString(samlReq)
			if err != nil {
				return err
			}
			// Remove deflate
			req := flate.NewReader(bytes.NewReader(reqBytes))
			// Read the XML
			decoder := xml.NewDecoder(req)
			loginReq := &saml.AuthnRequest{}
			if err = decoder.Decode(loginReq); err != nil {
				return err
			}

			if err = i.validateRequest(loginReq, r); err != nil {
				return err
			}

			// create saveable request
			saveableRequest, err := model.NewAuthnRequest(loginReq, relayState)
			if err != nil {
				return err
			}

			// check for cookie to see if user has a current session
			if cookie, err := r.Cookie(i.cookieName); err == nil {
				// Found a session cookie
				if data, err := i.UserCache.Get(cookie.Value); err == nil {
					// Cookie matched user in cache
					user := &model.User{}
					if err = proto.Unmarshal(data, user); err == nil {
						log.Infof("found existing session for %s", user.Name)
						return i.respond(saveableRequest, user, w, r)
					}
				}
			}

			// check to see if they presented a client cert
			if clientCert, err := getCertFromRequest(r); err == nil {
				user := &model.User{
					Name:    getSubjectDN(clientCert.Subject),
					Format:  "urn:oasis:names:tc:SAML:1.1:nameid-format:X509SubjectName",
					Context: "urn:oasis:names:tc:SAML:2.0:ac:classes:X509",
					IP:      getIP(r).String()}

				// Add attributes
				err = i.setUserAttributes(user)
				if err != nil {
					return err
				}
				log.Infof("successful PKI login for %s", user.Name)
				return i.respond(saveableRequest, user, w, r)
			}
			// need to display the login form
			data, err := proto.Marshal(saveableRequest)
			if err != nil {
				return err
			}
			id := uuid.New().String()
			err = i.TempCache.Set(id, data)
			if err != nil {
				return err
			}
			http.Redirect(w, r, fmt.Sprintf("/ui/login.html?requestId=%s",
				url.QueryEscape(id)), http.StatusTemporaryRedirect)
			return nil
		}()
		if err != nil {
			log.Error(err)
			http.Error(w, err.Error(), http.StatusBadRequest)
		}
	}
}
<commit_msg>Support DSA signatures<commit_after>// Copyright © 2017 Aaron Donovan <amdonov@gmail.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package idp

import (
	"bytes"
	"compress/flate"
	"crypto"
	"crypto/dsa"
	"crypto/rsa"
	"crypto/sha1"
	"encoding/asn1"
	"encoding/base64"
	"encoding/xml"
	"errors"
	"fmt"
	"math/big"
	"net/http"
	"net/url"
	"strings"

	"github.com/amdonov/lite-idp/model"
	"github.com/amdonov/lite-idp/saml"
	"github.com/golang/protobuf/proto"
	"github.com/google/uuid"
	log "github.com/sirupsen/logrus"
)

func (i *IDP) validateRequest(request *saml.AuthnRequest, r *http.Request) error {
	// Only accept requests from registered service providers
	if request.Issuer == "" {
		return errors.New("request does not contain an issuer")
	}
	log.Infof("received authentication request from %s", request.Issuer)
	sp, ok := i.sps[request.Issuer]
	if !ok {
		return errors.New("request from an unregistered issuer")
	}
	// Determine the right assertion consumer service
	var acs *AssertionConsumerService
	for i, a := range sp.AssertionConsumerServices {
		// Find either the matching service or the default
		if a.Index == request.AssertionConsumerServiceIndex {
			acs = &sp.AssertionConsumerServices[i]
			break
		}
		if a.Location == request.AssertionConsumerServiceURL {
			acs = &sp.AssertionConsumerServices[i]
			break
		}
		if a.IsDefault {
			acs = &sp.AssertionConsumerServices[i]
		}
	}
	if acs == nil {
		return errors.New("unable to determine assertion consumer service")
	}
	// Don't allow a different URL than specified in the metadata
	if request.AssertionConsumerServiceURL == "" {
		request.AssertionConsumerServiceURL = acs.Location
	} else if request.AssertionConsumerServiceURL != acs.Location {
		return errors.New("assertion consumer location in request does not match metadata")
	}
	// At this point, we're OK with the request
	// Need to validate the signature
	// Have to use the raw query as pointed out in the spec.
	// https://docs.oasis-open.org/security/saml/v2.0/saml-bindings-2.0-os.pdf
	// Line 621

	// Split up the parts
	params := strings.Split(r.URL.RawQuery, "&")
	pMap := make(map[string]string, len(params))
	for i := range params {
		parts := strings.Split(params[i], "=")
		if len(parts) != 2 {
			return errors.New("trouble validating signature on request")
		}
		pMap[parts[0]] = parts[1]
	}
	// Order them
	sigparts := []string{fmt.Sprintf("SAMLRequest=%s", pMap["SAMLRequest"])}
	if state, ok := pMap["RelayState"]; ok {
		sigparts = append(sigparts, fmt.Sprintf("RelayState=%s", state))
	}
	sigparts = append(sigparts, fmt.Sprintf("SigAlg=%s", pMap["SigAlg"]))
	sig := []byte(strings.Join(sigparts, "&"))
	// Validate the signature
	signature, err := base64.StdEncoding.DecodeString(r.Form.Get("Signature"))
	if err != nil {
		return err
	}
	h := sha1.New()
	h.Write(sig)
	sum := h.Sum(nil)
	switch r.Form.Get("SigAlg") {
	case "http://www.w3.org/2000/09/xmldsig#dsa-sha1":
		dsaSig := new(dsaSignature)
		if rest, err := asn1.Unmarshal(signature, dsaSig); err != nil {
			return err
		} else if len(rest) != 0 {
			return errors.New("trailing data after DSA signature")
		}
		if dsaSig.R.Sign() <= 0 || dsaSig.S.Sign() <= 0 {
			return errors.New("DSA signature contained zero or negative values")
		}
		if !dsa.Verify(sp.publicKey.(*dsa.PublicKey), sum, dsaSig.R, dsaSig.S) {
			return errors.New("DSA verification failure")
		}
		return nil
	case "http://www.w3.org/2000/09/xmldsig#rsa-sha1":
		return rsa.VerifyPKCS1v15(sp.publicKey.(*rsa.PublicKey), crypto.SHA1, sum, signature)
	default:
		return fmt.Errorf("unsupported signature algorithm, %s", r.Form.Get("SigAlg"))
	}
}

func (i *IDP) DefaultRedirectSSOHandler() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		err := func() error {
			err := r.ParseForm()
			if err != nil {
				return err
			}
			relayState := r.Form.Get("RelayState")
			if len(relayState) > 80 {
				return errors.New("RelayState cannot be longer than 80 characters")
			}

			samlReq := r.Form.Get("SAMLRequest")
			// URL decoding is already performed
			// remove base64 encoding
			reqBytes, err := base64.StdEncoding.DecodeString(samlReq)
			if err != nil {
				return err
			}
			// Remove deflate
			req := flate.NewReader(bytes.NewReader(reqBytes))
			// Read the XML
			decoder := xml.NewDecoder(req)
			loginReq := &saml.AuthnRequest{}
			if err = decoder.Decode(loginReq); err != nil {
				return err
			}

			if err = i.validateRequest(loginReq, r); err != nil {
				return err
			}

			// create saveable request
			saveableRequest, err := model.NewAuthnRequest(loginReq, relayState)
			if err != nil {
				return err
			}

			// check for cookie to see if user has a current session
			if cookie, err := r.Cookie(i.cookieName); err == nil {
				// Found a session cookie
				if data, err := i.UserCache.Get(cookie.Value); err == nil {
					// Cookie matched user in cache
					user := &model.User{}
					if err = proto.Unmarshal(data, user); err == nil {
						log.Infof("found existing session for %s", user.Name)
						return i.respond(saveableRequest, user, w, r)
					}
				}
			}

			// check to see if they presented a client cert
			if clientCert, err := getCertFromRequest(r); err == nil {
				user := &model.User{
					Name:    getSubjectDN(clientCert.Subject),
					Format:  "urn:oasis:names:tc:SAML:1.1:nameid-format:X509SubjectName",
					Context: "urn:oasis:names:tc:SAML:2.0:ac:classes:X509",
					IP:      getIP(r).String()}

				// Add attributes
				err = i.setUserAttributes(user)
				if err != nil {
					return err
				}
				log.Infof("successful PKI login for %s", user.Name)
				return i.respond(saveableRequest, user, w, r)
			}
			// need to display the login form
			data, err := proto.Marshal(saveableRequest)
			if err != nil {
				return err
			}
			id := uuid.New().String()
			err = i.TempCache.Set(id, data)
			if err != nil {
				return err
			}
			http.Redirect(w, r, fmt.Sprintf("/ui/login.html?requestId=%s",
				url.QueryEscape(id)), http.StatusTemporaryRedirect)
			return nil
		}()
		if err != nil {
			log.Error(err)
			http.Error(w, err.Error(), http.StatusBadRequest)
		}
	}
}

type dsaSignature struct {
	R, S *big.Int
}
<|endoftext|>


<commit_before>package attachments

import (
	"bytes"
	"errors"
	"image"
	"image/color"
	"image/color/palette"
	"image/draw"
	"image/gif"
	"image/jpeg"
	"image/png"
	"io"
	"io/ioutil"
	"strings"

	"github.com/keybase/client/go/chat/types"
	"github.com/keybase/client/go/chat/utils"

	_ "github.com/keybase/golang-ico" // for image decoding
	"github.com/nfnt/resize"
	"golang.org/x/image/bmp"
	_ "golang.org/x/image/bmp" // for image decoding
	"golang.org/x/image/tiff"
	"golang.org/x/net/context"

	"camlistore.org/pkg/images"
)

const (
	previewImageWidth  = 640
	previewImageHeight = 640
)

type PreviewRes struct {
	Source            []byte
	ContentType       string
	BaseWidth         int
	BaseHeight        int
	BaseDurationMs    int
	PreviewWidth      int
	PreviewHeight     int
	PreviewDurationMs int
}

func IsFatalImageErr(err error) bool {
	switch err {
	case image.ErrFormat,
		bmp.ErrUnsupported:
		return true
	}
	switch err.(type) {
	case png.FormatError,
		png.UnsupportedError,
		tiff.FormatError,
		tiff.UnsupportedError,
		jpeg.FormatError,
		jpeg.UnsupportedError:
		return true
	}
	return false
}

// Preview creates preview assets from src.  It returns an in-memory BufferSource
// and the content type of the preview asset.
func Preview(ctx context.Context, log utils.DebugLabeler, src ReadResetter, contentType,
	basename string, nvh types.NativeVideoHelper) (res *PreviewRes, err error) {
	defer func() {
		if IsFatalImageErr(err) {
			log.Debug(ctx, "squashing %v", err)
			err = nil
			res = nil
		}
	}()
	switch contentType {
	case "image/jpeg", "image/png", "image/vnd.microsoft.icon", "image/x-icon":
		return previewImage(ctx, log, src, basename, contentType)
	case "image/gif":
		return previewGIF(ctx, log, src, basename)
	}
	if strings.HasPrefix(contentType, "video") {
		pre, err := previewVideo(ctx, log, src, basename, nvh)
		if err == nil {
			log.Debug(ctx, "Preview: found video preview for filename: %s contentType: %s", basename,
				contentType)
			return pre, nil
		}
		log.Debug(ctx, "Preview: failed to get video preview for filename: %s contentType: %s err: %s",
			basename, contentType, err)
		return previewVideoBlank(ctx, log, src, basename)
	}
	return nil, nil
}

// previewVideoBlank previews a video by inserting a black rectangle with a play button on it.
func previewVideoBlank(ctx context.Context, log utils.DebugLabeler, src io.Reader,
	basename string) (res *PreviewRes, err error) {
	const width, height = 300, 150
	img := image.NewNRGBA(image.Rect(0, 0, width, height))
	for y := 0; y < height; y++ {
		for x := 0; x < width; x++ {
			img.Set(x, y, color.NRGBA{
				R: 0,
				G: 0,
				B: 0,
				A: 255,
			})
		}
	}
	var out bytes.Buffer
	if err := png.Encode(&out, img); err != nil {
		return res, err
	}
	imagePreview, err := previewImage(ctx, log, &out, basename, "image/png")
	if err != nil {
		return res, err
	}
	return &PreviewRes{
		Source:         imagePreview.Source,
		ContentType:    "image/png",
		BaseWidth:      imagePreview.BaseWidth,
		BaseHeight:     imagePreview.BaseHeight,
		BaseDurationMs: 1,
		PreviewHeight:  imagePreview.PreviewHeight,
		PreviewWidth:   imagePreview.PreviewWidth,
	}, nil
}

// previewImage will resize a single-frame image.
func previewImage(ctx context.Context, log utils.DebugLabeler, src io.Reader, basename, contentType string) (res *PreviewRes, err error) {
	defer log.Trace(ctx, &err, "previewImage")()
	// images.Decode in camlistore correctly handles exif orientation information.
	log.Debug(ctx, "previewImage: decoding image")
	img, _, err := images.Decode(src, nil)
	if err != nil {
		return nil, err
	}

	width, height := previewDimensions(img.Bounds())

	log.Debug(ctx, "previewImage: resizing image: bounds: %s", img.Bounds())
	preview := resize.Resize(width, height, img, resize.Bicubic)
	var buf bytes.Buffer

	var encodeContentType string
	switch contentType {
	case "image/vnd.microsoft.icon", "image/x-icon", "image/png":
		encodeContentType = "image/png"
		if err := png.Encode(&buf, preview); err != nil {
			return nil, err
		}
	default:
		encodeContentType = "image/jpeg"
		if err := jpeg.Encode(&buf, preview, &jpeg.Options{Quality: 90}); err != nil {
			return nil, err
		}
	}

	return &PreviewRes{
		Source:        buf.Bytes(),
		ContentType:   encodeContentType,
		BaseWidth:     img.Bounds().Dx(),
		BaseHeight:    img.Bounds().Dy(),
		PreviewWidth:  int(width),
		PreviewHeight: int(height),
	}, nil
}

// previewGIF handles resizing multiple frames in an animated gif.
// Based on code in https://github.com/dpup/go-scratch/blob/master/gif-resize/gif-resize.go
func previewGIF(ctx context.Context, log utils.DebugLabeler, src io.Reader, basename string) (*PreviewRes, error) {
	raw, err := ioutil.ReadAll(src)
	if err != nil {
		return nil, err
	}
	g, err := gif.DecodeAll(bytes.NewReader(raw))
	if err != nil {
		return nil, err
	}

	frames := len(g.Image)
	if frames == 0 {
		return nil, errors.New("no image frames in GIF")
	}

	log.Debug(ctx, "previewGIF: number of frames = %d", frames)

	var baseDuration int
	if frames > 1 {
		if len(raw) < 10*1024*1024 {
			log.Debug(ctx, "previewGif: not resizing because multiple-frame original < 10MB")

			// don't resize if multiple frames and < 5MB
			bounds := g.Image[0].Bounds()
			duration := gifDuration(g)
			res := &PreviewRes{
				Source:            raw,
				ContentType:       "image/gif",
				BaseWidth:         bounds.Dx(),
				BaseHeight:        bounds.Dy(),
				PreviewWidth:      bounds.Dx(),
				PreviewHeight:     bounds.Dy(),
				BaseDurationMs:    duration,
				PreviewDurationMs: duration,
			}
			return res, nil
		}

		log.Debug(ctx, "previewGif: large multiple-frame gif: %d, just using frame 0", len(raw))
		baseDuration = gifDuration(g)
		g.Image = g.Image[:1]
		g.Delay = g.Delay[:1]
		g.Disposal = g.Disposal[:1]
	}

	// create a new image based on the first frame to draw
	// the incremental frames
	origBounds := g.Image[0].Bounds()
	img := image.NewRGBA(origBounds)

	// draw each frame, then resize it, replacing the existing frames.
	width, height := previewDimensions(origBounds)
	log.Debug(ctx, "previewGif: resizing to %d x %d", width, height)
	for index, frame := range g.Image {
		bounds := frame.Bounds()
		draw.Draw(img, bounds, frame, bounds.Min, draw.Over)
		g.Image[index] = imageToPaletted(resize.Resize(width, height, img, resize.Bicubic))
		log.Debug(ctx, "previewGIF: resized frame %d", index)
	}

	// change the image Config to the new size
	g.Config.Width = int(width)
	g.Config.Height = int(height)

	// encode all the frames into buf
	var buf bytes.Buffer
	if err := gif.EncodeAll(&buf, g); err != nil {
		return nil, err
	}

	res := &PreviewRes{
		Source:         buf.Bytes(),
		ContentType:    "image/gif",
		BaseWidth:      origBounds.Dx(),
		BaseHeight:     origBounds.Dy(),
		PreviewWidth:   int(width),
		PreviewHeight:  int(height),
		BaseDurationMs: baseDuration,
	}

	if len(g.Image) > 1 {
		res.PreviewDurationMs = gifDuration(g)
	}

	return res, nil
}

func previewDimensions(origBounds image.Rectangle) (uint, uint) {
	origWidth := uint(origBounds.Dx())
	origHeight := uint(origBounds.Dy())

	if previewImageWidth >= origWidth && previewImageHeight >= origHeight {
		return origWidth, origHeight
	}

	newWidth, newHeight := origWidth, origHeight
	// Preserve aspect ratio
	if origWidth > previewImageWidth {
		newHeight = origHeight * previewImageWidth / origWidth
		if newHeight < 1 {
			newHeight = 1
		}
		newWidth = previewImageWidth
	}

	if newHeight > previewImageHeight {
		newWidth = newWidth * previewImageHeight / newHeight
		if newWidth < 1 {
			newWidth = 1
		}
		newHeight = previewImageHeight
	}

	return newWidth, newHeight
}

// imageToPaletted converts image.Image to *image.Paletted.
// From https://github.com/dpup/go-scratch/blob/master/gif-resize/gif-resize.go
func imageToPaletted(img image.Image) *image.Paletted {
	b := img.Bounds()
	pm := image.NewPaletted(b, palette.Plan9)
	draw.FloydSteinberg.Draw(pm, b, img, image.Point{})
	return pm
}

// gifDuration returns the duration of one loop of an animated gif
// in milliseconds.
func gifDuration(g *gif.GIF) int {
	var total int
	for _, d := range g.Delay {
		total += d
	}

	// total is in 100ths of a second, multiply by 10 to get milliseconds
	return total * 10
}
<commit_msg>recover during image preview (#24002)<commit_after>package attachments

import (
	"bytes"
	"errors"
	"fmt"
	"image"
	"image/color"
	"image/color/palette"
	"image/draw"
	"image/gif"
	"image/jpeg"
	"image/png"
	"io"
	"io/ioutil"
	"strings"

	"github.com/keybase/client/go/chat/types"
	"github.com/keybase/client/go/chat/utils"

	_ "github.com/keybase/golang-ico" // for image decoding
	"github.com/nfnt/resize"
	"golang.org/x/image/bmp"
	_ "golang.org/x/image/bmp" // for image decoding
	"golang.org/x/image/tiff"
	"golang.org/x/net/context"

	"camlistore.org/pkg/images"
)

const (
	previewImageWidth  = 640
	previewImageHeight = 640
)

type PreviewRes struct {
	Source            []byte
	ContentType       string
	BaseWidth         int
	BaseHeight        int
	BaseDurationMs    int
	PreviewWidth      int
	PreviewHeight     int
	PreviewDurationMs int
}

func IsFatalImageErr(err error) bool {
	switch err {
	case image.ErrFormat,
		bmp.ErrUnsupported:
		return true
	}
	switch err.(type) {
	case png.FormatError,
		png.UnsupportedError,
		tiff.FormatError,
		tiff.UnsupportedError,
		jpeg.FormatError,
		jpeg.UnsupportedError:
		return true
	}
	return false
}

// Preview creates preview assets from src.  It returns an in-memory BufferSource
// and the content type of the preview asset.
func Preview(ctx context.Context, log utils.DebugLabeler, src ReadResetter, contentType,
	basename string, nvh types.NativeVideoHelper) (res *PreviewRes, err error) {
	defer func() {
		if IsFatalImageErr(err) {
			log.Debug(ctx, "squashing %v", err)
			err = nil
			res = nil
		}
	}()
	switch contentType {
	case "image/jpeg", "image/png", "image/vnd.microsoft.icon", "image/x-icon":
		return previewImage(ctx, log, src, basename, contentType)
	case "image/gif":
		return previewGIF(ctx, log, src, basename)
	}
	if strings.HasPrefix(contentType, "video") {
		pre, err := previewVideo(ctx, log, src, basename, nvh)
		if err == nil {
			log.Debug(ctx, "Preview: found video preview for filename: %s contentType: %s", basename,
				contentType)
			return pre, nil
		}
		log.Debug(ctx, "Preview: failed to get video preview for filename: %s contentType: %s err: %s",
			basename, contentType, err)
		return previewVideoBlank(ctx, log, src, basename)
	}
	return nil, nil
}

// previewVideoBlank previews a video by inserting a black rectangle with a play button on it.
func previewVideoBlank(ctx context.Context, log utils.DebugLabeler, src io.Reader,
	basename string) (res *PreviewRes, err error) {
	const width, height = 300, 150
	img := image.NewNRGBA(image.Rect(0, 0, width, height))
	for y := 0; y < height; y++ {
		for x := 0; x < width; x++ {
			img.Set(x, y, color.NRGBA{
				R: 0,
				G: 0,
				B: 0,
				A: 255,
			})
		}
	}
	var out bytes.Buffer
	if err := png.Encode(&out, img); err != nil {
		return res, err
	}
	imagePreview, err := previewImage(ctx, log, &out, basename, "image/png")
	if err != nil {
		return res, err
	}
	return &PreviewRes{
		Source:         imagePreview.Source,
		ContentType:    "image/png",
		BaseWidth:      imagePreview.BaseWidth,
		BaseHeight:     imagePreview.BaseHeight,
		BaseDurationMs: 1,
		PreviewHeight:  imagePreview.PreviewHeight,
		PreviewWidth:   imagePreview.PreviewWidth,
	}, nil
}

// previewImage will resize a single-frame image.
func previewImage(ctx context.Context, log utils.DebugLabeler, src io.Reader, basename, contentType string) (res *PreviewRes, err error) {
	defer func() {
		// decoding ico images can cause a panic, let's catch anything here.
		// https://github.com/biessek/golang-ico/issues/4
		if r := recover(); r != nil {
			log.Debug(ctx, "Recovered %v", r)
			res = nil
			err = fmt.Errorf("unable to preview image: %v", r)
		}
	}()
	defer log.Trace(ctx, &err, "previewImage")()
	// images.Decode in camlistore correctly handles exif orientation information.
	log.Debug(ctx, "previewImage: decoding image")
	img, _, err := images.Decode(src, nil)
	if err != nil {
		return nil, err
	}

	width, height := previewDimensions(img.Bounds())

	log.Debug(ctx, "previewImage: resizing image: bounds: %s", img.Bounds())
	preview := resize.Resize(width, height, img, resize.Bicubic)
	var buf bytes.Buffer

	var encodeContentType string
	switch contentType {
	case "image/vnd.microsoft.icon", "image/x-icon", "image/png":
		encodeContentType = "image/png"
		if err := png.Encode(&buf, preview); err != nil {
			return nil, err
		}
	default:
		encodeContentType = "image/jpeg"
		if err := jpeg.Encode(&buf, preview, &jpeg.Options{Quality: 90}); err != nil {
			return nil, err
		}
	}

	return &PreviewRes{
		Source:        buf.Bytes(),
		ContentType:   encodeContentType,
		BaseWidth:     img.Bounds().Dx(),
		BaseHeight:    img.Bounds().Dy(),
		PreviewWidth:  int(width),
		PreviewHeight: int(height),
	}, nil
}

// previewGIF handles resizing multiple frames in an animated gif.
// Based on code in https://github.com/dpup/go-scratch/blob/master/gif-resize/gif-resize.go
func previewGIF(ctx context.Context, log utils.DebugLabeler, src io.Reader, basename string) (*PreviewRes, error) {
	raw, err := ioutil.ReadAll(src)
	if err != nil {
		return nil, err
	}
	g, err := gif.DecodeAll(bytes.NewReader(raw))
	if err != nil {
		return nil, err
	}

	frames := len(g.Image)
	if frames == 0 {
		return nil, errors.New("no image frames in GIF")
	}

	log.Debug(ctx, "previewGIF: number of frames = %d", frames)

	var baseDuration int
	if frames > 1 {
		if len(raw) < 10*1024*1024 {
			log.Debug(ctx, "previewGif: not resizing because multiple-frame original < 10MB")

			// don't resize if multiple frames and < 5MB
			bounds := g.Image[0].Bounds()
			duration := gifDuration(g)
			res := &PreviewRes{
				Source:            raw,
				ContentType:       "image/gif",
				BaseWidth:         bounds.Dx(),
				BaseHeight:        bounds.Dy(),
				PreviewWidth:      bounds.Dx(),
				PreviewHeight:     bounds.Dy(),
				BaseDurationMs:    duration,
				PreviewDurationMs: duration,
			}
			return res, nil
		}

		log.Debug(ctx, "previewGif: large multiple-frame gif: %d, just using frame 0", len(raw))
		baseDuration = gifDuration(g)
		g.Image = g.Image[:1]
		g.Delay = g.Delay[:1]
		g.Disposal = g.Disposal[:1]
	}

	// create a new image based on the first frame to draw
	// the incremental frames
	origBounds := g.Image[0].Bounds()
	img := image.NewRGBA(origBounds)

	// draw each frame, then resize it, replacing the existing frames.
	width, height := previewDimensions(origBounds)
	log.Debug(ctx, "previewGif: resizing to %d x %d", width, height)
	for index, frame := range g.Image {
		bounds := frame.Bounds()
		draw.Draw(img, bounds, frame, bounds.Min, draw.Over)
		g.Image[index] = imageToPaletted(resize.Resize(width, height, img, resize.Bicubic))
		log.Debug(ctx, "previewGIF: resized frame %d", index)
	}

	// change the image Config to the new size
	g.Config.Width = int(width)
	g.Config.Height = int(height)

	// encode all the frames into buf
	var buf bytes.Buffer
	if err := gif.EncodeAll(&buf, g); err != nil {
		return nil, err
	}

	res := &PreviewRes{
		Source:         buf.Bytes(),
		ContentType:    "image/gif",
		BaseWidth:      origBounds.Dx(),
		BaseHeight:     origBounds.Dy(),
		PreviewWidth:   int(width),
		PreviewHeight:  int(height),
		BaseDurationMs: baseDuration,
	}

	if len(g.Image) > 1 {
		res.PreviewDurationMs = gifDuration(g)
	}

	return res, nil
}

func previewDimensions(origBounds image.Rectangle) (uint, uint) {
	origWidth := uint(origBounds.Dx())
	origHeight := uint(origBounds.Dy())

	if previewImageWidth >= origWidth && previewImageHeight >= origHeight {
		return origWidth, origHeight
	}

	newWidth, newHeight := origWidth, origHeight
	// Preserve aspect ratio
	if origWidth > previewImageWidth {
		newHeight = origHeight * previewImageWidth / origWidth
		if newHeight < 1 {
			newHeight = 1
		}
		newWidth = previewImageWidth
	}

	if newHeight > previewImageHeight {
		newWidth = newWidth * previewImageHeight / newHeight
		if newWidth < 1 {
			newWidth = 1
		}
		newHeight = previewImageHeight
	}

	return newWidth, newHeight
}

// imageToPaletted converts image.Image to *image.Paletted.
// From https://github.com/dpup/go-scratch/blob/master/gif-resize/gif-resize.go
func imageToPaletted(img image.Image) *image.Paletted {
	b := img.Bounds()
	pm := image.NewPaletted(b, palette.Plan9)
	draw.FloydSteinberg.Draw(pm, b, img, image.Point{})
	return pm
}

// gifDuration returns the duration of one loop of an animated gif
// in milliseconds.
func gifDuration(g *gif.GIF) int {
	var total int
	for _, d := range g.Delay {
		total += d
	}

	// total is in 100ths of a second, multiply by 10 to get milliseconds
	return total * 10
}
<|endoftext|>