<commit_before>package main

import (
	"bufio"
	"os"
	"net/http"
	"time"
	"fmt"
	"errors"
	"encoding/json"
)

type BreakerCount struct {
	OpenCount   int64
	ClosedCount int64
}

type LatencyHistogram struct {
	mean          int64
	median        int64
	min           int64
	max           int64
	percentile25  int64
	percentile75  int64
	percentile90  int64
	percentile95  int64
	percentile99  int64
	percentile995 int64
	percentile999 int64
}

type CircuitBreaker struct {
	Name              string
	SuccessCount      int64
	FailCount         int64
	ShortCircuitCount int64
	WindowDuration    time.Duration
	CurrentTime       time.Time
	BreakerStatus     BreakerCount
	Latency           LatencyHistogram
}

type SSEString string

// A snapshot transcription of the hystrix.stream JSON object
// This is here for legacy support only. Only update if the fields change or
// In the event of an inevitable bug.
type HystrixStream struct {
	// Forgive my ridiculous formatting in this ridiculous object
	CurrentConcurrentExecutionCount int
	CurrentTime          string            `json:"currentTime,string"`
	ErrorPercentage      int               `json:"errorPercentage,int"`
	ErrorCount           int               `json:"errorCount,int"`
	Group                string            `json:"group,string"`
	IsCircuitBreakerOpen bool              `json:"isCircuitBreakerOpen,bool"`
	LatencyExecute       HystrixHistogram  `json:"latencyExecute,HystrixHistogram"`
	LatencyExecuteMean   int               `json:"latencyExecute_mean,int"`
	LatencyTotal         HystrixHistogram  `json:"latencyTotal,HystrixHistogram"`
	LatencyTotalMean     int               `json:"latencyTotal_mean,int"`
	Name                 string            `json:"name,string"`
	ReportingHosts       int               `json:"reportingHosts,int"`
	RequestCount         int               `json:"requestCount,int"`
	RollingCountCollapsedRequests   int    `json:"rollingCountCollapsedRequests,int"`
	RollingCountExceptionsThrown    int    `json:"rollingCountExceptionsThrown,int"`
	RollingCountFailure             int    `json:"rollingCountFailure,int"`
	RollingCountFallbackFailure     int    `json:"rollingCountFallbackFailure,int"`
	RollingCountFallbackRejection   int    `json:"rollingCountFallbackRejection,int"`
	RollingCountResponseFromCache   int    `json:"rollingCountResponseFromCache,int"`
	RollingCountSemaphoreRejected   int    `json:"rollingCountSemaphoreRejected,int"`
	RollingCountShortCircuited      int    `json:"rollingCountShortCircuited,int"`
	RollingCountSuccess             int    `json:"rollingCountSuccess,int"`
	RollingCountThreadPoolRejected  int    `json:"rollingCountThreadPoolRejected,int"`
	RollingCountTimeout             int    `json:"rollingCOuntTimeout,int"`
	Type                            string `json:"type,string"`
	// Don't blame me for these awful names.
	// I'm preserving the bad names Hystrix uses
	PropertyValueCircuitBreakerEnabled                            bool   `json:"propertyValue_circuitBreakerEnabled,bool"`
	PropertyValueCircuitBreakerErrorThresholdPercentage           int    `json:"propertyValue_circuitBreakerErrorThresholdPercentage,int"`
	PropertyValueCircuitBreakerForceOpen                          bool   `json:"propertyValue_circuitBreakerForceOpen,bool"`
	PropertyValueCircuitBreakerForceClosed                        bool   `json:"propertyValue_circuitBreakerForceClosed,bool"`
	PropertyValueCircuitBreakerRequestVolumeThreshold             int    `json:"propertyValue_circuitBreakerRequestVolumeThreshold,int"`
	PropertyValueCircuitBreakerSleepWindowInMilliseconds          int    `json:"propertyValue_circuitBreakerSleepWindowInMilliseconds,int"`
	PropertyValueExecutionIsolationSemaphoreMaxConcurrentRequests int    `json:"propertyValue_executionIsolationSemaphoreMaxConcurrentRequests,int"`
	PropertyValueExecutionIsolationStrategy                       string `json:"propertyValue_executionIsolationStrategy,string"`
	PropertyValueExecutionIsolationThreadPoolKeyOverride          string `json:"propertyValue_executionIsolationThreadPoolKeyOverride,string"`
	PropertyValueExecutionIsolationThreadTimeoutInMilliseconds    int    `json:"propertyValue_executionIsolationThreadTimeoutInMilliseconds,string"`
	PropertyValueFallbackIsolationSemaphoreMaxConcurrentRequests  int    `json:"propertyValue_fallbackIsolationSeampahoreMaxConcurrentRequests,int"`
	PropertyValueMetricsRollingStatisticalWindowInMilliseconds    int    `json:"propertyValue_metricsRollingStatisticalWindowInMilliseconds,int"`
	PropertyValueRequestCacheEnabled                              bool   `json:"propertyValue_requestCacheEnabled,bool"`
	PropertyValueRequestLogEnabled                                bool   `json:"propertyValue_requestLogEnabled,bool"`
}

// A snapshot transcription of the histogram objects hystrix.stream JSON object
// This is here for legacy support only. Only update if the fields change or
// In the event of an inevitable bug.
type HystrixHistogram struct {
	//minimum
	percentile0   int `json:"0,string"`
	percentile25  int `json:"25,string"`
	//median
	percentile50  int `json:"50,string"`
	percentile75  int `json:"75,string"`
	percentile90  int `json:"90,string"`
	percentile95  int `json:"95,string"`
	percentile995 int `json:"99.5,string"`
	//maximum
	percentile100 int `json:"100,string"`
}

func (s SSEString) ParseHystrixStream() (HystrixStream, error) {
	// The eventsource string isn't big short circuit
	if len(s) < 8 {
		return HystrixStream{}, errors.New("Event string too short to parse")
	}

	// The eventsource string isn't data
	if s[:6] != "data: " {
		return HystrixStream{}, errors.New("Can't parse non-data event")
	}

	// Try to parse JSON
	var ret HystrixStream
	resp := json.Unmarshal([]byte(s[7:]), &ret)

	if resp == nil {
		return ret, nil
	} else {
		return HystrixStream{}, resp
	}
}

func (h HystrixStream) ToCircuitBreaker() (CircuitBreaker, error) {
	return CircuitBreaker{}, nil
}

func (c CircuitBreaker) ToJSON() string {
	return fmt.Sprintf(
		"{" +
			"\"name\":%v," +
			"\"successCount\":%v," +
			"\"failCount\":%v," +
			"\"shortCircuitCount\":%v," +
			"\"windowDuration\":%v," +
			"\"currentTime\":%v," +
			"\"breakerStatus\":%v," +
			"\"latency\":%v" +
		"}",
		c.Name,
		c.SuccessCount,
		c.FailCount,
		c.ShortCircuitCount,
		(c.WindowDuration.Nanoseconds() / 1000),
		c.CurrentTime.Format(time.RFC3339),
		c.BreakerStatus.toJSON(),
		c.Latency.toJSON())
}

func (l LatencyHistogram) toJSON() string {
	return fmt.Sprintf(
		"{" +
			"\"mean\":%v," +
			"\"median\":%v," +
			"\"min\":%v," +
			"\"max\":%v,"+
			"\"25\":%v," +
			"\"75\":%v," +
			"\"90\":%v," +
			"\"95\":%v," +
			"\"99\":%v," +
			"\"99.5\"%v," +
			"\"99.9\"%v" +
		"}",
		l.mean,
		l.median,
		l.min,
		l.max,
		l.percentile25,
		l.percentile75,
		l.percentile90,
		l.percentile95,
		l.percentile99,
		l.percentile995,
		l.percentile999)
}

func (b BreakerCount) toJSON() string {
	return fmt.Sprintf(
		"{" +
			"\"open\":%v," +
			"\"closed\":%v" +
		"}",
		b.OpenCount,
		b.ClosedCount)
}

func main() {
	req, err := http.NewRequest("GET", "/hystrix.stream", nil)
	if err != nil {
		os.Exit(1)
	}
	req.Header.Set("Cache-Control", "no-cache")
	req.Header.Set("Accept", "text/event-stream")
	client := http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		os.Exit(1)
	}
	stdout :=  bufio.NewWriter(os.Stdout)
	stdout.ReadFrom(resp.Body)
	defer stdout.Flush()
}

<commit_msg>Add missing json tag to hystrix object<commit_after>package main

import (
	"bufio"
	"os"
	"net/http"
	"time"
	"fmt"
	"errors"
	"encoding/json"
)

type BreakerCount struct {
	OpenCount   int64
	ClosedCount int64
}

type LatencyHistogram struct {
	mean          int64
	median        int64
	min           int64
	max           int64
	percentile25  int64
	percentile75  int64
	percentile90  int64
	percentile95  int64
	percentile99  int64
	percentile995 int64
	percentile999 int64
}

type CircuitBreaker struct {
	Name              string
	SuccessCount      int64
	FailCount         int64
	ShortCircuitCount int64
	WindowDuration    time.Duration
	CurrentTime       time.Time
	BreakerStatus     BreakerCount
	Latency           LatencyHistogram
}

type SSEString string

// A snapshot transcription of the hystrix.stream JSON object
// This is here for legacy support only. Only update if the fields change or
// In the event of an inevitable bug.
type HystrixStream struct {
	// Forgive my ridiculous formatting in this ridiculous object
	CurrentConcurrentExecutionCount int    `json:"currentConcurrentExecutionCount,int"`
	CurrentTime          string            `json:"currentTime,string"`
	ErrorPercentage      int               `json:"errorPercentage,int"`
	ErrorCount           int               `json:"errorCount,int"`
	Group                string            `json:"group,string"`
	IsCircuitBreakerOpen bool              `json:"isCircuitBreakerOpen,bool"`
	LatencyExecute       HystrixHistogram  `json:"latencyExecute,HystrixHistogram"`
	LatencyExecuteMean   int               `json:"latencyExecute_mean,int"`
	LatencyTotal         HystrixHistogram  `json:"latencyTotal,HystrixHistogram"`
	LatencyTotalMean     int               `json:"latencyTotal_mean,int"`
	Name                 string            `json:"name,string"`
	ReportingHosts       int               `json:"reportingHosts,int"`
	RequestCount         int               `json:"requestCount,int"`
	RollingCountCollapsedRequests   int    `json:"rollingCountCollapsedRequests,int"`
	RollingCountExceptionsThrown    int    `json:"rollingCountExceptionsThrown,int"`
	RollingCountFailure             int    `json:"rollingCountFailure,int"`
	RollingCountFallbackFailure     int    `json:"rollingCountFallbackFailure,int"`
	RollingCountFallbackRejection   int    `json:"rollingCountFallbackRejection,int"`
	RollingCountResponseFromCache   int    `json:"rollingCountResponseFromCache,int"`
	RollingCountSemaphoreRejected   int    `json:"rollingCountSemaphoreRejected,int"`
	RollingCountShortCircuited      int    `json:"rollingCountShortCircuited,int"`
	RollingCountSuccess             int    `json:"rollingCountSuccess,int"`
	RollingCountThreadPoolRejected  int    `json:"rollingCountThreadPoolRejected,int"`
	RollingCountTimeout             int    `json:"rollingCOuntTimeout,int"`
	Type                            string `json:"type,string"`
	// Don't blame me for these awful names.
	// I'm preserving the bad names Hystrix uses
	PropertyValueCircuitBreakerEnabled                            bool   `json:"propertyValue_circuitBreakerEnabled,bool"`
	PropertyValueCircuitBreakerErrorThresholdPercentage           int    `json:"propertyValue_circuitBreakerErrorThresholdPercentage,int"`
	PropertyValueCircuitBreakerForceOpen                          bool   `json:"propertyValue_circuitBreakerForceOpen,bool"`
	PropertyValueCircuitBreakerForceClosed                        bool   `json:"propertyValue_circuitBreakerForceClosed,bool"`
	PropertyValueCircuitBreakerRequestVolumeThreshold             int    `json:"propertyValue_circuitBreakerRequestVolumeThreshold,int"`
	PropertyValueCircuitBreakerSleepWindowInMilliseconds          int    `json:"propertyValue_circuitBreakerSleepWindowInMilliseconds,int"`
	PropertyValueExecutionIsolationSemaphoreMaxConcurrentRequests int    `json:"propertyValue_executionIsolationSemaphoreMaxConcurrentRequests,int"`
	PropertyValueExecutionIsolationStrategy                       string `json:"propertyValue_executionIsolationStrategy,string"`
	PropertyValueExecutionIsolationThreadPoolKeyOverride          string `json:"propertyValue_executionIsolationThreadPoolKeyOverride,string"`
	PropertyValueExecutionIsolationThreadTimeoutInMilliseconds    int    `json:"propertyValue_executionIsolationThreadTimeoutInMilliseconds,string"`
	PropertyValueFallbackIsolationSemaphoreMaxConcurrentRequests  int    `json:"propertyValue_fallbackIsolationSeampahoreMaxConcurrentRequests,int"`
	PropertyValueMetricsRollingStatisticalWindowInMilliseconds    int    `json:"propertyValue_metricsRollingStatisticalWindowInMilliseconds,int"`
	PropertyValueRequestCacheEnabled                              bool   `json:"propertyValue_requestCacheEnabled,bool"`
	PropertyValueRequestLogEnabled                                bool   `json:"propertyValue_requestLogEnabled,bool"`
}

// A snapshot transcription of the histogram objects hystrix.stream JSON object
// This is here for legacy support only. Only update if the fields change or
// In the event of an inevitable bug.
type HystrixHistogram struct {
	//minimum
	percentile0   int `json:"0,string"`
	percentile25  int `json:"25,string"`
	//median
	percentile50  int `json:"50,string"`
	percentile75  int `json:"75,string"`
	percentile90  int `json:"90,string"`
	percentile95  int `json:"95,string"`
	percentile995 int `json:"99.5,string"`
	//maximum
	percentile100 int `json:"100,string"`
}

func (s SSEString) ParseHystrixStream() (HystrixStream, error) {
	// The eventsource string isn't big short circuit
	if len(s) < 8 {
		return HystrixStream{}, errors.New("Event string too short to parse")
	}

	// The eventsource string isn't data
	if s[:6] != "data: " {
		return HystrixStream{}, errors.New("Can't parse non-data event")
	}

	// Try to parse JSON
	var ret HystrixStream
	resp := json.Unmarshal([]byte(s[7:]), &ret)

	if resp == nil {
		return ret, nil
	} else {
		return HystrixStream{}, resp
	}
}

func (h HystrixStream) ToCircuitBreaker() (CircuitBreaker, error) {
	return CircuitBreaker{}, nil
}

func (c CircuitBreaker) ToJSON() string {
	return fmt.Sprintf(
		"{" +
			"\"name\":%v," +
			"\"successCount\":%v," +
			"\"failCount\":%v," +
			"\"shortCircuitCount\":%v," +
			"\"windowDuration\":%v," +
			"\"currentTime\":%v," +
			"\"breakerStatus\":%v," +
			"\"latency\":%v" +
		"}",
		c.Name,
		c.SuccessCount,
		c.FailCount,
		c.ShortCircuitCount,
		(c.WindowDuration.Nanoseconds() / 1000),
		c.CurrentTime.Format(time.RFC3339),
		c.BreakerStatus.toJSON(),
		c.Latency.toJSON())
}

func (l LatencyHistogram) toJSON() string {
	return fmt.Sprintf(
		"{" +
			"\"mean\":%v," +
			"\"median\":%v," +
			"\"min\":%v," +
			"\"max\":%v,"+
			"\"25\":%v," +
			"\"75\":%v," +
			"\"90\":%v," +
			"\"95\":%v," +
			"\"99\":%v," +
			"\"99.5\"%v," +
			"\"99.9\"%v" +
		"}",
		l.mean,
		l.median,
		l.min,
		l.max,
		l.percentile25,
		l.percentile75,
		l.percentile90,
		l.percentile95,
		l.percentile99,
		l.percentile995,
		l.percentile999)
}

func (b BreakerCount) toJSON() string {
	return fmt.Sprintf(
		"{" +
			"\"open\":%v," +
			"\"closed\":%v" +
		"}",
		b.OpenCount,
		b.ClosedCount)
}

func main() {
	req, err := http.NewRequest("GET", "/hystrix.stream", nil)
	if err != nil {
		os.Exit(1)
	}
	req.Header.Set("Cache-Control", "no-cache")
	req.Header.Set("Accept", "text/event-stream")
	client := http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		os.Exit(1)
	}
	stdout :=  bufio.NewWriter(os.Stdout)
	stdout.ReadFrom(resp.Body)
	defer stdout.Flush()
}

<|endoftext|>


<commit_before>package acceptance

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	"gopkg.in/yaml.v2"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/pivotal-cf-experimental/pivnet-resource/concourse"
	"github.com/pivotal-cf-experimental/pivnet-resource/metadata"
	"github.com/pivotal-cf-experimental/pivnet-resource/pivnet"
	"github.com/pivotal-cf-experimental/pivnet-resource/versions"
)

var _ = Describe("Lifecycle test", func() {
	var (
		releaseType     = "Minor Release"
		releaseDate     = "2015-12-17"
		eulaSlug        = "pivotal_beta_eula"
		description     = "this release is for automated-testing only."
		releaseNotesURL = "https://example.com"

		metadataFile   = "metadata"
		productVersion string

		filePrefix = "pivnet-resource-test-file"

		command       *exec.Cmd
		stdinContents []byte
		outRequest    concourse.OutRequest
		rootDir       string
	)

	BeforeEach(func() {
		var err error

		By("Creating a temporary root dir")
		rootDir, err = ioutil.TempDir("", "")
		Expect(err).ShouldNot(HaveOccurred())

		By("Generating 'random' product version")
		productVersion = fmt.Sprintf("%d", time.Now().Nanosecond())

		By("Creating a metadata struct")
		productMetadata := metadata.Metadata{
			Release: &metadata.Release{
				ReleaseType:     releaseType,
				EULASlug:        eulaSlug,
				ReleaseDate:     releaseDate,
				Description:     description,
				ReleaseNotesURL: releaseNotesURL,
				Version:         productVersion,
			},
		}

		By("Marshaling the metadata to yaml")
		metadataBytes, err := yaml.Marshal(productMetadata)
		Expect(err).ShouldNot(HaveOccurred())

		By("Writing the metadata to a file")
		err = ioutil.WriteFile(
			filepath.Join(rootDir, metadataFile),
			metadataBytes,
			os.ModePerm)
		Expect(err).ShouldNot(HaveOccurred())

		By("Creating command object")
		command = exec.Command(outPath, rootDir)

		By("Creating default request")
		outRequest = concourse.OutRequest{
			Source: concourse.Source{
				APIToken:        pivnetAPIToken,
				AccessKeyID:     awsAccessKeyID,
				SecretAccessKey: awsSecretAccessKey,
				ProductSlug:     productSlug,
				Endpoint:        endpoint,
				Bucket:          pivnetBucketName,
				Region:          pivnetRegion,
			},
			Params: concourse.OutParams{
				FileGlob:       "*",
				FilepathPrefix: s3FilepathPrefix,
				MetadataFile:   metadataFile,
			},
		}

		stdinContents, err = json.Marshal(outRequest)
		Expect(err).ShouldNot(HaveOccurred())
	})

	AfterEach(func() {
		By("Removing local temp files")
		err := os.RemoveAll(rootDir)
		Expect(err).ShouldNot(HaveOccurred())
	})

	Describe("Creating a new release", func() {
		// We do not delete the release as it causes race conditions with other tests

		Context("when S3 source and params are configured correctly", func() {
			var (
				client *s3client

				sourcesDir      = "sources"
				sourceFileNames []string
				sourceFilePaths []string
				remotePaths     []string

				totalFiles = 3
			)

			BeforeEach(func() {
				By("Creating aws client")
				var err error
				client, err = NewS3Client(
					awsAccessKeyID,
					awsSecretAccessKey,
					pivnetRegion,
					pivnetBucketName,
				)
				Expect(err).ShouldNot(HaveOccurred())

				By("Creating a temporary sources dir")
				sourcesFullPath := filepath.Join(rootDir, sourcesDir)
				err = os.Mkdir(sourcesFullPath, os.ModePerm)
				Expect(err).ShouldNot(HaveOccurred())

				By("Creating local temp files")
				sourceFileNames = make([]string, totalFiles)
				sourceFilePaths = make([]string, totalFiles)
				remotePaths = make([]string, totalFiles)
				for i := 0; i < totalFiles; i++ {
					sourceFileNames[i] = fmt.Sprintf(
						"%s-%d",
						filePrefix,
						time.Now().Nanosecond(),
					)

					sourceFilePaths[i] = filepath.Join(
						sourcesFullPath,
						sourceFileNames[i],
					)

					err = ioutil.WriteFile(
						sourceFilePaths[i],
						[]byte("some content"),
						os.ModePerm,
					)
					Expect(err).ShouldNot(HaveOccurred())

					remotePaths[i] = fmt.Sprintf(
						"product_files/%s/%s",
						s3FilepathPrefix,
						sourceFileNames[i],
					)
				}

				outRequest.Params.FileGlob = fmt.Sprintf("%s/*", sourcesDir)
				outRequest.Params.FilepathPrefix = s3FilepathPrefix

				stdinContents, err = json.Marshal(outRequest)
				Expect(err).ShouldNot(HaveOccurred())
			})

			AfterEach(func() {
				By("Removing uploaded file")
				for i := 0; i < totalFiles; i++ {
					client.DeleteFile(pivnetBucketName, remotePaths[i])
				}
			})

			It("uploads files to s3 and creates files on pivnet", func() {
				By("Getting existing list of product files")
				existingProductFiles := getProductFiles(productSlug)

				By("Verifying existing product files does not yet contain new files")
				var existingProductFileNames []string
				for _, f := range existingProductFiles {
					existingProductFileNames = append(existingProductFileNames, f.Name)
				}
				for i := 0; i < totalFiles; i++ {
					Expect(existingProductFileNames).NotTo(ContainElement(sourceFileNames[i]))
				}

				By("Running the command")
				session := run(command, stdinContents)
				Eventually(session, executableTimeout).Should(gexec.Exit(0))

				By("Verifying uploaded files can be downloaded directly from S3")
				for i := 0; i < totalFiles; i++ {
					localDownloadPath := fmt.Sprintf("%s-downloaded", sourceFilePaths[i])
					err := client.DownloadFile(pivnetBucketName, remotePaths[i], localDownloadPath)
					Expect(err).ShouldNot(HaveOccurred())
				}

				By("Outputting a valid json response")
				response := concourse.OutResponse{}
				err := json.Unmarshal(session.Out.Contents(), &response)
				Expect(err).ShouldNot(HaveOccurred())

				By("Validating the release was created correctly")
				release, err := pivnetClient.GetRelease(productSlug, productVersion)
				Expect(err).NotTo(HaveOccurred())

				releaseETag, err := pivnetClient.ReleaseETag(productSlug, release)
				Expect(err).NotTo(HaveOccurred())

				expectedVersion := fmt.Sprintf("%s#%s", productVersion, releaseETag)
				Expect(response.Version.ProductVersion).To(Equal(expectedVersion))

				By("Getting updated list of product files")
				updatedProductFiles := getProductFiles(productSlug)

				By("Verifying number of product files has increased by the expected amount")
				newProductFileCount := len(updatedProductFiles) - len(existingProductFiles)
				Expect(newProductFileCount).To(Equal(totalFiles))

				By("Verifying updated product files contains new files")
				var newProductFiles []pivnet.ProductFile
				for _, p := range updatedProductFiles {
					if stringInSlice(p.Name, sourceFileNames) {
						newProductFiles = append(newProductFiles, p)
					}
				}
				Expect(len(newProductFiles)).To(Equal(totalFiles))

				By("Getting newly-created release")
				release, err = pivnetClient.GetRelease(productSlug, productVersion)
				Expect(err).ShouldNot(HaveOccurred())

				releaseETag, err = pivnetClient.ReleaseETag(productSlug, release)
				Expect(err).NotTo(HaveOccurred())

				versionWithETag, err := versions.CombineVersionAndETag(productVersion, releaseETag)
				Expect(err).NotTo(HaveOccurred())

				By("Verifying release contains new product files")
				productFilesFromRelease, err := pivnetClient.GetProductFiles(release)
				Expect(err).ShouldNot(HaveOccurred())

				Expect(len(productFilesFromRelease.ProductFiles)).To(Equal(totalFiles))
				for _, p := range productFilesFromRelease.ProductFiles {
					Expect(sourceFileNames).To(ContainElement(p.Name))

					productFile, err := pivnetClient.GetProductFile(
						productSlug,
						release.ID,
						p.ID,
					)
					Expect(err).ShouldNot(HaveOccurred())
					// Contents are fixed at 'some contents' so the MD5 is known.
					Expect(productFile.MD5).To(Equal("9893532233caff98cd083a116b013c0b"))
				}

				By("Downloading all files via in command and glob")
				inRequest := concourse.InRequest{
					Source: concourse.Source{
						APIToken:    pivnetAPIToken,
						ProductSlug: productSlug,
						Endpoint:    endpoint,
					},
					Params: concourse.InParams{
						Globs: []string{"*"},
					},
					Version: concourse.Version{
						ProductVersion: versionWithETag,
					},
				}

				destDirectory, err := ioutil.TempDir("", "pivnet-out-test")
				Expect(err).NotTo(HaveOccurred())

				stdinContents, err = json.Marshal(inRequest)
				Expect(err).NotTo(HaveOccurred())

				downloadCmd := exec.Command(inPath, destDirectory)

				By("Running the command")
				inSession := run(downloadCmd, stdinContents)
				Eventually(inSession, executableTimeout).Should(gexec.Exit(0))

				By("Validating number of downloaded files")
				files, err := ioutil.ReadDir(destDirectory)
				Expect(err).ShouldNot(HaveOccurred())

				expectedFileCount := totalFiles + 1 // the version file will be present.
				Expect(err).ShouldNot(HaveOccurred())
				Expect(files).To(HaveLen(expectedFileCount))

				By("Validating files have non-zero-length content")
				for _, f := range files {
					Expect(f.Size()).To(BeNumerically(">", 0))
				}

				By("Downloading no files via in command and no glob")
				inRequest = concourse.InRequest{
					Source: concourse.Source{
						APIToken:    pivnetAPIToken,
						ProductSlug: productSlug,
						Endpoint:    endpoint,
					},
					Version: concourse.Version{
						ProductVersion: versionWithETag,
					},
				}

				destDirectory, err = ioutil.TempDir("", "pivnet-out-test")
				Expect(err).NotTo(HaveOccurred())

				stdinContents, err = json.Marshal(inRequest)
				Expect(err).NotTo(HaveOccurred())

				downloadCmd = exec.Command(inPath, destDirectory)

				By("Running the command")
				inSession = run(downloadCmd, stdinContents)
				Eventually(inSession, executableTimeout).Should(gexec.Exit(0))

				By("Validating number of downloaded files")
				files, err = ioutil.ReadDir(destDirectory)
				Expect(err).ShouldNot(HaveOccurred())

				expectedFileCount = 1 // the version file will be present.
				Expect(err).ShouldNot(HaveOccurred())
				Expect(files).To(HaveLen(expectedFileCount))

				Expect(files[0].Name()).To(Equal("version"))

				By("Expecting error with in command and mismatched globs")
				inRequest = concourse.InRequest{
					Source: concourse.Source{
						APIToken:    pivnetAPIToken,
						ProductSlug: productSlug,
						Endpoint:    endpoint,
					},
					Params: concourse.InParams{
						Globs: []string{filePrefix + "*", "badglob"},
					},
					Version: concourse.Version{
						ProductVersion: versionWithETag,
					},
				}

				destDirectory, err = ioutil.TempDir("", "pivnet-out-test")
				Expect(err).NotTo(HaveOccurred())

				stdinContents, err = json.Marshal(inRequest)
				Expect(err).NotTo(HaveOccurred())

				downloadCmd = exec.Command(inPath, destDirectory)

				By("Running the command, expecting error")
				inSession = run(downloadCmd, stdinContents)
				Eventually(inSession, executableTimeout).Should(gexec.Exit(1))

				By("Deleting created files on pivnet")
				for _, p := range newProductFiles {
					_, err := pivnetClient.DeleteProductFile(productSlug, p.ID)
					Expect(err).ShouldNot(HaveOccurred())
				}
			})
		})
	})
})
<commit_msg>Update lifecycle test to account for metadata files.<commit_after>package acceptance

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	"gopkg.in/yaml.v2"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/pivotal-cf-experimental/pivnet-resource/concourse"
	"github.com/pivotal-cf-experimental/pivnet-resource/metadata"
	"github.com/pivotal-cf-experimental/pivnet-resource/pivnet"
	"github.com/pivotal-cf-experimental/pivnet-resource/versions"
)

var _ = Describe("Lifecycle test", func() {
	var (
		releaseType     = "Minor Release"
		releaseDate     = "2015-12-17"
		eulaSlug        = "pivotal_beta_eula"
		description     = "this release is for automated-testing only."
		releaseNotesURL = "https://example.com"

		metadataFile   = "metadata"
		productVersion string

		filePrefix = "pivnet-resource-test-file"

		command       *exec.Cmd
		stdinContents []byte
		outRequest    concourse.OutRequest
		rootDir       string
	)

	BeforeEach(func() {
		var err error

		By("Creating a temporary root dir")
		rootDir, err = ioutil.TempDir("", "")
		Expect(err).ShouldNot(HaveOccurred())

		By("Generating 'random' product version")
		productVersion = fmt.Sprintf("%d", time.Now().Nanosecond())

		By("Creating a metadata struct")
		productMetadata := metadata.Metadata{
			Release: &metadata.Release{
				ReleaseType:     releaseType,
				EULASlug:        eulaSlug,
				ReleaseDate:     releaseDate,
				Description:     description,
				ReleaseNotesURL: releaseNotesURL,
				Version:         productVersion,
			},
		}

		By("Marshaling the metadata to yaml")
		metadataBytes, err := yaml.Marshal(productMetadata)
		Expect(err).ShouldNot(HaveOccurred())

		By("Writing the metadata to a file")
		err = ioutil.WriteFile(
			filepath.Join(rootDir, metadataFile),
			metadataBytes,
			os.ModePerm)
		Expect(err).ShouldNot(HaveOccurred())

		By("Creating command object")
		command = exec.Command(outPath, rootDir)

		By("Creating default request")
		outRequest = concourse.OutRequest{
			Source: concourse.Source{
				APIToken:        pivnetAPIToken,
				AccessKeyID:     awsAccessKeyID,
				SecretAccessKey: awsSecretAccessKey,
				ProductSlug:     productSlug,
				Endpoint:        endpoint,
				Bucket:          pivnetBucketName,
				Region:          pivnetRegion,
			},
			Params: concourse.OutParams{
				FileGlob:       "*",
				FilepathPrefix: s3FilepathPrefix,
				MetadataFile:   metadataFile,
			},
		}

		stdinContents, err = json.Marshal(outRequest)
		Expect(err).ShouldNot(HaveOccurred())
	})

	AfterEach(func() {
		By("Removing local temp files")
		err := os.RemoveAll(rootDir)
		Expect(err).ShouldNot(HaveOccurred())
	})

	Describe("Creating a new release", func() {
		// We do not delete the release as it causes race conditions with other tests

		Context("when S3 source and params are configured correctly", func() {
			var (
				client *s3client

				sourcesDir      = "sources"
				sourceFileNames []string
				sourceFilePaths []string
				remotePaths     []string

				totalFiles = 3
			)

			BeforeEach(func() {
				By("Creating aws client")
				var err error
				client, err = NewS3Client(
					awsAccessKeyID,
					awsSecretAccessKey,
					pivnetRegion,
					pivnetBucketName,
				)
				Expect(err).ShouldNot(HaveOccurred())

				By("Creating a temporary sources dir")
				sourcesFullPath := filepath.Join(rootDir, sourcesDir)
				err = os.Mkdir(sourcesFullPath, os.ModePerm)
				Expect(err).ShouldNot(HaveOccurred())

				By("Creating local temp files")
				sourceFileNames = make([]string, totalFiles)
				sourceFilePaths = make([]string, totalFiles)
				remotePaths = make([]string, totalFiles)
				for i := 0; i < totalFiles; i++ {
					sourceFileNames[i] = fmt.Sprintf(
						"%s-%d",
						filePrefix,
						time.Now().Nanosecond(),
					)

					sourceFilePaths[i] = filepath.Join(
						sourcesFullPath,
						sourceFileNames[i],
					)

					err = ioutil.WriteFile(
						sourceFilePaths[i],
						[]byte("some content"),
						os.ModePerm,
					)
					Expect(err).ShouldNot(HaveOccurred())

					remotePaths[i] = fmt.Sprintf(
						"product_files/%s/%s",
						s3FilepathPrefix,
						sourceFileNames[i],
					)
				}

				outRequest.Params.FileGlob = fmt.Sprintf("%s/*", sourcesDir)
				outRequest.Params.FilepathPrefix = s3FilepathPrefix

				stdinContents, err = json.Marshal(outRequest)
				Expect(err).ShouldNot(HaveOccurred())
			})

			AfterEach(func() {
				By("Removing uploaded file")
				for i := 0; i < totalFiles; i++ {
					client.DeleteFile(pivnetBucketName, remotePaths[i])
				}
			})

			It("uploads files to s3 and creates files on pivnet", func() {
				By("Getting existing list of product files")
				existingProductFiles := getProductFiles(productSlug)

				By("Verifying existing product files does not yet contain new files")
				var existingProductFileNames []string
				for _, f := range existingProductFiles {
					existingProductFileNames = append(existingProductFileNames, f.Name)
				}
				for i := 0; i < totalFiles; i++ {
					Expect(existingProductFileNames).NotTo(ContainElement(sourceFileNames[i]))
				}

				By("Running the command")
				session := run(command, stdinContents)
				Eventually(session, executableTimeout).Should(gexec.Exit(0))

				By("Verifying uploaded files can be downloaded directly from S3")
				for i := 0; i < totalFiles; i++ {
					localDownloadPath := fmt.Sprintf("%s-downloaded", sourceFilePaths[i])
					err := client.DownloadFile(pivnetBucketName, remotePaths[i], localDownloadPath)
					Expect(err).ShouldNot(HaveOccurred())
				}

				By("Outputting a valid json response")
				response := concourse.OutResponse{}
				err := json.Unmarshal(session.Out.Contents(), &response)
				Expect(err).ShouldNot(HaveOccurred())

				By("Validating the release was created correctly")
				release, err := pivnetClient.GetRelease(productSlug, productVersion)
				Expect(err).NotTo(HaveOccurred())

				releaseETag, err := pivnetClient.ReleaseETag(productSlug, release)
				Expect(err).NotTo(HaveOccurred())

				expectedVersion := fmt.Sprintf("%s#%s", productVersion, releaseETag)
				Expect(response.Version.ProductVersion).To(Equal(expectedVersion))

				By("Getting updated list of product files")
				updatedProductFiles := getProductFiles(productSlug)

				By("Verifying number of product files has increased by the expected amount")
				newProductFileCount := len(updatedProductFiles) - len(existingProductFiles)
				Expect(newProductFileCount).To(Equal(totalFiles))

				By("Verifying updated product files contains new files")
				var newProductFiles []pivnet.ProductFile
				for _, p := range updatedProductFiles {
					if stringInSlice(p.Name, sourceFileNames) {
						newProductFiles = append(newProductFiles, p)
					}
				}
				Expect(len(newProductFiles)).To(Equal(totalFiles))

				By("Getting newly-created release")
				release, err = pivnetClient.GetRelease(productSlug, productVersion)
				Expect(err).ShouldNot(HaveOccurred())

				releaseETag, err = pivnetClient.ReleaseETag(productSlug, release)
				Expect(err).NotTo(HaveOccurred())

				versionWithETag, err := versions.CombineVersionAndETag(productVersion, releaseETag)
				Expect(err).NotTo(HaveOccurred())

				By("Verifying release contains new product files")
				productFilesFromRelease, err := pivnetClient.GetProductFiles(release)
				Expect(err).ShouldNot(HaveOccurred())

				Expect(len(productFilesFromRelease.ProductFiles)).To(Equal(totalFiles))
				for _, p := range productFilesFromRelease.ProductFiles {
					Expect(sourceFileNames).To(ContainElement(p.Name))

					productFile, err := pivnetClient.GetProductFile(
						productSlug,
						release.ID,
						p.ID,
					)
					Expect(err).ShouldNot(HaveOccurred())
					// Contents are fixed at 'some contents' so the MD5 is known.
					Expect(productFile.MD5).To(Equal("9893532233caff98cd083a116b013c0b"))
				}

				By("Downloading all files via in command and glob")
				inRequest := concourse.InRequest{
					Source: concourse.Source{
						APIToken:    pivnetAPIToken,
						ProductSlug: productSlug,
						Endpoint:    endpoint,
					},
					Params: concourse.InParams{
						Globs: []string{"*"},
					},
					Version: concourse.Version{
						ProductVersion: versionWithETag,
					},
				}

				destDirectory, err := ioutil.TempDir("", "pivnet-out-test")
				Expect(err).NotTo(HaveOccurred())

				stdinContents, err = json.Marshal(inRequest)
				Expect(err).NotTo(HaveOccurred())

				downloadCmd := exec.Command(inPath, destDirectory)

				By("Running the command")
				inSession := run(downloadCmd, stdinContents)
				Eventually(inSession, executableTimeout).Should(gexec.Exit(0))

				By("Validating number of downloaded files")
				files, err := ioutil.ReadDir(destDirectory)
				Expect(err).ShouldNot(HaveOccurred())

				// one file is version; two files are metadata
				expectedFileCount := totalFiles + 3
				Expect(err).ShouldNot(HaveOccurred())
				Expect(files).To(HaveLen(expectedFileCount))

				By("Validating files have non-zero-length content")
				for _, f := range files {
					Expect(f.Size()).To(BeNumerically(">", 0))
				}

				By("Downloading no files via in command and no glob")
				inRequest = concourse.InRequest{
					Source: concourse.Source{
						APIToken:    pivnetAPIToken,
						ProductSlug: productSlug,
						Endpoint:    endpoint,
					},
					Version: concourse.Version{
						ProductVersion: versionWithETag,
					},
				}

				destDirectory, err = ioutil.TempDir("", "pivnet-out-test")
				Expect(err).NotTo(HaveOccurred())

				stdinContents, err = json.Marshal(inRequest)
				Expect(err).NotTo(HaveOccurred())

				downloadCmd = exec.Command(inPath, destDirectory)

				By("Running the command")
				inSession = run(downloadCmd, stdinContents)
				Eventually(inSession, executableTimeout).Should(gexec.Exit(0))

				By("Validating number of downloaded files")
				files, err = ioutil.ReadDir(destDirectory)
				Expect(err).ShouldNot(HaveOccurred())

				expectedFileCount = 1 // the version file will be present.
				Expect(err).ShouldNot(HaveOccurred())
				Expect(files).To(HaveLen(expectedFileCount))

				Expect(files[0].Name()).To(Equal("version"))

				By("Expecting error with in command and mismatched globs")
				inRequest = concourse.InRequest{
					Source: concourse.Source{
						APIToken:    pivnetAPIToken,
						ProductSlug: productSlug,
						Endpoint:    endpoint,
					},
					Params: concourse.InParams{
						Globs: []string{filePrefix + "*", "badglob"},
					},
					Version: concourse.Version{
						ProductVersion: versionWithETag,
					},
				}

				destDirectory, err = ioutil.TempDir("", "pivnet-out-test")
				Expect(err).NotTo(HaveOccurred())

				stdinContents, err = json.Marshal(inRequest)
				Expect(err).NotTo(HaveOccurred())

				downloadCmd = exec.Command(inPath, destDirectory)

				By("Running the command, expecting error")
				inSession = run(downloadCmd, stdinContents)
				Eventually(inSession, executableTimeout).Should(gexec.Exit(1))

				By("Deleting created files on pivnet")
				for _, p := range newProductFiles {
					_, err := pivnetClient.DeleteProductFile(productSlug, p.ID)
					Expect(err).ShouldNot(HaveOccurred())
				}
			})
		})
	})
})
<|endoftext|>


<commit_before>package sudoku

import (
	"fmt"
	"math/rand"
)

type SolveDirections []*SolveStep

const (
	NAKED_SINGLE = iota
	HIDDEN_SINGLE_IN_ROW
	HIDDEN_SINGLE_IN_COL
	HIDDEN_SINGLE_IN_BLOCK
)

type SolveStep struct {
	TargetCells  CellList
	PointerCells CellList
	Nums         []int
	Technique    SolveTechnique
}

type SolveTechnique interface {
	Name() string
	Description(*SolveStep) string
	Find(*Grid) *SolveStep
	IsFill() bool
}

type fillSolveTechnique struct {
}

type cullSolveTechnique struct {
}

var fillTechniques []SolveTechnique
var cullTechniques []SolveTechnique

func init() {
	//TODO: init techniques with enough space
	fillTechniques = append(fillTechniques, nakedSingleTechnique{})
	fillTechniques = append(fillTechniques, hiddenSingleInRow{})
	fillTechniques = append(fillTechniques, hiddenSingleInCol{})
	fillTechniques = append(fillTechniques, hiddenSingleInBlock{})
	cullTechniques = append(cullTechniques, pointingPairRow{})
	cullTechniques = append(cullTechniques, pointingPairCol{})
}

type nakedSingleTechnique struct {
	*fillSolveTechnique
}

type hiddenSingleInRow struct {
	*fillSolveTechnique
}

type hiddenSingleInCol struct {
	*fillSolveTechnique
}

type hiddenSingleInBlock struct {
	*fillSolveTechnique
}

type pointingPairRow struct {
	*cullSolveTechnique
}

type pointingPairCol struct {
	*cullSolveTechnique
}

func (self *fillSolveTechnique) IsFill() bool {
	return true
}

func (self *cullSolveTechnique) IsFill() bool {
	return false
}

func newFillSolveStep(cell *Cell, num int, technique SolveTechnique) *SolveStep {
	//TODO: why do these need to be pulled out separately?
	cellArr := [...]*Cell{cell}
	numArr := [...]int{num}
	return &SolveStep{cellArr[:], nil, numArr[:], technique}
}

func (self *SolveStep) Apply(grid *Grid) {
	if self.Technique.IsFill() {
		if len(self.TargetCells) == 0 || len(self.Nums) == 0 {
			return
		}
		cell := self.TargetCells[0].InGrid(grid)
		cell.SetNumber(self.Nums[0])
	} else {
		for _, cell := range self.TargetCells {
			gridCell := cell.InGrid(grid)
			for _, exclude := range self.Nums {
				gridCell.setExcluded(exclude, true)
			}
		}
	}
}

func (self nakedSingleTechnique) Name() string {
	return "Only Legal Number"
}

func (self nakedSingleTechnique) Description(step *SolveStep) string {
	if len(step.Nums) == 0 {
		return ""
	}
	num := step.Nums[0]
	return fmt.Sprintf("%d is the only remaining valid number for that cell", num)
}

func (self nakedSingleTechnique) Find(grid *Grid) *SolveStep {
	//This will be a random item
	obj := grid.queue.NewGetter().GetSmallerThan(2)
	if obj == nil {
		//There weren't any cells with one option.
		return nil
	}
	cell := obj.(*Cell)
	return newFillSolveStep(cell, cell.implicitNumber(), self)
}

func (self hiddenSingleInRow) Name() string {
	return "Necessary In Row"
}

func (self hiddenSingleInRow) Description(step *SolveStep) string {
	//TODO: format the text to say "first/second/third/etc"
	if len(step.TargetCells) == 0 || len(step.Nums) == 0 {
		return ""
	}
	cell := step.TargetCells[0]
	num := step.Nums[0]
	return fmt.Sprintf("%d is required in the %d row, and %d is the only column it fits", num, cell.Row+1, cell.Col+1)
}

func (self hiddenSingleInRow) Find(grid *Grid) *SolveStep {
	getter := func(index int) []*Cell {
		return grid.Row(index)
	}
	return necessaryInCollection(grid, self, getter)
}

func (self hiddenSingleInCol) Name() string {
	return "Necessary In Col"
}

func (self hiddenSingleInCol) Description(step *SolveStep) string {
	//TODO: format the text to say "first/second/third/etc"
	if len(step.TargetCells) == 0 || len(step.Nums) == 0 {
		return ""
	}
	cell := step.TargetCells[0]
	num := step.Nums[0]
	return fmt.Sprintf("%d is required in the %d column, and %d is the only row it fits", num, cell.Row+1, cell.Col+1)
}

func (self hiddenSingleInCol) Find(grid *Grid) *SolveStep {
	getter := func(index int) []*Cell {
		return grid.Col(index)
	}
	return necessaryInCollection(grid, self, getter)
}

func (self hiddenSingleInBlock) Name() string {
	return "Necessary In Block"
}

func (self hiddenSingleInBlock) Description(step *SolveStep) string {
	//TODO: format the text to say "first/second/third/etc"
	if len(step.TargetCells) == 0 || len(step.Nums) == 0 {
		return ""
	}
	cell := step.TargetCells[0]
	num := step.Nums[0]
	return fmt.Sprintf("%d is required in the %d block, and %d, %d is the only cell it fits", num, cell.Block+1, cell.Row+1, cell.Col+1)
}

func (self hiddenSingleInBlock) Find(grid *Grid) *SolveStep {
	getter := func(index int) []*Cell {
		return grid.Block(index)
	}
	return necessaryInCollection(grid, self, getter)
}

func necessaryInCollection(grid *Grid, technique SolveTechnique, collectionGetter func(index int) []*Cell) *SolveStep {
	//This will be a random item
	indexes := rand.Perm(DIM)

	for _, i := range indexes {
		seenInCollection := make([]int, DIM)
		collection := collectionGetter(i)
		for _, cell := range collection {
			for _, possibility := range cell.Possibilities() {
				seenInCollection[possibility-1]++
			}
		}
		seenIndexes := rand.Perm(DIM)
		for _, index := range seenIndexes {
			seen := seenInCollection[index]
			if seen == 1 {
				//Okay, we know our target number. Which cell was it?
				for _, cell := range collection {
					if cell.Possible(index + 1) {
						//Found it!
						return newFillSolveStep(cell, index+1, technique)
					}
				}
			}
		}
	}
	//Nope.
	return nil
}

func (self pointingPairRow) Name() string {
	return "Pointing pair row"
}

func (self pointingPairRow) Description(step *SolveStep) string {
	if len(step.Nums) == 0 {
		return ""
	}
	return fmt.Sprintf("%d is only possible in row %d of block %d, which means it can't be in any other cell in that row not in that block", step.Nums[0], step.TargetCells.Row(), step.PointerCells.Block())
}

func (self pointingPairRow) Find(grid *Grid) *SolveStep {
	//Within each block, for each number, see if all items that allow it are aligned in a row or column.
	//TODO: randomize order of blocks.
	//TODO: should we create a FilterByLegal method on lists of cells?
	//TODO: this is substantially duplicated in pointingPaircol
	for i := 0; i < DIM; i++ {
		block := grid.Block(i)
		//TODO: randomize order of numbers to test for.
		for num := 0; num < DIM; num++ {
			cells := block.FilterByPossible(num + 1)
			//cellList is now a list of all cells that have that number.
			if len(cells) == 0 || len(cells) > BLOCK_DIM {
				//Meh, not a match.
				continue
			}
			//Okay, it's possible it's a match. Are all rows the same?
			if cells.SameRow() {
				//Yup!
				return &SolveStep{grid.Row(cells.Row()).RemoveCells(cells), cells, []int{num + 1}, self}
			}
		}
	}
	return nil
}

func (self pointingPairCol) Name() string {
	return "Pointing pair col"
}

func (self pointingPairCol) Description(step *SolveStep) string {
	if len(step.Nums) == 0 {
		return ""
	}
	return fmt.Sprintf("%d is only possible in column %d of block %d, which means it can't be in any other cell in that column not in that block", step.Nums[0], step.TargetCells.Col(), step.PointerCells.Block())
}

func (self pointingPairCol) Find(grid *Grid) *SolveStep {
	//Within each block, for each number, see if all items that allow it are aligned in a row or column.
	//TODO: randomize order of blocks.
	//TODO: should we create a FilterByLegal method on lists of cells?
	//TODO: this is substantially duplicated in pointingPairRow
	for i := 0; i < DIM; i++ {
		block := grid.Block(i)
		//TODO: randomize order of numbers to test for.
		for num := 0; num < DIM; num++ {
			cells := block.FilterByPossible(num + 1)
			//cellList is now a list of all cells that have that number.
			if len(cells) == 0 || len(cells) > BLOCK_DIM {
				//Meh, not a match.
				continue
			}
			//Okay, are all cols?
			if cells.SameCol() {
				//Yup!
				return &SolveStep{grid.Col(cells.Col()).RemoveCells(cells), cells, []int{num + 1}, self}
			}
		}
	}
	return nil
}

func (self *Grid) HumanSolve() SolveDirections {
	var results []*SolveStep
	for !self.Solved() {
		//TODO: try the techniques in parallel
		//TODO: pick the technique based on a weighting of how common a human is to pick each one.
		//TODO: provide hints to the techniques of where to look based on the last filled cell
		techniqueOrder := rand.Perm(len(fillTechniques))
		for _, index := range techniqueOrder {
			technique := fillTechniques[index]
			step := technique.Find(self)
			if step != nil {
				results = append(results, step)
				step.Apply(self)
				break
			}
		}
		techniqueOrder = rand.Perm(len(cullTechniques))
		for _, index := range techniqueOrder {
			technique := cullTechniques[index]
			step := technique.Find(self)
			if step != nil {
				results = append(results, step)
				step.Apply(self)
				break
			}
		}
	}
	if !self.Solved() {
		//We couldn't solve the puzzle.
		return nil
	}
	return results
}
<commit_msg>pointingPairrRow/Col cull all cells in their block from target cells.<commit_after>package sudoku

import (
	"fmt"
	"math/rand"
)

type SolveDirections []*SolveStep

const (
	NAKED_SINGLE = iota
	HIDDEN_SINGLE_IN_ROW
	HIDDEN_SINGLE_IN_COL
	HIDDEN_SINGLE_IN_BLOCK
)

type SolveStep struct {
	TargetCells  CellList
	PointerCells CellList
	Nums         []int
	Technique    SolveTechnique
}

type SolveTechnique interface {
	Name() string
	Description(*SolveStep) string
	Find(*Grid) *SolveStep
	IsFill() bool
}

type fillSolveTechnique struct {
}

type cullSolveTechnique struct {
}

var fillTechniques []SolveTechnique
var cullTechniques []SolveTechnique

func init() {
	//TODO: init techniques with enough space
	fillTechniques = append(fillTechniques, nakedSingleTechnique{})
	fillTechniques = append(fillTechniques, hiddenSingleInRow{})
	fillTechniques = append(fillTechniques, hiddenSingleInCol{})
	fillTechniques = append(fillTechniques, hiddenSingleInBlock{})
	cullTechniques = append(cullTechniques, pointingPairRow{})
	cullTechniques = append(cullTechniques, pointingPairCol{})
}

type nakedSingleTechnique struct {
	*fillSolveTechnique
}

type hiddenSingleInRow struct {
	*fillSolveTechnique
}

type hiddenSingleInCol struct {
	*fillSolveTechnique
}

type hiddenSingleInBlock struct {
	*fillSolveTechnique
}

type pointingPairRow struct {
	*cullSolveTechnique
}

type pointingPairCol struct {
	*cullSolveTechnique
}

func (self *fillSolveTechnique) IsFill() bool {
	return true
}

func (self *cullSolveTechnique) IsFill() bool {
	return false
}

func newFillSolveStep(cell *Cell, num int, technique SolveTechnique) *SolveStep {
	//TODO: why do these need to be pulled out separately?
	cellArr := [...]*Cell{cell}
	numArr := [...]int{num}
	return &SolveStep{cellArr[:], nil, numArr[:], technique}
}

func (self *SolveStep) Apply(grid *Grid) {
	if self.Technique.IsFill() {
		if len(self.TargetCells) == 0 || len(self.Nums) == 0 {
			return
		}
		cell := self.TargetCells[0].InGrid(grid)
		cell.SetNumber(self.Nums[0])
	} else {
		for _, cell := range self.TargetCells {
			gridCell := cell.InGrid(grid)
			for _, exclude := range self.Nums {
				gridCell.setExcluded(exclude, true)
			}
		}
	}
}

func (self nakedSingleTechnique) Name() string {
	return "Only Legal Number"
}

func (self nakedSingleTechnique) Description(step *SolveStep) string {
	if len(step.Nums) == 0 {
		return ""
	}
	num := step.Nums[0]
	return fmt.Sprintf("%d is the only remaining valid number for that cell", num)
}

func (self nakedSingleTechnique) Find(grid *Grid) *SolveStep {
	//This will be a random item
	obj := grid.queue.NewGetter().GetSmallerThan(2)
	if obj == nil {
		//There weren't any cells with one option.
		return nil
	}
	cell := obj.(*Cell)
	return newFillSolveStep(cell, cell.implicitNumber(), self)
}

func (self hiddenSingleInRow) Name() string {
	return "Necessary In Row"
}

func (self hiddenSingleInRow) Description(step *SolveStep) string {
	//TODO: format the text to say "first/second/third/etc"
	if len(step.TargetCells) == 0 || len(step.Nums) == 0 {
		return ""
	}
	cell := step.TargetCells[0]
	num := step.Nums[0]
	return fmt.Sprintf("%d is required in the %d row, and %d is the only column it fits", num, cell.Row+1, cell.Col+1)
}

func (self hiddenSingleInRow) Find(grid *Grid) *SolveStep {
	getter := func(index int) []*Cell {
		return grid.Row(index)
	}
	return necessaryInCollection(grid, self, getter)
}

func (self hiddenSingleInCol) Name() string {
	return "Necessary In Col"
}

func (self hiddenSingleInCol) Description(step *SolveStep) string {
	//TODO: format the text to say "first/second/third/etc"
	if len(step.TargetCells) == 0 || len(step.Nums) == 0 {
		return ""
	}
	cell := step.TargetCells[0]
	num := step.Nums[0]
	return fmt.Sprintf("%d is required in the %d column, and %d is the only row it fits", num, cell.Row+1, cell.Col+1)
}

func (self hiddenSingleInCol) Find(grid *Grid) *SolveStep {
	getter := func(index int) []*Cell {
		return grid.Col(index)
	}
	return necessaryInCollection(grid, self, getter)
}

func (self hiddenSingleInBlock) Name() string {
	return "Necessary In Block"
}

func (self hiddenSingleInBlock) Description(step *SolveStep) string {
	//TODO: format the text to say "first/second/third/etc"
	if len(step.TargetCells) == 0 || len(step.Nums) == 0 {
		return ""
	}
	cell := step.TargetCells[0]
	num := step.Nums[0]
	return fmt.Sprintf("%d is required in the %d block, and %d, %d is the only cell it fits", num, cell.Block+1, cell.Row+1, cell.Col+1)
}

func (self hiddenSingleInBlock) Find(grid *Grid) *SolveStep {
	getter := func(index int) []*Cell {
		return grid.Block(index)
	}
	return necessaryInCollection(grid, self, getter)
}

func necessaryInCollection(grid *Grid, technique SolveTechnique, collectionGetter func(index int) []*Cell) *SolveStep {
	//This will be a random item
	indexes := rand.Perm(DIM)

	for _, i := range indexes {
		seenInCollection := make([]int, DIM)
		collection := collectionGetter(i)
		for _, cell := range collection {
			for _, possibility := range cell.Possibilities() {
				seenInCollection[possibility-1]++
			}
		}
		seenIndexes := rand.Perm(DIM)
		for _, index := range seenIndexes {
			seen := seenInCollection[index]
			if seen == 1 {
				//Okay, we know our target number. Which cell was it?
				for _, cell := range collection {
					if cell.Possible(index + 1) {
						//Found it!
						return newFillSolveStep(cell, index+1, technique)
					}
				}
			}
		}
	}
	//Nope.
	return nil
}

func (self pointingPairRow) Name() string {
	return "Pointing pair row"
}

func (self pointingPairRow) Description(step *SolveStep) string {
	if len(step.Nums) == 0 {
		return ""
	}
	return fmt.Sprintf("%d is only possible in row %d of block %d, which means it can't be in any other cell in that row not in that block", step.Nums[0], step.TargetCells.Row(), step.PointerCells.Block())
}

func (self pointingPairRow) Find(grid *Grid) *SolveStep {
	//Within each block, for each number, see if all items that allow it are aligned in a row or column.
	//TODO: randomize order of blocks.
	//TODO: should we create a FilterByLegal method on lists of cells?
	//TODO: this is substantially duplicated in pointingPaircol
	for i := 0; i < DIM; i++ {
		block := grid.Block(i)
		//TODO: randomize order of numbers to test for.
		for num := 0; num < DIM; num++ {
			cells := block.FilterByPossible(num + 1)
			//cellList is now a list of all cells that have that number.
			if len(cells) == 0 || len(cells) > BLOCK_DIM {
				//Meh, not a match.
				continue
			}
			//Okay, it's possible it's a match. Are all rows the same?
			if cells.SameRow() {
				//Yup!
				return &SolveStep{grid.Row(cells.Row()).RemoveCells(block), cells, []int{num + 1}, self}
			}
		}
	}
	return nil
}

func (self pointingPairCol) Name() string {
	return "Pointing pair col"
}

func (self pointingPairCol) Description(step *SolveStep) string {
	if len(step.Nums) == 0 {
		return ""
	}
	return fmt.Sprintf("%d is only possible in column %d of block %d, which means it can't be in any other cell in that column not in that block", step.Nums[0], step.TargetCells.Col(), step.PointerCells.Block())
}

func (self pointingPairCol) Find(grid *Grid) *SolveStep {
	//Within each block, for each number, see if all items that allow it are aligned in a row or column.
	//TODO: randomize order of blocks.
	//TODO: should we create a FilterByLegal method on lists of cells?
	//TODO: this is substantially duplicated in pointingPairRow
	for i := 0; i < DIM; i++ {
		block := grid.Block(i)
		//TODO: randomize order of numbers to test for.
		for num := 0; num < DIM; num++ {
			cells := block.FilterByPossible(num + 1)
			//cellList is now a list of all cells that have that number.
			if len(cells) == 0 || len(cells) > BLOCK_DIM {
				//Meh, not a match.
				continue
			}
			//Okay, are all cols?
			if cells.SameCol() {
				//Yup!
				return &SolveStep{grid.Col(cells.Col()).RemoveCells(block), cells, []int{num + 1}, self}
			}
		}
	}
	return nil
}

func (self *Grid) HumanSolve() SolveDirections {
	var results []*SolveStep
	for !self.Solved() {
		//TODO: try the techniques in parallel
		//TODO: pick the technique based on a weighting of how common a human is to pick each one.
		//TODO: provide hints to the techniques of where to look based on the last filled cell
		techniqueOrder := rand.Perm(len(fillTechniques))
		for _, index := range techniqueOrder {
			technique := fillTechniques[index]
			step := technique.Find(self)
			if step != nil {
				results = append(results, step)
				step.Apply(self)
				break
			}
		}
		techniqueOrder = rand.Perm(len(cullTechniques))
		for _, index := range techniqueOrder {
			technique := cullTechniques[index]
			step := technique.Find(self)
			if step != nil {
				results = append(results, step)
				step.Apply(self)
				break
			}
		}
	}
	if !self.Solved() {
		//We couldn't solve the puzzle.
		return nil
	}
	return results
}
<|endoftext|>


<commit_before>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package action

import (
	"fmt"

	"github.com/pkg/errors"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/cli-runtime/pkg/resource"

	"helm.sh/helm/v3/pkg/kube"
)

func existingResourceConflict(resources kube.ResourceList) error {
	err := resources.Visit(func(info *resource.Info, err error) error {
		if err != nil {
			return err
		}

		helper := resource.NewHelper(info.Client, info.Mapping)
		existing, err := helper.Get(info.Namespace, info.Name, info.Export)
		if err != nil {
			if apierrors.IsNotFound(err) {
				return nil
			}

			return errors.Wrap(err, "could not get information about the resource")
		}

		return fmt.Errorf("existing resource conflict: namespace: %s, name: %s, existing: [gvk: %s, ] / new: [gvk: %s]", info.Namespace, info.Name, existing.GetObjectKind().GroupVersionKind(), info.Mapping.GroupVersionKind)
	})
	return err
}
<commit_msg>fix error output<commit_after>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package action

import (
	"fmt"

	"github.com/pkg/errors"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/cli-runtime/pkg/resource"

	"helm.sh/helm/v3/pkg/kube"
)

func existingResourceConflict(resources kube.ResourceList) error {
	err := resources.Visit(func(info *resource.Info, err error) error {
		if err != nil {
			return err
		}

		helper := resource.NewHelper(info.Client, info.Mapping)
		existing, err := helper.Get(info.Namespace, info.Name, info.Export)
		if err != nil {
			if apierrors.IsNotFound(err) {
				return nil
			}

			return errors.Wrap(err, "could not get information about the resource")
		}

		return fmt.Errorf("existing resource conflict: namespace: %s, name: %s, existing_kind: %s, new_kind: %s", info.Namespace, info.Name, existing.GetObjectKind().GroupVersionKind(), info.Mapping.GroupVersionKind)
	})
	return err
}
<|endoftext|>


<commit_before>package checker

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"time"

	"github.com/go-chi/chi"
	"github.com/hellofresh/health-go"
	"github.com/hellofresh/janus/pkg/api"
	log "github.com/sirupsen/logrus"
)

// NewOverviewHandler creates instance of all status checks handler
func NewOverviewHandler(repo api.Repository) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		definitions, err := repo.FindValidAPIHealthChecks()
		if err != nil {
			log.WithError(err).Error("Error fetching API definitions for health check registering")
			w.WriteHeader(http.StatusInternalServerError)
			w.Write([]byte(err.Error()))
			return
		}

		log.WithField("len", len(definitions)).Debug("Loading health check endpoints")
		health.Reset()

		for _, definition := range definitions {
			log.WithField("name", definition.Name).Debug("Health check registered")
			health.Register(health.Config{
				Name:      definition.Name,
				Timeout:   time.Second * time.Duration(definition.HealthCheck.Timeout),
				SkipOnErr: true,
				Check:     check(definition),
			})
		}

		health.HandlerFunc(w, r)
	}
}

// NewStatusHandler creates instance of single proxy status check handler
func NewStatusHandler(repo api.Repository) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		definitions, err := repo.FindValidAPIHealthChecks()
		if err != nil {
			log.WithError(err).Error("Error fetching API definitions for health check registering")
			w.WriteHeader(http.StatusInternalServerError)
			w.Write([]byte(err.Error()))
			return
		}

		name := chi.URLParam(r, "name")
		for _, definition := range definitions {
			if name == definition.Name {
				resp, err := doStatusRequest(definition, false)
				if err != nil {
					log.WithField("name", name).WithError(err).Error("Error requesting service health status")
					w.WriteHeader(http.StatusInternalServerError)
					w.Write([]byte(err.Error()))
					return
				}

				body, err := ioutil.ReadAll(resp.Body)
				if closeErr := resp.Body.Close(); closeErr != nil {
					log.WithField("name", name).WithError(closeErr).Error("Error closing health status body")
				}

				if err != nil {
					log.WithField("name", name).WithError(err).Error("Error reading health status body")
					w.WriteHeader(http.StatusInternalServerError)
					w.Write([]byte(err.Error()))
					return
				}

				w.WriteHeader(resp.StatusCode)
				w.Write(body)
				return
			}
		}

		w.WriteHeader(http.StatusNotFound)
		w.Write([]byte("Definition name is not found"))
	}
}

func doStatusRequest(definition *api.Definition, closeBody bool) (*http.Response, error) {
	req, err := http.NewRequest(http.MethodGet, definition.HealthCheck.URL, nil)
	if err != nil {
		log.WithError(err).Error("Creating the request for the health check failed")
		return nil, err
	}

	// Inform to close the connection after the transaction is complete
	req.Header.Set("Connection", "close")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		log.WithError(err).Error("Making the request for the health check failed")
		return resp, err
	}

	if closeBody {
		defer resp.Body.Close()
	}

	return resp, err
}

func check(definition *api.Definition) func() error {
	return func() error {
		resp, err := doStatusRequest(definition, true)
		if err != nil {
			return err
		}

		if resp.StatusCode >= http.StatusInternalServerError {
			return fmt.Errorf("%s is not available at the moment", definition.Name)
		}

		if resp.StatusCode >= http.StatusBadRequest {
			return fmt.Errorf("%s is partially available at the moment", definition.Name)
		}

		return nil
	}
}
<commit_msg>Do not register empty healthcheck urls<commit_after>package checker

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"time"

	"github.com/go-chi/chi"
	"github.com/hellofresh/health-go"
	"github.com/hellofresh/janus/pkg/api"
	log "github.com/sirupsen/logrus"
)

// NewOverviewHandler creates instance of all status checks handler
func NewOverviewHandler(repo api.Repository) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		definitions, err := repo.FindValidAPIHealthChecks()
		if err != nil {
			log.WithError(err).Error("Error fetching API definitions for health check registering")
			w.WriteHeader(http.StatusInternalServerError)
			w.Write([]byte(err.Error()))
			return
		}

		log.WithField("len", len(definitions)).Debug("Loading health check endpoints")
		health.Reset()

		for _, definition := range definitions {
			if definition.HealthCheck.URL != "" {
				log.WithField("name", definition.Name).Debug("Registering health check")
				health.Register(health.Config{
					Name:      definition.Name,
					Timeout:   time.Second * time.Duration(definition.HealthCheck.Timeout),
					SkipOnErr: true,
					Check:     check(definition),
				})
			}
		}

		health.HandlerFunc(w, r)
	}
}

// NewStatusHandler creates instance of single proxy status check handler
func NewStatusHandler(repo api.Repository) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		definitions, err := repo.FindValidAPIHealthChecks()
		if err != nil {
			log.WithError(err).Error("Error fetching API definitions for health check registering")
			w.WriteHeader(http.StatusInternalServerError)
			w.Write([]byte(err.Error()))
			return
		}

		name := chi.URLParam(r, "name")
		for _, definition := range definitions {
			if name == definition.Name {
				resp, err := doStatusRequest(definition, false)
				if err != nil {
					log.WithField("name", name).WithError(err).Error("Error requesting service health status")
					w.WriteHeader(http.StatusInternalServerError)
					w.Write([]byte(err.Error()))
					return
				}

				body, err := ioutil.ReadAll(resp.Body)
				if closeErr := resp.Body.Close(); closeErr != nil {
					log.WithField("name", name).WithError(closeErr).Error("Error closing health status body")
				}

				if err != nil {
					log.WithField("name", name).WithError(err).Error("Error reading health status body")
					w.WriteHeader(http.StatusInternalServerError)
					w.Write([]byte(err.Error()))
					return
				}

				w.WriteHeader(resp.StatusCode)
				w.Write(body)
				return
			}
		}

		w.WriteHeader(http.StatusNotFound)
		w.Write([]byte("Definition name is not found"))
	}
}

func doStatusRequest(definition *api.Definition, closeBody bool) (*http.Response, error) {
	req, err := http.NewRequest(http.MethodGet, definition.HealthCheck.URL, nil)
	if err != nil {
		log.WithError(err).Error("Creating the request for the health check failed")
		return nil, err
	}

	// Inform to close the connection after the transaction is complete
	req.Header.Set("Connection", "close")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		log.WithError(err).Error("Making the request for the health check failed")
		return resp, err
	}

	if closeBody {
		defer resp.Body.Close()
	}

	return resp, err
}

func check(definition *api.Definition) func() error {
	return func() error {
		resp, err := doStatusRequest(definition, true)
		if err != nil {
			return err
		}

		if resp.StatusCode >= http.StatusInternalServerError {
			return fmt.Errorf("%s is not available at the moment", definition.Name)
		}

		if resp.StatusCode >= http.StatusBadRequest {
			return fmt.Errorf("%s is partially available at the moment", definition.Name)
		}

		return nil
	}
}
<|endoftext|>


<commit_before>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

/*Package cli describes the operating environment for the Helm CLI.

Helm's environment encapsulates all of the service dependencies Helm has.
These dependencies are expressed as interfaces so that alternate implementations
(mocks, etc.) can be easily generated.
*/
package cli

import (
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/spf13/pflag"
	"k8s.io/cli-runtime/pkg/genericclioptions"

	"helm.sh/helm/v3/pkg/helmpath"
)

// defaultMaxHistory sets the maximum number of releases to 0: unlimited
const defaultMaxHistory = 10

// EnvSettings describes all of the environment settings.
type EnvSettings struct {
	namespace string
	config    *genericclioptions.ConfigFlags

	// KubeConfig is the path to the kubeconfig file
	KubeConfig string
	// KubeContext is the name of the kubeconfig context.
	KubeContext string
	// Bearer KubeToken used for authentication
	KubeToken string
	// Username to impersonate for the operation
	KubeAsUser string
	// Groups to impersonate for the operation, multiple groups parsed from a comma delimited list
	KubeAsGroups []string
	// Kubernetes API Server Endpoint for authentication
	KubeAPIServer string
	// Custom certificate authority file.
	KubeCaFile string
	// Debug indicates whether or not Helm is running in Debug mode.
	Debug bool
	// RegistryConfig is the path to the registry config file.
	RegistryConfig string
	// RepositoryConfig is the path to the repositories file.
	RepositoryConfig string
	// RepositoryCache is the path to the repository cache directory.
	RepositoryCache string
	// PluginsDirectory is the path to the plugins directory.
	PluginsDirectory string
	// MaxHistory is the max release history maintained.
	MaxHistory int
}

func New() *EnvSettings {
	env := &EnvSettings{
		namespace:        os.Getenv("HELM_NAMESPACE"),
		MaxHistory:       envIntOr("HELM_MAX_HISTORY", defaultMaxHistory),
		KubeContext:      os.Getenv("HELM_KUBECONTEXT"),
		KubeToken:        os.Getenv("HELM_KUBETOKEN"),
		KubeAsUser:       os.Getenv("HELM_KUBEASUSER"),
		KubeAsGroups:     envCSV("HELM_KUBEASGROUPS"),
		KubeAPIServer:    os.Getenv("HELM_KUBEAPISERVER"),
		KubeCaFile:       os.Getenv("HELM_KUBECAFILE"),
		PluginsDirectory: envOr("HELM_PLUGINS", helmpath.DataPath("plugins")),
		RegistryConfig:   envOr("HELM_REGISTRY_CONFIG", helmpath.ConfigPath("registry.json")),
		RepositoryConfig: envOr("HELM_REPOSITORY_CONFIG", helmpath.ConfigPath("repositories.yaml")),
		RepositoryCache:  envOr("HELM_REPOSITORY_CACHE", helmpath.CachePath("repository")),
	}
	env.Debug, _ = strconv.ParseBool(os.Getenv("HELM_DEBUG"))

	// bind to kubernetes config flags
	env.config = &genericclioptions.ConfigFlags{
		Namespace:        &env.namespace,
		Context:          &env.KubeContext,
		BearerToken:      &env.KubeToken,
		APIServer:        &env.KubeAPIServer,
		CAFile:           &env.KubeCaFile,
		KubeConfig:       &env.KubeConfig,
		Impersonate:      &env.KubeAsUser,
		ImpersonateGroup: &env.KubeAsGroups,
	}
	return env
}

// AddFlags binds flags to the given flagset.
func (s *EnvSettings) AddFlags(fs *pflag.FlagSet) {
	fs.StringVarP(&s.namespace, "namespace", "n", s.namespace, "namespace scope for this request")
	fs.StringVar(&s.KubeConfig, "kubeconfig", "", "path to the kubeconfig file")
	fs.StringVar(&s.KubeContext, "kube-context", s.KubeContext, "name of the kubeconfig context to use")
	fs.StringVar(&s.KubeToken, "kube-token", s.KubeToken, "bearer token used for authentication")
	fs.StringVar(&s.KubeAsUser, "kube-as-user", s.KubeAsUser, "username to impersonate for the operation")
	fs.StringArrayVar(&s.KubeAsGroups, "kube-as-group", s.KubeAsGroups, "group to impersonate for the operation, this flag can be repeated to specify multiple groups.")
	fs.StringVar(&s.KubeAPIServer, "kube-apiserver", s.KubeAPIServer, "the address and the port for the Kubernetes API server")
	fs.StringVar(&s.KubeCaFile, "kube-ca-file", s.KubeCaFile, "the certificate authority file for the Kubernetes API server connection")
	fs.BoolVar(&s.Debug, "debug", s.Debug, "enable verbose output")
	fs.StringVar(&s.RegistryConfig, "registry-config", s.RegistryConfig, "path to the registry config file")
	fs.StringVar(&s.RepositoryConfig, "repository-config", s.RepositoryConfig, "path to the file containing repository names and URLs")
	fs.StringVar(&s.RepositoryCache, "repository-cache", s.RepositoryCache, "path to the file containing cached repository indexes")
}

func envOr(name, def string) string {
	if v, ok := os.LookupEnv(name); ok {
		return v
	}
	return def
}

func envIntOr(name string, def int) int {
	if name == "" {
		return def
	}
	envVal := envOr(name, strconv.Itoa(def))
	ret, err := strconv.Atoi(envVal)
	if err != nil {
		return def
	}
	return ret
}

func envCSV(name string) (ls []string) {
	trimmed := strings.Trim(os.Getenv(name), ", ")
	if trimmed != "" {
		ls = strings.Split(trimmed, ",")
	}
	return
}

func (s *EnvSettings) EnvVars() map[string]string {
	envvars := map[string]string{
		"HELM_BIN":               os.Args[0],
		"HELM_CACHE_HOME":        helmpath.CachePath(""),
		"HELM_CONFIG_HOME":       helmpath.ConfigPath(""),
		"HELM_DATA_HOME":         helmpath.DataPath(""),
		"HELM_DEBUG":             fmt.Sprint(s.Debug),
		"HELM_PLUGINS":           s.PluginsDirectory,
		"HELM_REGISTRY_CONFIG":   s.RegistryConfig,
		"HELM_REPOSITORY_CACHE":  s.RepositoryCache,
		"HELM_REPOSITORY_CONFIG": s.RepositoryConfig,
		"HELM_NAMESPACE":         s.Namespace(),
		"HELM_MAX_HISTORY":       strconv.Itoa(s.MaxHistory),

		// broken, these are populated from helm flags and not kubeconfig.
		"HELM_KUBECONTEXT":   s.KubeContext,
		"HELM_KUBETOKEN":     s.KubeToken,
		"HELM_KUBEASUSER":    s.KubeAsUser,
		"HELM_KUBEASGROUPS":  strings.Join(s.KubeAsGroups, ","),
		"HELM_KUBEAPISERVER": s.KubeAPIServer,
		"HELM_KUBECAFILE":    s.KubeCaFile,
	}
	if s.KubeConfig != "" {
		envvars["KUBECONFIG"] = s.KubeConfig
	}
	return envvars
}

// Namespace gets the namespace from the configuration
func (s *EnvSettings) Namespace() string {
	if ns, _, err := s.config.ToRawKubeConfigLoader().Namespace(); err == nil {
		return ns
	}
	return "default"
}

// RESTClientGetter gets the kubeconfig from EnvSettings
func (s *EnvSettings) RESTClientGetter() genericclioptions.RESTClientGetter {
	return s.config
}
<commit_msg>add SetNamespace method for EnvSettings, in order to set namespace in helm sdk<commit_after>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

/*Package cli describes the operating environment for the Helm CLI.

Helm's environment encapsulates all of the service dependencies Helm has.
These dependencies are expressed as interfaces so that alternate implementations
(mocks, etc.) can be easily generated.
*/
package cli

import (
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/spf13/pflag"
	"k8s.io/cli-runtime/pkg/genericclioptions"

	"helm.sh/helm/v3/pkg/helmpath"
)

// defaultMaxHistory sets the maximum number of releases to 0: unlimited
const defaultMaxHistory = 10

// EnvSettings describes all of the environment settings.
type EnvSettings struct {
	namespace string
	config    *genericclioptions.ConfigFlags

	// KubeConfig is the path to the kubeconfig file
	KubeConfig string
	// KubeContext is the name of the kubeconfig context.
	KubeContext string
	// Bearer KubeToken used for authentication
	KubeToken string
	// Username to impersonate for the operation
	KubeAsUser string
	// Groups to impersonate for the operation, multiple groups parsed from a comma delimited list
	KubeAsGroups []string
	// Kubernetes API Server Endpoint for authentication
	KubeAPIServer string
	// Custom certificate authority file.
	KubeCaFile string
	// Debug indicates whether or not Helm is running in Debug mode.
	Debug bool
	// RegistryConfig is the path to the registry config file.
	RegistryConfig string
	// RepositoryConfig is the path to the repositories file.
	RepositoryConfig string
	// RepositoryCache is the path to the repository cache directory.
	RepositoryCache string
	// PluginsDirectory is the path to the plugins directory.
	PluginsDirectory string
	// MaxHistory is the max release history maintained.
	MaxHistory int
}

func New() *EnvSettings {
	env := &EnvSettings{
		namespace:        os.Getenv("HELM_NAMESPACE"),
		MaxHistory:       envIntOr("HELM_MAX_HISTORY", defaultMaxHistory),
		KubeContext:      os.Getenv("HELM_KUBECONTEXT"),
		KubeToken:        os.Getenv("HELM_KUBETOKEN"),
		KubeAsUser:       os.Getenv("HELM_KUBEASUSER"),
		KubeAsGroups:     envCSV("HELM_KUBEASGROUPS"),
		KubeAPIServer:    os.Getenv("HELM_KUBEAPISERVER"),
		KubeCaFile:       os.Getenv("HELM_KUBECAFILE"),
		PluginsDirectory: envOr("HELM_PLUGINS", helmpath.DataPath("plugins")),
		RegistryConfig:   envOr("HELM_REGISTRY_CONFIG", helmpath.ConfigPath("registry.json")),
		RepositoryConfig: envOr("HELM_REPOSITORY_CONFIG", helmpath.ConfigPath("repositories.yaml")),
		RepositoryCache:  envOr("HELM_REPOSITORY_CACHE", helmpath.CachePath("repository")),
	}
	env.Debug, _ = strconv.ParseBool(os.Getenv("HELM_DEBUG"))

	// bind to kubernetes config flags
	env.config = &genericclioptions.ConfigFlags{
		Namespace:        &env.namespace,
		Context:          &env.KubeContext,
		BearerToken:      &env.KubeToken,
		APIServer:        &env.KubeAPIServer,
		CAFile:           &env.KubeCaFile,
		KubeConfig:       &env.KubeConfig,
		Impersonate:      &env.KubeAsUser,
		ImpersonateGroup: &env.KubeAsGroups,
	}
	return env
}

// AddFlags binds flags to the given flagset.
func (s *EnvSettings) AddFlags(fs *pflag.FlagSet) {
	fs.StringVarP(&s.namespace, "namespace", "n", s.namespace, "namespace scope for this request")
	fs.StringVar(&s.KubeConfig, "kubeconfig", "", "path to the kubeconfig file")
	fs.StringVar(&s.KubeContext, "kube-context", s.KubeContext, "name of the kubeconfig context to use")
	fs.StringVar(&s.KubeToken, "kube-token", s.KubeToken, "bearer token used for authentication")
	fs.StringVar(&s.KubeAsUser, "kube-as-user", s.KubeAsUser, "username to impersonate for the operation")
	fs.StringArrayVar(&s.KubeAsGroups, "kube-as-group", s.KubeAsGroups, "group to impersonate for the operation, this flag can be repeated to specify multiple groups.")
	fs.StringVar(&s.KubeAPIServer, "kube-apiserver", s.KubeAPIServer, "the address and the port for the Kubernetes API server")
	fs.StringVar(&s.KubeCaFile, "kube-ca-file", s.KubeCaFile, "the certificate authority file for the Kubernetes API server connection")
	fs.BoolVar(&s.Debug, "debug", s.Debug, "enable verbose output")
	fs.StringVar(&s.RegistryConfig, "registry-config", s.RegistryConfig, "path to the registry config file")
	fs.StringVar(&s.RepositoryConfig, "repository-config", s.RepositoryConfig, "path to the file containing repository names and URLs")
	fs.StringVar(&s.RepositoryCache, "repository-cache", s.RepositoryCache, "path to the file containing cached repository indexes")
}

func envOr(name, def string) string {
	if v, ok := os.LookupEnv(name); ok {
		return v
	}
	return def
}

func envIntOr(name string, def int) int {
	if name == "" {
		return def
	}
	envVal := envOr(name, strconv.Itoa(def))
	ret, err := strconv.Atoi(envVal)
	if err != nil {
		return def
	}
	return ret
}

func envCSV(name string) (ls []string) {
	trimmed := strings.Trim(os.Getenv(name), ", ")
	if trimmed != "" {
		ls = strings.Split(trimmed, ",")
	}
	return
}

func (s *EnvSettings) EnvVars() map[string]string {
	envvars := map[string]string{
		"HELM_BIN":               os.Args[0],
		"HELM_CACHE_HOME":        helmpath.CachePath(""),
		"HELM_CONFIG_HOME":       helmpath.ConfigPath(""),
		"HELM_DATA_HOME":         helmpath.DataPath(""),
		"HELM_DEBUG":             fmt.Sprint(s.Debug),
		"HELM_PLUGINS":           s.PluginsDirectory,
		"HELM_REGISTRY_CONFIG":   s.RegistryConfig,
		"HELM_REPOSITORY_CACHE":  s.RepositoryCache,
		"HELM_REPOSITORY_CONFIG": s.RepositoryConfig,
		"HELM_NAMESPACE":         s.Namespace(),
		"HELM_MAX_HISTORY":       strconv.Itoa(s.MaxHistory),

		// broken, these are populated from helm flags and not kubeconfig.
		"HELM_KUBECONTEXT":   s.KubeContext,
		"HELM_KUBETOKEN":     s.KubeToken,
		"HELM_KUBEASUSER":    s.KubeAsUser,
		"HELM_KUBEASGROUPS":  strings.Join(s.KubeAsGroups, ","),
		"HELM_KUBEAPISERVER": s.KubeAPIServer,
		"HELM_KUBECAFILE":    s.KubeCaFile,
	}
	if s.KubeConfig != "" {
		envvars["KUBECONFIG"] = s.KubeConfig
	}
	return envvars
}

// Namespace gets the namespace from the configuration
func (s *EnvSettings) Namespace() string {
	if ns, _, err := s.config.ToRawKubeConfigLoader().Namespace(); err == nil {
		return ns
	}
	return "default"
}

// SetNamespace sets the namespace in the configuration
func (s *EnvSettings) SetNamespace(namespace string) {
	s.namespace = namespace
}

// RESTClientGetter gets the kubeconfig from EnvSettings
func (s *EnvSettings) RESTClientGetter() genericclioptions.RESTClientGetter {
	return s.config
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package build

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"sync"

	"github.com/google/uuid"
	"github.com/olekukonko/tablewriter"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"golang.org/x/net/context"
	"google.golang.org/api/cloudbuild/v1"
	"google.golang.org/api/option"

	"k8s.io/release/pkg/command"
	"k8s.io/release/pkg/gcp"
	"k8s.io/release/pkg/release"
	"sigs.k8s.io/yaml"
)

const (
	gcsSourceDir = "/source"
	gcsLogsDir   = "/logs"

	DefaultCloudbuildFile = "cloudbuild.yaml"
)

// TODO: Pull some of these options in cmd/gcbuilder, so they don't have to be public.
type Options struct {
	BuildDir       string
	ConfigDir      string
	CloudbuildFile string
	LogDir         string
	ScratchBucket  string
	Project        string
	AllowDirty     bool
	NoSource       bool
	Async          bool
	DiskSize       string
	Variant        string
	EnvPassthrough string
}

func PrepareBuilds(o *Options) error {
	if o.ConfigDir == "" {
		return errors.New("expected a config directory to be provided")
	}

	if bazelWorkspace := os.Getenv("BUILD_WORKSPACE_DIRECTORY"); bazelWorkspace != "" {
		if err := os.Chdir(bazelWorkspace); err != nil {
			return errors.Wrapf(err, "failed to chdir to bazel workspace (%s)", bazelWorkspace)
		}
	}

	if o.BuildDir == "" {
		o.BuildDir = o.ConfigDir
	}

	logrus.Infof("Build directory: %s\n", o.BuildDir)

	// Canonicalize the config directory to be an absolute path.
	// As we're about to cd into the build directory, we need a consistent way to reference the config files
	// when the config directory is not the same as the build directory.
	absConfigDir, absErr := filepath.Abs(o.ConfigDir)
	if absErr != nil {
		return errors.Wrapf(absErr, "could not resolve absolute path for config directory")
	}

	o.ConfigDir = absConfigDir
	o.CloudbuildFile = path.Join(o.ConfigDir, o.CloudbuildFile)

	configDirErr := o.ValidateConfigDir()
	if configDirErr != nil {
		return errors.Wrapf(configDirErr, "could not validate config directory")
	}

	logrus.Infof("Config directory: %s\n", o.ConfigDir)

	logrus.Infof("Changing to build directory: %s\n", o.BuildDir)
	if err := os.Chdir(o.BuildDir); err != nil {
		return errors.Wrapf(err, "failed to chdir to build directory (%s)", o.BuildDir)
	}

	return nil
}

func (o *Options) ValidateConfigDir() error {
	configDir := o.ConfigDir
	dirInfo, err := os.Stat(o.ConfigDir)
	if os.IsNotExist(err) {
		logrus.Infof("Config directory (%s) does not exist", configDir)
		return err
	}

	if !dirInfo.IsDir() {
		logrus.Infof("Config directory (%s) is not actually a directory", configDir)
		return err
	}

	_, err = os.Stat(o.CloudbuildFile)
	if os.IsNotExist(err) {
		logrus.Infof("%s does not exist", o.CloudbuildFile)
		return err
	}

	return nil
}

func (o *Options) uploadBuildDir(targetBucket string) (string, error) {
	f, err := ioutil.TempFile("", "")
	if err != nil {
		return "", errors.Wrapf(err, "failed to create temp file")
	}
	name := f.Name()
	_ = f.Close()
	defer os.Remove(name)

	logrus.Infof("Creating source tarball at %s...", name)
	tarCmdErr := command.Execute(
		gcp.TarExecutable,
		"--exclude",
		".git",
		"-czf",
		name,
		".",
	)
	if tarCmdErr != nil {
		return "", errors.Wrapf(err, "failed to tar files")
	}

	u := uuid.New()
	uploaded := fmt.Sprintf("%s/%s.tgz", targetBucket, u.String())
	logrus.Infof("Uploading %s to %s...", name, uploaded)
	cpErr := command.Execute(
		gcp.GSUtilExecutable,
		"cp",
		name,
		uploaded,
	)
	if cpErr != nil {
		return "", errors.Wrapf(err, "failed to upload files")
	}

	return uploaded, nil
}

func getExtraSubs(o *Options) map[string]string {
	envs := strings.Split(o.EnvPassthrough, ",")
	subs := map[string]string{}
	for _, e := range envs {
		e = strings.TrimSpace(e)
		if e != "" {
			subs[e] = os.Getenv(e)
		}
	}
	return subs
}

func RunSingleJob(o *Options, jobName, uploaded, version string, subs map[string]string) error {
	s := make([]string, 0, len(subs)+1)
	for k, v := range subs {
		s = append(s, fmt.Sprintf("_%s=%s", k, v))
	}

	s = append(s, "_GIT_TAG="+version)
	args := []string{
		"builds", "submit",
		"--verbosity", "info",
		"--config", o.CloudbuildFile,
		"--substitutions", strings.Join(s, ","),
	}

	if o.Project != "" {
		args = append(args, "--project", o.Project)
	}

	if o.Async {
		args = append(args, "--async")
	}

	if o.ScratchBucket != "" {
		args = append(
			args,
			"--gcs-log-dir",
			o.ScratchBucket+gcsLogsDir,
			"--gcs-source-staging-dir",
			o.ScratchBucket+gcsSourceDir,
		)
	}

	if uploaded != "" {
		args = append(args, uploaded)
	} else {
		if o.NoSource {
			args = append(args, "--no-source")
		} else {
			args = append(args, ".")
		}
	}

	if o.DiskSize != "" {
		diskSizeInt, intErr := strconv.Atoi(o.DiskSize)
		if intErr != nil {
			return intErr
		}

		if diskSizeInt > 1000 {
			return errors.New("Selected disk size must be no greater than 1000 GB")
		} else if diskSizeInt <= 0 {
			return errors.New("Selected disk size must be greater than 0 GB")
		}

		diskSizeArg := fmt.Sprintf("--disk-size=%s", o.DiskSize)
		args = append(args, diskSizeArg)
	}

	cmd := command.New(gcp.GCloudExecutable, args...)

	if o.LogDir != "" {
		p := path.Join(o.LogDir, strings.ReplaceAll(jobName, "/", "-")+".log")
		f, err := os.Create(p)

		if err != nil {
			return errors.Wrapf(err, "couldn't create %s", p)
		}

		defer f.Close()
		cmd.AddWriter(f)
	}

	if err := cmd.RunSuccess(); err != nil {
		return errors.Wrapf(err, "error running %s", cmd.String())
	}

	return nil
}

type variants map[string]map[string]string

func getVariants(o *Options) (variants, error) {
	content, err := ioutil.ReadFile(path.Join(o.ConfigDir, "variants.yaml"))
	if err != nil {
		if !os.IsNotExist(err) {
			return nil, errors.Wrapf(err, "failed to load variants.yaml")
		}
		if o.Variant != "" {
			return nil, errors.Errorf("no variants.yaml found, but a build variant (%q) was specified", o.Variant)
		}
		return nil, nil
	}
	v := struct {
		Variants variants `json:"variants"`
	}{}
	if err := yaml.UnmarshalStrict(content, &v); err != nil {
		return nil, errors.Wrapf(err, "failed to read variants.yaml")
	}
	if o.Variant != "" {
		va, ok := v.Variants[o.Variant]
		if !ok {
			return nil, errors.Errorf("requested variant %q, which is not present in variants.yaml", o.Variant)
		}
		return variants{o.Variant: va}, nil
	}
	return v.Variants, nil
}

func RunBuildJobs(o *Options) []error {
	var uploaded string
	if o.ScratchBucket != "" {
		if !o.NoSource {
			var err error
			uploaded, err = o.uploadBuildDir(o.ScratchBucket + gcsSourceDir)
			if err != nil {
				return []error{errors.Wrapf(err, "failed to upload source")}
			}
		}
	} else {
		logrus.Info("Skipping advance upload and relying on gcloud...")
	}

	logrus.Info("Running build jobs...")
	repo := release.NewRepo()
	err := repo.Open()
	if err != nil {
		return []error{err}
	}

	tag, err := repo.GetTag()
	if err != nil {
		return []error{err}
	}

	if !o.AllowDirty && strings.HasSuffix(tag, "-dirty") {
		return []error{errors.New("the working copy is dirty")}
	}

	vs, err := getVariants(o)
	if err != nil {
		return []error{err}
	}
	if len(vs) == 0 {
		logrus.Info("No variants.yaml, starting single build job...")
		if err := RunSingleJob(o, "build", uploaded, tag, getExtraSubs(o)); err != nil {
			return []error{err}
		}
		return nil
	}

	logrus.Infof("Found variants.yaml, starting %d build jobs...", len(vs))

	w := sync.WaitGroup{}
	w.Add(len(vs))
	var jobErrors []error
	extraSubs := getExtraSubs(o)
	for k, v := range vs {
		go func(job string, vc map[string]string) {
			defer w.Done()
			logrus.Infof("Starting job %q...", job)
			if err := RunSingleJob(o, job, uploaded, tag, mergeMaps(extraSubs, vc)); err != nil {
				logrus.Infof("Job %q failed: %v", job, err)
				jobErrors = append(jobErrors, errors.Wrapf(err, "job %q failed", job))
			} else {
				logrus.Infof("Job %q completed", job)
			}
		}(k, v)
	}
	w.Wait()
	return jobErrors
}

func mergeMaps(maps ...map[string]string) map[string]string {
	out := map[string]string{}
	for _, m := range maps {
		for k, v := range m {
			out[k] = v
		}
	}
	return out
}

func ListJobs(project string, lastJobs int64) error {
	ctx := context.Background()
	opts := option.WithCredentialsFile("")

	service, err := cloudbuild.NewService(ctx, opts)
	if err != nil {
		return errors.Wrap(err, "failed to fetching gcloud credentials... try running \"gcloud auth application-default login\"")
	}

	req, err := service.Projects.Builds.List(project).PageSize(lastJobs).Do()
	if err != nil {
		return errors.Wrap(err, "failed to listing the builds")
	}

	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"Start Time", "Finish Time", "Status", "Console Logs"})

	for _, build := range req.Builds {
		table.Append([]string{strings.TrimSpace(build.StartTime), strings.TrimSpace(build.FinishTime), strings.TrimSpace(build.Status), strings.TrimSpace(build.LogUrl)})
	}
	table.Render()

	return nil
}

func GetJobsByTag(project, tagsFilter string) ([]*cloudbuild.Build, error) {
	ctx := context.Background()
	opts := option.WithCredentialsFile("")

	service, err := cloudbuild.NewService(ctx, opts)
	if err != nil {
		return nil, errors.Wrap(err, "failed to fetching gcloud credentials... try running \"gcloud auth application-default login\"")
	}

	req, err := service.Projects.Builds.List(project).Filter(tagsFilter).PageSize(50).Do()
	if err != nil {
		return nil, errors.Wrap(err, "failed to listing the builds")
	}

	return req.Builds, nil
}
<commit_msg>Remove unnecessary newlines from log messages<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package build

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"sync"

	"github.com/google/uuid"
	"github.com/olekukonko/tablewriter"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"golang.org/x/net/context"
	"google.golang.org/api/cloudbuild/v1"
	"google.golang.org/api/option"

	"k8s.io/release/pkg/command"
	"k8s.io/release/pkg/gcp"
	"k8s.io/release/pkg/release"
	"sigs.k8s.io/yaml"
)

const (
	gcsSourceDir = "/source"
	gcsLogsDir   = "/logs"

	DefaultCloudbuildFile = "cloudbuild.yaml"
)

// TODO: Pull some of these options in cmd/gcbuilder, so they don't have to be public.
type Options struct {
	BuildDir       string
	ConfigDir      string
	CloudbuildFile string
	LogDir         string
	ScratchBucket  string
	Project        string
	AllowDirty     bool
	NoSource       bool
	Async          bool
	DiskSize       string
	Variant        string
	EnvPassthrough string
}

func PrepareBuilds(o *Options) error {
	if o.ConfigDir == "" {
		return errors.New("expected a config directory to be provided")
	}

	if bazelWorkspace := os.Getenv("BUILD_WORKSPACE_DIRECTORY"); bazelWorkspace != "" {
		if err := os.Chdir(bazelWorkspace); err != nil {
			return errors.Wrapf(err, "failed to chdir to bazel workspace (%s)", bazelWorkspace)
		}
	}

	if o.BuildDir == "" {
		o.BuildDir = o.ConfigDir
	}

	logrus.Infof("Build directory: %s", o.BuildDir)

	// Canonicalize the config directory to be an absolute path.
	// As we're about to cd into the build directory, we need a consistent way to reference the config files
	// when the config directory is not the same as the build directory.
	absConfigDir, absErr := filepath.Abs(o.ConfigDir)
	if absErr != nil {
		return errors.Wrapf(absErr, "could not resolve absolute path for config directory")
	}

	o.ConfigDir = absConfigDir
	o.CloudbuildFile = path.Join(o.ConfigDir, o.CloudbuildFile)

	configDirErr := o.ValidateConfigDir()
	if configDirErr != nil {
		return errors.Wrapf(configDirErr, "could not validate config directory")
	}

	logrus.Infof("Config directory: %s", o.ConfigDir)

	logrus.Infof("Changing to build directory: %s", o.BuildDir)
	if err := os.Chdir(o.BuildDir); err != nil {
		return errors.Wrapf(err, "failed to chdir to build directory (%s)", o.BuildDir)
	}

	return nil
}

func (o *Options) ValidateConfigDir() error {
	configDir := o.ConfigDir
	dirInfo, err := os.Stat(o.ConfigDir)
	if os.IsNotExist(err) {
		logrus.Infof("Config directory (%s) does not exist", configDir)
		return err
	}

	if !dirInfo.IsDir() {
		logrus.Infof("Config directory (%s) is not actually a directory", configDir)
		return err
	}

	_, err = os.Stat(o.CloudbuildFile)
	if os.IsNotExist(err) {
		logrus.Infof("%s does not exist", o.CloudbuildFile)
		return err
	}

	return nil
}

func (o *Options) uploadBuildDir(targetBucket string) (string, error) {
	f, err := ioutil.TempFile("", "")
	if err != nil {
		return "", errors.Wrapf(err, "failed to create temp file")
	}
	name := f.Name()
	_ = f.Close()
	defer os.Remove(name)

	logrus.Infof("Creating source tarball at %s...", name)
	tarCmdErr := command.Execute(
		gcp.TarExecutable,
		"--exclude",
		".git",
		"-czf",
		name,
		".",
	)
	if tarCmdErr != nil {
		return "", errors.Wrapf(err, "failed to tar files")
	}

	u := uuid.New()
	uploaded := fmt.Sprintf("%s/%s.tgz", targetBucket, u.String())
	logrus.Infof("Uploading %s to %s...", name, uploaded)
	cpErr := command.Execute(
		gcp.GSUtilExecutable,
		"cp",
		name,
		uploaded,
	)
	if cpErr != nil {
		return "", errors.Wrapf(err, "failed to upload files")
	}

	return uploaded, nil
}

func getExtraSubs(o *Options) map[string]string {
	envs := strings.Split(o.EnvPassthrough, ",")
	subs := map[string]string{}
	for _, e := range envs {
		e = strings.TrimSpace(e)
		if e != "" {
			subs[e] = os.Getenv(e)
		}
	}
	return subs
}

func RunSingleJob(o *Options, jobName, uploaded, version string, subs map[string]string) error {
	s := make([]string, 0, len(subs)+1)
	for k, v := range subs {
		s = append(s, fmt.Sprintf("_%s=%s", k, v))
	}

	s = append(s, "_GIT_TAG="+version)
	args := []string{
		"builds", "submit",
		"--verbosity", "info",
		"--config", o.CloudbuildFile,
		"--substitutions", strings.Join(s, ","),
	}

	if o.Project != "" {
		args = append(args, "--project", o.Project)
	}

	if o.Async {
		args = append(args, "--async")
	}

	if o.ScratchBucket != "" {
		args = append(
			args,
			"--gcs-log-dir",
			o.ScratchBucket+gcsLogsDir,
			"--gcs-source-staging-dir",
			o.ScratchBucket+gcsSourceDir,
		)
	}

	if uploaded != "" {
		args = append(args, uploaded)
	} else {
		if o.NoSource {
			args = append(args, "--no-source")
		} else {
			args = append(args, ".")
		}
	}

	if o.DiskSize != "" {
		diskSizeInt, intErr := strconv.Atoi(o.DiskSize)
		if intErr != nil {
			return intErr
		}

		if diskSizeInt > 1000 {
			return errors.New("Selected disk size must be no greater than 1000 GB")
		} else if diskSizeInt <= 0 {
			return errors.New("Selected disk size must be greater than 0 GB")
		}

		diskSizeArg := fmt.Sprintf("--disk-size=%s", o.DiskSize)
		args = append(args, diskSizeArg)
	}

	cmd := command.New(gcp.GCloudExecutable, args...)

	if o.LogDir != "" {
		p := path.Join(o.LogDir, strings.ReplaceAll(jobName, "/", "-")+".log")
		f, err := os.Create(p)

		if err != nil {
			return errors.Wrapf(err, "couldn't create %s", p)
		}

		defer f.Close()
		cmd.AddWriter(f)
	}

	if err := cmd.RunSuccess(); err != nil {
		return errors.Wrapf(err, "error running %s", cmd.String())
	}

	return nil
}

type variants map[string]map[string]string

func getVariants(o *Options) (variants, error) {
	content, err := ioutil.ReadFile(path.Join(o.ConfigDir, "variants.yaml"))
	if err != nil {
		if !os.IsNotExist(err) {
			return nil, errors.Wrapf(err, "failed to load variants.yaml")
		}
		if o.Variant != "" {
			return nil, errors.Errorf("no variants.yaml found, but a build variant (%q) was specified", o.Variant)
		}
		return nil, nil
	}
	v := struct {
		Variants variants `json:"variants"`
	}{}
	if err := yaml.UnmarshalStrict(content, &v); err != nil {
		return nil, errors.Wrapf(err, "failed to read variants.yaml")
	}
	if o.Variant != "" {
		va, ok := v.Variants[o.Variant]
		if !ok {
			return nil, errors.Errorf("requested variant %q, which is not present in variants.yaml", o.Variant)
		}
		return variants{o.Variant: va}, nil
	}
	return v.Variants, nil
}

func RunBuildJobs(o *Options) []error {
	var uploaded string
	if o.ScratchBucket != "" {
		if !o.NoSource {
			var err error
			uploaded, err = o.uploadBuildDir(o.ScratchBucket + gcsSourceDir)
			if err != nil {
				return []error{errors.Wrapf(err, "failed to upload source")}
			}
		}
	} else {
		logrus.Info("Skipping advance upload and relying on gcloud...")
	}

	logrus.Info("Running build jobs...")
	repo := release.NewRepo()
	err := repo.Open()
	if err != nil {
		return []error{err}
	}

	tag, err := repo.GetTag()
	if err != nil {
		return []error{err}
	}

	if !o.AllowDirty && strings.HasSuffix(tag, "-dirty") {
		return []error{errors.New("the working copy is dirty")}
	}

	vs, err := getVariants(o)
	if err != nil {
		return []error{err}
	}
	if len(vs) == 0 {
		logrus.Info("No variants.yaml, starting single build job...")
		if err := RunSingleJob(o, "build", uploaded, tag, getExtraSubs(o)); err != nil {
			return []error{err}
		}
		return nil
	}

	logrus.Infof("Found variants.yaml, starting %d build jobs...", len(vs))

	w := sync.WaitGroup{}
	w.Add(len(vs))
	var jobErrors []error
	extraSubs := getExtraSubs(o)
	for k, v := range vs {
		go func(job string, vc map[string]string) {
			defer w.Done()
			logrus.Infof("Starting job %q...", job)
			if err := RunSingleJob(o, job, uploaded, tag, mergeMaps(extraSubs, vc)); err != nil {
				logrus.Infof("Job %q failed: %v", job, err)
				jobErrors = append(jobErrors, errors.Wrapf(err, "job %q failed", job))
			} else {
				logrus.Infof("Job %q completed", job)
			}
		}(k, v)
	}
	w.Wait()
	return jobErrors
}

func mergeMaps(maps ...map[string]string) map[string]string {
	out := map[string]string{}
	for _, m := range maps {
		for k, v := range m {
			out[k] = v
		}
	}
	return out
}

func ListJobs(project string, lastJobs int64) error {
	ctx := context.Background()
	opts := option.WithCredentialsFile("")

	service, err := cloudbuild.NewService(ctx, opts)
	if err != nil {
		return errors.Wrap(err, "failed to fetching gcloud credentials... try running \"gcloud auth application-default login\"")
	}

	req, err := service.Projects.Builds.List(project).PageSize(lastJobs).Do()
	if err != nil {
		return errors.Wrap(err, "failed to listing the builds")
	}

	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"Start Time", "Finish Time", "Status", "Console Logs"})

	for _, build := range req.Builds {
		table.Append([]string{strings.TrimSpace(build.StartTime), strings.TrimSpace(build.FinishTime), strings.TrimSpace(build.Status), strings.TrimSpace(build.LogUrl)})
	}
	table.Render()

	return nil
}

func GetJobsByTag(project, tagsFilter string) ([]*cloudbuild.Build, error) {
	ctx := context.Background()
	opts := option.WithCredentialsFile("")

	service, err := cloudbuild.NewService(ctx, opts)
	if err != nil {
		return nil, errors.Wrap(err, "failed to fetching gcloud credentials... try running \"gcloud auth application-default login\"")
	}

	req, err := service.Projects.Builds.List(project).Filter(tagsFilter).PageSize(50).Do()
	if err != nil {
		return nil, errors.Wrap(err, "failed to listing the builds")
	}

	return req.Builds, nil
}
<|endoftext|>