<commit_before>package main

import (
	"archive/zip"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"gopkg.in/yaml.v2"
)

var (
	err error
)

type Config struct {
	Static []StaticProject
	Paths  PathConfig
}

type PathConfig struct {
	Deployd string
	Static  string
}

type StaticProject struct {
	Name       string
	Branch     string
	Domain     string
	Subdomain  string
	GitHub     bool
	Bucket     string
	Owner      string
	Repository string
}

func main() {
	configPath := flag.String("config", "/etc/deployd.conf", "Path to the config file")

	flag.Parse()

	if _, err = os.Stat(*configPath); os.IsNotExist(err) {
		log.Fatal(err)
	}

	data, err := ioutil.ReadFile(*configPath)

	if err != nil {
		log.Fatal(err)
	}

	var config Config
	err = yaml.Unmarshal(data, &config)

	if err != nil {
		log.Fatal(err)
	}

	if _, err := os.Stat(config.Paths.Deployd); os.IsNotExist(err) {
		os.MkdirAll(config.Paths.Deployd, 0700)
	}

	if _, err := os.Stat(config.Paths.Static); os.IsNotExist(err) {
		os.MkdirAll(config.Paths.Static, 0700)
	}

	for _, project := range config.Static {
		archivePath := fmt.Sprintf("%v/%v-%v.zip", config.Paths.Deployd, project.Name, project.Branch)

		err = os.RemoveAll(archivePath)

		if err != nil {
			log.Fatal(err)
		}

		archive, err := os.Create(archivePath)
		defer archive.Close()

		if err != nil {
			log.Fatal(err)
		}

		var archiveLocation string

		if project.GitHub {
			archiveLocation = fmt.Sprintf("https://github.com/%v/%v/archive/%v.zip", project.Owner, project.Repository, project.Branch)
		} else {
			archiveLocation = fmt.Sprintf("https://s3.amazonaws.com/%v/%v-latest.zip", project.Bucket, project.Branch)
		}

		response, err := http.Get(archiveLocation)
		defer response.Body.Close()

		if err != nil {
			log.Fatal(err)
		}

		_, err = io.Copy(archive, response.Body)

		if err != nil {
			log.Fatal(err)
		}

		unarchivedPath := fmt.Sprintf("%v/%v-%v", config.Paths.Deployd, project.Name, project.Branch)

		err = os.RemoveAll(unarchivedPath)

		if err != nil {
			log.Fatal(err)
		}

		err = unzip(archivePath, unarchivedPath)

		if err != nil {
			log.Fatal(err)
		}

		if project.GitHub {
			unarchivedPath = fmt.Sprintf("%v/%v-%v", unarchivedPath, project.Repository, project.Branch)
		}

		domainPath := fmt.Sprintf("%v/%v", config.Paths.Static, project.Domain)
		projectPath := fmt.Sprintf("%v/%v/%v", config.Paths.Static, project.Domain, project.Subdomain)

		err = os.RemoveAll(projectPath)

		if err != nil {
			log.Fatal(err)
		}

		os.Mkdir(domainPath, 0700)

		err = os.Rename(unarchivedPath, projectPath)

		if err != nil {
			log.Fatal(err)
		}

		err = os.RemoveAll(archivePath)

		if err != nil {
			log.Fatal(err)
		}
	}
}

// unzip function by http://stackoverflow.com/users/1129149/swtdrgn
// http://stackoverflow.com/questions/20357223/easy-way-to-unzip-file-with-golang/24430720#24430720

func unzip(src, dest string) error {
	r, err := zip.OpenReader(src)
	if err != nil {
		return err
	}
	defer r.Close()

	for _, f := range r.File {
		rc, err := f.Open()
		if err != nil {
			return err
		}
		defer rc.Close()

		fpath := filepath.Join(dest, f.Name)
		if f.FileInfo().IsDir() {
			os.MkdirAll(fpath, f.Mode())
		} else {
			var fdir string
			if lastIndex := strings.LastIndex(fpath, string(os.PathSeparator)); lastIndex > -1 {
				fdir = fpath[:lastIndex]
			}

			err = os.MkdirAll(fdir, f.Mode())
			if err != nil {
				log.Fatal(err)
				return err
			}
			f, err := os.OpenFile(
				fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
			if err != nil {
				return err
			}
			defer f.Close()

			_, err = io.Copy(f, rc)
			if err != nil {
				return err
			}
		}
	}
	return nil
}
<commit_msg>Added database of checksums to prevent duplicate deploys<commit_after>package main

import (
	"archive/zip"
	"crypto/md5"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"gopkg.in/yaml.v2"
)

var (
	err error
)

type Config struct {
	Static []StaticProject
	Paths  PathConfig
}

type PathConfig struct {
	Deployd string
	Static  string
	Cache   string
}

type StaticProject struct {
	Name       string
	Branch     string
	Domain     string
	Subdomain  string
	GitHub     bool
	Bucket     string
	Owner      string
	Repository string
}

type CacheRecord struct {
	Domain    string
	Subdomain string
	Checksum  string
}

func main() {
	configPath := flag.String("config", "/etc/deployd.conf", "Path to the config file")

	flag.Parse()

	if _, err = os.Stat(*configPath); os.IsNotExist(err) {
		log.Fatal(err)
	}

	data, err := ioutil.ReadFile(*configPath)

	if err != nil {
		log.Fatal(err)
	}

	var config Config
	err = yaml.Unmarshal(data, &config)

	if err != nil {
		log.Fatal(err)
	}

	var cache []CacheRecord

	if _, err = os.Stat(config.Paths.Cache); os.IsNotExist(err) {
		cacheFile, err := os.Create(config.Paths.Cache)
		cacheFile.Close()

		if err != nil {
			log.Fatal(err)
		}
	} else {
		cacheData, err := ioutil.ReadFile(config.Paths.Cache)

		if err != nil {
			log.Fatal(err)
		}

		err = yaml.Unmarshal(cacheData, &cache)

		if err != nil {
			log.Fatal(err)
		}
	}

	if _, err := os.Stat(config.Paths.Deployd); os.IsNotExist(err) {
		os.MkdirAll(config.Paths.Deployd, 0700)
	}

	if _, err := os.Stat(config.Paths.Static); os.IsNotExist(err) {
		os.MkdirAll(config.Paths.Static, 0700)
	}

	for _, project := range config.Static {
		var record CacheRecord

		for _, r := range cache {
			if r.Domain == project.Domain && r.Subdomain == project.Subdomain {
				record = r
				break
			}
		}

		if record.Domain == "" {
			record.Domain = project.Domain
			record.Subdomain = project.Subdomain
		}

		archivePath := fmt.Sprintf("%v/%v-%v.zip", config.Paths.Deployd, project.Name, project.Branch)

		err = os.RemoveAll(archivePath)

		if err != nil {
			log.Fatal(err)
		}

		archive, err := os.Create(archivePath)
		defer archive.Close()

		if err != nil {
			log.Fatal(err)
		}

		var archiveLocation string

		if project.GitHub {
			archiveLocation = fmt.Sprintf("https://github.com/%v/%v/archive/%v.zip", project.Owner, project.Repository, project.Branch)
		} else {
			archiveLocation = fmt.Sprintf("https://s3.amazonaws.com/%v/%v-latest.zip", project.Bucket, project.Branch)
		}

		response, err := http.Get(archiveLocation)
		defer response.Body.Close()

		if err != nil {
			log.Fatal(err)
		}

		_, err = io.Copy(archive, response.Body)

		if err != nil {
			log.Fatal(err)
		}

		archiveData, err := ioutil.ReadFile(archivePath)

		if err != nil {
			log.Fatal(err)
		}

		rawChecksum := md5.Sum(archiveData)
		checksum := fmt.Sprintf("%x", rawChecksum)

		if checksum == record.Checksum {
			continue
		}
		fmt.Println("Replacing")
		record.Checksum = checksum

		cache = append(cache, record)

		unarchivedPath := fmt.Sprintf("%v/%v-%v", config.Paths.Deployd, project.Name, project.Branch)

		err = os.RemoveAll(unarchivedPath)

		if err != nil {
			log.Fatal(err)
		}

		err = unzip(archivePath, unarchivedPath)

		if err != nil {
			log.Fatal(err)
		}

		if project.GitHub {
			unarchivedPath = fmt.Sprintf("%v/%v-%v", unarchivedPath, project.Repository, project.Branch)
		}

		domainPath := fmt.Sprintf("%v/%v", config.Paths.Static, project.Domain)
		projectPath := fmt.Sprintf("%v/%v/%v", config.Paths.Static, project.Domain, project.Subdomain)

		err = os.RemoveAll(projectPath)

		if err != nil {
			log.Fatal(err)
		}

		os.Mkdir(domainPath, 0700)

		err = os.Rename(unarchivedPath, projectPath)

		if err != nil {
			log.Fatal(err)
		}

		err = os.RemoveAll(archivePath)

		if err != nil {
			log.Fatal(err)
		}
	}

	cacheData, err := yaml.Marshal(&cache)

	if err != nil {
		log.Fatal(err)
	}

	err = ioutil.WriteFile(config.Paths.Cache, cacheData, 0700)

	if err != nil {
		log.Fatal(err)
	}
}

// unzip function by http://stackoverflow.com/users/1129149/swtdrgn
// http://stackoverflow.com/questions/20357223/easy-way-to-unzip-file-with-golang/24430720#24430720

func unzip(src, dest string) error {
	r, err := zip.OpenReader(src)
	if err != nil {
		return err
	}
	defer r.Close()

	for _, f := range r.File {
		rc, err := f.Open()
		if err != nil {
			return err
		}
		defer rc.Close()

		fpath := filepath.Join(dest, f.Name)
		if f.FileInfo().IsDir() {
			os.MkdirAll(fpath, f.Mode())
		} else {
			var fdir string
			if lastIndex := strings.LastIndex(fpath, string(os.PathSeparator)); lastIndex > -1 {
				fdir = fpath[:lastIndex]
			}

			err = os.MkdirAll(fdir, f.Mode())
			if err != nil {
				log.Fatal(err)
				return err
			}
			f, err := os.OpenFile(
				fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
			if err != nil {
				return err
			}
			defer f.Close()

			_, err = io.Copy(f, rc)
			if err != nil {
				return err
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package netlify

import (
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/cenkalti/backoff"
)

const MaxFilesForSyncDeploy = 1000
const PreProcessingTimeout = time.Minute * 5

// Deploy represents a specific deploy of a site
type Deploy struct {
	Id     string `json:"id"`
	SiteId string `json:"site_id"`
	UserId string `json:"user_id"`

	// State of the deploy (uploading/uploaded/processing/ready/error)
	State string `json:"state"`

	// Cause of error if State is "error"
	ErrorMessage string `json:"error_message"`

	// Shas of files that needs to be uploaded before the deploy is ready
	Required []string `json:"required"`

	DeployUrl     string `json:"deploy_url"`
	SiteUrl       string `json:"url"`
	ScreenshotUrl string `json:"screenshot_url"`

	CreatedAt Timestamp `json:"created_at"`
	UpdatedAt Timestamp `json:"updated_at"`

	Branch    string `json:"branch,omitempty"`
	CommitRef string `json:"commit_ref,omitempty"`

	client *Client
	logger *logrus.Entry
}

func (d Deploy) log() *logrus.Entry {
	if d.logger == nil {
		d.logger = d.client.log.WithFields(logrus.Fields{
			"function": "deploy",
			"id":       d.Id,
			"site_id":  d.SiteId,
			"user_id":  d.UserId,
		})
	}

	return d.logger.WithField("state", d.State)
}

// DeploysService is used to access all Deploy related API methods
type DeploysService struct {
	site   *Site
	client *Client
}

type uploadError struct {
	err   error
	mutex *sync.Mutex
}

func (u *uploadError) Set(err error) {
	if err != nil {
		u.mutex.Lock()
		defer u.mutex.Unlock()
		if u.err != nil {
			u.err = err
		}
	}
}

func (u *uploadError) Empty() bool {
	u.mutex.Lock()
	defer u.mutex.Unlock()
	return u.err == nil
}

type deployFiles struct {
	Files     *map[string]string `json:"files"`
	Async     bool               `json:"async"`
	Branch    string             `json:"branch,omitempty"`
	CommitRef string             `json:"commit_ref,omitempty"`
}

func (s *DeploysService) apiPath() string {
	if s.site != nil {
		return path.Join(s.site.apiPath(), "deploys")
	}
	return "/deploys"
}

// Create a new deploy
//
// Example: site.Deploys.Create("/path/to/site-dir", true)
// If the target is a zip file, it must have the extension .zip
func (s *DeploysService) Create(dirOrZip string) (*Deploy, *Response, error) {
	return s.create(dirOrZip, false)
}

// CreateDraft a new draft deploy. Draft deploys will be uploaded and processed, but
// won't affect the active deploy for a site.
func (s *DeploysService) CreateDraft(dirOrZip string) (*Deploy, *Response, error) {
	return s.create(dirOrZip, true)
}

func (s *DeploysService) create(dirOrZip string, draft bool) (*Deploy, *Response, error) {
	if s.site == nil {
		return nil, nil, errors.New("You can only create a new deploy for an existing site (site.Deploys.Create(dirOrZip)))")
	}

	params := url.Values{}
	if draft {
		params["draft"] = []string{"true"}
	}
	options := &RequestOptions{QueryParams: &params}
	deploy := &Deploy{client: s.client}
	resp, err := s.client.Request("POST", s.apiPath(), options, deploy)

	if err != nil {
		return deploy, resp, err
	}

	resp, err = deploy.Deploy(dirOrZip)
	return deploy, resp, err
}

// List all deploys. Takes ListOptions to control pagination.
func (s *DeploysService) List(options *ListOptions) ([]Deploy, *Response, error) {
	deploys := new([]Deploy)

	reqOptions := &RequestOptions{QueryParams: options.toQueryParamsMap()}

	resp, err := s.client.Request("GET", s.apiPath(), reqOptions, deploys)

	for _, deploy := range *deploys {
		deploy.client = s.client
	}

	return *deploys, resp, err
}

// Get a specific deploy.
func (d *DeploysService) Get(id string) (*Deploy, *Response, error) {
	deploy := &Deploy{Id: id, client: d.client}
	resp, err := deploy.Reload()

	return deploy, resp, err
}

func (deploy *Deploy) apiPath() string {
	return path.Join("/deploys", deploy.Id)
}

func (deploy *Deploy) Deploy(dirOrZip string) (*Response, error) {
	if strings.HasSuffix(dirOrZip, ".zip") {
		return deploy.deployZip(dirOrZip)
	} else {
		return deploy.deployDir(dirOrZip)
	}
}

// Reload a deploy from the API
func (deploy *Deploy) Reload() (*Response, error) {
	if deploy.Id == "" {
		return nil, errors.New("Cannot fetch deploy without an ID")
	}
	return deploy.client.Request("GET", deploy.apiPath(), nil, deploy)
}

// Restore an old deploy. Sets the deploy as the active deploy for a site
func (deploy *Deploy) Restore() (*Response, error) {
	return deploy.client.Request("POST", path.Join(deploy.apiPath(), "restore"), nil, deploy)
}

// Alias for restore. Published a specific deploy.
func (deploy *Deploy) Publish() (*Response, error) {
	return deploy.Restore()
}

func (deploy *Deploy) uploadFile(dir, path string, sharedError uploadError) error {
	if !sharedError.Empty() {
		return errors.New("Canceled because upload has already failed")
	}

	log := deploy.log().WithFields(logrus.Fields{
		"dir":  dir,
		"path": path,
	})

	log.Infof("Uploading file: %v", path)
	file, err := os.Open(filepath.Join(dir, path))
	defer file.Close()

	if err != nil {
		log.Warnf("Error opening file %v: %v", path, err)
		return err
	}

	info, err := file.Stat()

	if err != nil {
		log.Warnf("Error getting file size %v: %v", path, err)
		return err
	}

	options := &RequestOptions{
		RawBody:       file,
		RawBodyLength: info.Size(),
		Headers:       &map[string]string{"Content-Type": "application/octet-stream"},
	}

	fileUrl, err := url.Parse(path)
	if err != nil {
		log.Warnf("Error parsing url %v: %v", path, err)
		return err
	}

	resp, err := deploy.client.Request("PUT", filepath.Join(deploy.apiPath(), "files", fileUrl.Path), options, nil)
	if resp != nil && resp.Response != nil && resp.Body != nil {
		resp.Body.Close()
	}
	if err != nil {
		log.Warnf("Error while uploading %v: %v", path, err)
		return err
	}

	log.Infof("Finished uploading file: %s", path)
	return err
}

// deployDir scans the given directory and deploys the files
// that have changed on Netlify.
func (deploy *Deploy) deployDir(dir string) (*Response, error) {
	return deploy.DeployDirWithGitInfo(dir, "", "")
}

// DeployDirWithGitInfo scans the given directory and deploys the files
// that have changed on Netlify.
//
// This function allows you to supply git information about the deploy
// when it hasn't been set previously be a Continuous Deployment process.
func (deploy *Deploy) DeployDirWithGitInfo(dir, branch, commitRef string) (*Response, error) {
	files := map[string]string{}
	log := deploy.log().WithFields(logrus.Fields{
		"dir":        dir,
		"branch":     branch,
		"commit_ref": commitRef,
	})
	defer log.Infof("Finished deploying directory %s", dir)

	log.Infof("Starting deploy of directory %s", dir)
	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() == false && info.Mode().IsRegular() {
			rel, err := filepath.Rel(dir, path)
			if err != nil {
				return err
			}

			if ignoreFile(rel) {
				return nil
			}

			sha := sha1.New()
			data, err := ioutil.ReadFile(path)

			if err != nil {
				return err
			}

			sha.Write(data)

			files[rel] = hex.EncodeToString(sha.Sum(nil))
		}

		return nil
	})
	if err != nil {
		log.WithError(err).Warn("Failed to walk directory structure")
		return nil, err
	}

	fileOptions := &deployFiles{
		Files:     &files,
		Branch:    branch,
		CommitRef: commitRef,
	}

	if len(files) > MaxFilesForSyncDeploy {
		log.Debugf("More files than sync can deploy %d vs %d", len(files), MaxFilesForSyncDeploy)
		fileOptions.Async = true
	}

	options := &RequestOptions{
		JsonBody: fileOptions,
	}

	log.Debug("Starting to do PUT to origin")
	resp, err := deploy.client.Request("PUT", deploy.apiPath(), options, deploy)
	if err != nil {
		return resp, err
	}

	if len(files) > MaxFilesForSyncDeploy {
		start := time.Now()
		log.Debug("Starting to poll for the deploy to get into ready || prepared state")
		for {
			resp, err := deploy.client.Request("GET", deploy.apiPath(), nil, deploy)
			if err != nil {
				log.WithError(err).Warnf("Error fetching deploy, waiting for 5 seconds before retry: %v", err)
				time.Sleep(5 * time.Second)
			}
			resp.Body.Close()

			log.Debugf("Deploy state: %v\n", deploy.State)
			if deploy.State == "prepared" || deploy.State == "ready" {
				break
			}
			if deploy.State == "error" {
				log.Warnf("deploy is in state error")
				return resp, errors.New("Error: preprocessing deploy failed")
			}
			if start.Add(PreProcessingTimeout).Before(time.Now()) {
				log.Warnf("Deploy timed out waiting for preprocessing")
				return resp, errors.New("Error: preprocessing deploy timed out")
			}
			log.Debug("Waiting for 2 seconds to retry getting deploy")
			time.Sleep(2 * time.Second)
		}
	}

	lookup := map[string]bool{}

	for _, sha := range deploy.Required {
		lookup[sha] = true
	}

	log.Infof("Going to deploy the %d required files", len(lookup))

	// Use a channel as a semaphore to limit # of parallel uploads
	sem := make(chan int, deploy.client.MaxConcurrentUploads)
	var wg sync.WaitGroup

	sharedErr := uploadError{err: nil, mutex: &sync.Mutex{}}
	for path, sha := range files {
		if lookup[sha] == true && err == nil {
			wg.Add(1)
			sem <- 1
			go func(path string) {
				defer func() {
					<-sem
					wg.Done()
				}()
				log.Debugf("Starting to upload %s/%s", path, sha)
				if !sharedErr.Empty() {
					return
				}

				b := backoff.NewExponentialBackOff()
				b.MaxElapsedTime = 2 * time.Minute
				err := backoff.Retry(func() error { return deploy.uploadFile(dir, path, sharedErr) }, b)
				if err != nil {
					sharedErr.Set(err)
				}
			}(path)
		}
	}

	log.Debugf("Waiting for required files to upload")
	wg.Wait()

	if !sharedErr.Empty() {
		return resp, sharedErr.err
	}

	return resp, err
}

// deployZip uploads a Zip file to Netlify and deploys the files
// that have changed.
func (deploy *Deploy) deployZip(zip string) (*Response, error) {
	log := deploy.log().WithFields(logrus.Fields{
		"function": "zip",
		"zip_path": zip,
	})
	log.Infof("Starting to deploy zip file %s", zip)
	zipPath, err := filepath.Abs(zip)
	if err != nil {
		return nil, err
	}

	log.Debugf("Opening zip file at %s", zipPath)
	zipFile, err := os.Open(zipPath)
	if err != nil {
		return nil, err
	}
	defer zipFile.Close()

	info, err := zipFile.Stat()
	if err != nil {
		return nil, err
	}

	log.WithFields(logrus.Fields{
		"name": info.Name(),
		"size": info.Size(),
		"mode": info.Mode(),
	}).Debugf("Opened file %s of %s bytes", info.Name(), info.Size())

	options := &RequestOptions{
		RawBody:       zipFile,
		RawBodyLength: info.Size(),
		Headers:       &map[string]string{"Content-Type": "application/zip"},
	}

	log.Debug("Excuting PUT request for zip file")
	resp, err := deploy.client.Request("PUT", deploy.apiPath(), options, deploy)
	if err != nil {
		log.WithError(err).Warn("Error while uploading zip file")
	}

	log.Info("Finished uploading zip file")
	return resp, err
}

func (deploy *Deploy) WaitForReady(timeout time.Duration) error {
	if deploy.State == "ready" {
		return nil
	}

	if timeout == 0 {
		timeout = defaultTimeout
	}

	timedOut := false
	time.AfterFunc(timeout*time.Second, func() {
		timedOut = true
	})

	done := make(chan error)

	go func() {
		for {
			time.Sleep(1 * time.Second)

			if timedOut {
				done <- errors.New("Timeout while waiting for processing")
				break
			}

			_, err := deploy.Reload()
			if err != nil || (deploy.State == "ready") {
				done <- err
				break
			}
		}
	}()

	err := <-done
	return err
}

func ignoreFile(rel string) bool {
	if strings.HasPrefix(rel, ".") || strings.Contains(rel, "/.") || strings.HasPrefix(rel, "__MACOS") {
		if strings.HasPrefix(rel, ".well-known/") {
			return false
		}
		return true
	}
	return false
}
<commit_msg>update logic for shared error<commit_after>package netlify

import (
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/cenkalti/backoff"
)

const MaxFilesForSyncDeploy = 1000
const PreProcessingTimeout = time.Minute * 5

// Deploy represents a specific deploy of a site
type Deploy struct {
	Id     string `json:"id"`
	SiteId string `json:"site_id"`
	UserId string `json:"user_id"`

	// State of the deploy (uploading/uploaded/processing/ready/error)
	State string `json:"state"`

	// Cause of error if State is "error"
	ErrorMessage string `json:"error_message"`

	// Shas of files that needs to be uploaded before the deploy is ready
	Required []string `json:"required"`

	DeployUrl     string `json:"deploy_url"`
	SiteUrl       string `json:"url"`
	ScreenshotUrl string `json:"screenshot_url"`

	CreatedAt Timestamp `json:"created_at"`
	UpdatedAt Timestamp `json:"updated_at"`

	Branch    string `json:"branch,omitempty"`
	CommitRef string `json:"commit_ref,omitempty"`

	client *Client
	logger *logrus.Entry
}

func (d Deploy) log() *logrus.Entry {
	if d.logger == nil {
		d.logger = d.client.log.WithFields(logrus.Fields{
			"function": "deploy",
			"id":       d.Id,
			"site_id":  d.SiteId,
			"user_id":  d.UserId,
		})
	}

	return d.logger.WithField("state", d.State)
}

// DeploysService is used to access all Deploy related API methods
type DeploysService struct {
	site   *Site
	client *Client
}

type uploadError struct {
	err   error
	mutex *sync.Mutex
}

func (u *uploadError) Set(err error) {
	if err != nil {
		u.mutex.Lock()
		defer u.mutex.Unlock()
		if u.err == nil {
			u.err = err
		}
	}
}

func (u *uploadError) Empty() bool {
	u.mutex.Lock()
	defer u.mutex.Unlock()
	return u.err == nil
}

type deployFiles struct {
	Files     *map[string]string `json:"files"`
	Async     bool               `json:"async"`
	Branch    string             `json:"branch,omitempty"`
	CommitRef string             `json:"commit_ref,omitempty"`
}

func (s *DeploysService) apiPath() string {
	if s.site != nil {
		return path.Join(s.site.apiPath(), "deploys")
	}
	return "/deploys"
}

// Create a new deploy
//
// Example: site.Deploys.Create("/path/to/site-dir", true)
// If the target is a zip file, it must have the extension .zip
func (s *DeploysService) Create(dirOrZip string) (*Deploy, *Response, error) {
	return s.create(dirOrZip, false)
}

// CreateDraft a new draft deploy. Draft deploys will be uploaded and processed, but
// won't affect the active deploy for a site.
func (s *DeploysService) CreateDraft(dirOrZip string) (*Deploy, *Response, error) {
	return s.create(dirOrZip, true)
}

func (s *DeploysService) create(dirOrZip string, draft bool) (*Deploy, *Response, error) {
	if s.site == nil {
		return nil, nil, errors.New("You can only create a new deploy for an existing site (site.Deploys.Create(dirOrZip)))")
	}

	params := url.Values{}
	if draft {
		params["draft"] = []string{"true"}
	}
	options := &RequestOptions{QueryParams: &params}
	deploy := &Deploy{client: s.client}
	resp, err := s.client.Request("POST", s.apiPath(), options, deploy)

	if err != nil {
		return deploy, resp, err
	}

	resp, err = deploy.Deploy(dirOrZip)
	return deploy, resp, err
}

// List all deploys. Takes ListOptions to control pagination.
func (s *DeploysService) List(options *ListOptions) ([]Deploy, *Response, error) {
	deploys := new([]Deploy)

	reqOptions := &RequestOptions{QueryParams: options.toQueryParamsMap()}

	resp, err := s.client.Request("GET", s.apiPath(), reqOptions, deploys)

	for _, deploy := range *deploys {
		deploy.client = s.client
	}

	return *deploys, resp, err
}

// Get a specific deploy.
func (d *DeploysService) Get(id string) (*Deploy, *Response, error) {
	deploy := &Deploy{Id: id, client: d.client}
	resp, err := deploy.Reload()

	return deploy, resp, err
}

func (deploy *Deploy) apiPath() string {
	return path.Join("/deploys", deploy.Id)
}

func (deploy *Deploy) Deploy(dirOrZip string) (*Response, error) {
	if strings.HasSuffix(dirOrZip, ".zip") {
		return deploy.deployZip(dirOrZip)
	} else {
		return deploy.deployDir(dirOrZip)
	}
}

// Reload a deploy from the API
func (deploy *Deploy) Reload() (*Response, error) {
	if deploy.Id == "" {
		return nil, errors.New("Cannot fetch deploy without an ID")
	}
	return deploy.client.Request("GET", deploy.apiPath(), nil, deploy)
}

// Restore an old deploy. Sets the deploy as the active deploy for a site
func (deploy *Deploy) Restore() (*Response, error) {
	return deploy.client.Request("POST", path.Join(deploy.apiPath(), "restore"), nil, deploy)
}

// Alias for restore. Published a specific deploy.
func (deploy *Deploy) Publish() (*Response, error) {
	return deploy.Restore()
}

func (deploy *Deploy) uploadFile(dir, path string, sharedError *uploadError) error {
	if !sharedError.Empty() {
		return errors.New("Canceled because upload has already failed")
	}

	log := deploy.log().WithFields(logrus.Fields{
		"dir":  dir,
		"path": path,
	})

	log.Infof("Uploading file: %v", path)
	file, err := os.Open(filepath.Join(dir, path))
	defer file.Close()

	if err != nil {
		log.Warnf("Error opening file %v: %v", path, err)
		return err
	}

	info, err := file.Stat()

	if err != nil {
		log.Warnf("Error getting file size %v: %v", path, err)
		return err
	}

	options := &RequestOptions{
		RawBody:       file,
		RawBodyLength: info.Size(),
		Headers:       &map[string]string{"Content-Type": "application/octet-stream"},
	}

	fileUrl, err := url.Parse(path)
	if err != nil {
		log.Warnf("Error parsing url %v: %v", path, err)
		return err
	}

	resp, err := deploy.client.Request("PUT", filepath.Join(deploy.apiPath(), "files", fileUrl.Path), options, nil)
	if resp != nil && resp.Response != nil && resp.Body != nil {
		resp.Body.Close()
	}
	if err != nil {
		log.Warnf("Error while uploading %v: %v", path, err)
		return err
	}

	log.Infof("Finished uploading file: %s", path)
	return err
}

// deployDir scans the given directory and deploys the files
// that have changed on Netlify.
func (deploy *Deploy) deployDir(dir string) (*Response, error) {
	return deploy.DeployDirWithGitInfo(dir, "", "")
}

// DeployDirWithGitInfo scans the given directory and deploys the files
// that have changed on Netlify.
//
// This function allows you to supply git information about the deploy
// when it hasn't been set previously be a Continuous Deployment process.
func (deploy *Deploy) DeployDirWithGitInfo(dir, branch, commitRef string) (*Response, error) {
	files := map[string]string{}
	log := deploy.log().WithFields(logrus.Fields{
		"dir":        dir,
		"branch":     branch,
		"commit_ref": commitRef,
	})
	defer log.Infof("Finished deploying directory %s", dir)

	log.Infof("Starting deploy of directory %s", dir)
	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() == false && info.Mode().IsRegular() {
			rel, err := filepath.Rel(dir, path)
			if err != nil {
				return err
			}

			if ignoreFile(rel) {
				return nil
			}

			sha := sha1.New()
			data, err := ioutil.ReadFile(path)

			if err != nil {
				return err
			}

			sha.Write(data)

			files[rel] = hex.EncodeToString(sha.Sum(nil))
		}

		return nil
	})
	if err != nil {
		log.WithError(err).Warn("Failed to walk directory structure")
		return nil, err
	}

	fileOptions := &deployFiles{
		Files:     &files,
		Branch:    branch,
		CommitRef: commitRef,
	}

	if len(files) > MaxFilesForSyncDeploy {
		log.Debugf("More files than sync can deploy %d vs %d", len(files), MaxFilesForSyncDeploy)
		fileOptions.Async = true
	}

	options := &RequestOptions{
		JsonBody: fileOptions,
	}

	log.Debug("Starting to do PUT to origin")
	resp, err := deploy.client.Request("PUT", deploy.apiPath(), options, deploy)
	if err != nil {
		return resp, err
	}

	if len(files) > MaxFilesForSyncDeploy {
		start := time.Now()
		log.Debug("Starting to poll for the deploy to get into ready || prepared state")
		for {
			resp, err := deploy.client.Request("GET", deploy.apiPath(), nil, deploy)
			if err != nil {
				log.WithError(err).Warnf("Error fetching deploy, waiting for 5 seconds before retry: %v", err)
				time.Sleep(5 * time.Second)
			}
			resp.Body.Close()

			log.Debugf("Deploy state: %v\n", deploy.State)
			if deploy.State == "prepared" || deploy.State == "ready" {
				break
			}
			if deploy.State == "error" {
				log.Warnf("deploy is in state error")
				return resp, errors.New("Error: preprocessing deploy failed")
			}
			if start.Add(PreProcessingTimeout).Before(time.Now()) {
				log.Warnf("Deploy timed out waiting for preprocessing")
				return resp, errors.New("Error: preprocessing deploy timed out")
			}
			log.Debug("Waiting for 2 seconds to retry getting deploy")
			time.Sleep(2 * time.Second)
		}
	}

	lookup := map[string]bool{}

	for _, sha := range deploy.Required {
		lookup[sha] = true
	}

	log.Infof("Going to deploy the %d required files", len(lookup))

	// Use a channel as a semaphore to limit # of parallel uploads
	sem := make(chan int, deploy.client.MaxConcurrentUploads)
	var wg sync.WaitGroup

	sharedErr := uploadError{err: nil, mutex: &sync.Mutex{}}
	for path, sha := range files {
		if lookup[sha] == true && sharedErr.Empty() {
			wg.Add(1)
			sem <- 1
			go func(path string) {
				defer func() {
					<-sem
					wg.Done()
				}()
				log.Debugf("Starting to upload %s/%s", path, sha)
				if !sharedErr.Empty() {
					return
				}

				b := backoff.NewExponentialBackOff()
				b.MaxElapsedTime = 2 * time.Minute
				err := backoff.Retry(func() error { return deploy.uploadFile(dir, path, &sharedErr) }, b)
				if err != nil {
					log.WithError(err).Warn("Error while uploading file %s: %v", path, err)
					sharedErr.Set(err)
				}
			}(path)
		}
	}

	log.Debugf("Waiting for required files to upload")
	wg.Wait()

	if !sharedErr.Empty() {
		return resp, sharedErr.err
	}

	return resp, err
}

// deployZip uploads a Zip file to Netlify and deploys the files
// that have changed.
func (deploy *Deploy) deployZip(zip string) (*Response, error) {
	log := deploy.log().WithFields(logrus.Fields{
		"function": "zip",
		"zip_path": zip,
	})
	log.Infof("Starting to deploy zip file %s", zip)
	zipPath, err := filepath.Abs(zip)
	if err != nil {
		return nil, err
	}

	log.Debugf("Opening zip file at %s", zipPath)
	zipFile, err := os.Open(zipPath)
	if err != nil {
		return nil, err
	}
	defer zipFile.Close()

	info, err := zipFile.Stat()
	if err != nil {
		return nil, err
	}

	log.WithFields(logrus.Fields{
		"name": info.Name(),
		"size": info.Size(),
		"mode": info.Mode(),
	}).Debugf("Opened file %s of %s bytes", info.Name(), info.Size())

	options := &RequestOptions{
		RawBody:       zipFile,
		RawBodyLength: info.Size(),
		Headers:       &map[string]string{"Content-Type": "application/zip"},
	}

	log.Debug("Excuting PUT request for zip file")
	resp, err := deploy.client.Request("PUT", deploy.apiPath(), options, deploy)
	if err != nil {
		log.WithError(err).Warn("Error while uploading zip file")
	}

	log.Info("Finished uploading zip file")
	return resp, err
}

func (deploy *Deploy) WaitForReady(timeout time.Duration) error {
	if deploy.State == "ready" {
		return nil
	}

	if timeout == 0 {
		timeout = defaultTimeout
	}

	timedOut := false
	time.AfterFunc(timeout*time.Second, func() {
		timedOut = true
	})

	done := make(chan error)

	go func() {
		for {
			time.Sleep(1 * time.Second)

			if timedOut {
				done <- errors.New("Timeout while waiting for processing")
				break
			}

			_, err := deploy.Reload()
			if err != nil || (deploy.State == "ready") {
				done <- err
				break
			}
		}
	}()

	err := <-done
	return err
}

func ignoreFile(rel string) bool {
	if strings.HasPrefix(rel, ".") || strings.Contains(rel, "/.") || strings.HasPrefix(rel, "__MACOS") {
		if strings.HasPrefix(rel, ".well-known/") {
			return false
		}
		return true
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2022 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package fetch

import "golang.org/x/mod/module"

// knownAlternatives lists module paths that are known to be forks of other
// modules.
// For example, github.com/msopentech/azure-sdk-for-go
// is an alternative to github.com/Azure/azure-sdk-for-go.
// Map keys are case-sensitive and should not include a final major version
// like "/v3" or ".v3" for gopkg.in paths.
//
// When a module has a go.mod file, we can detect alternatives by comparing the
// module path with the path in the go.mod file. This list is for modules
// without go.mod files.
var knownAlternatives = map[string]string{
	"github.com/msopentech/azure-sdk-for-go":       "github.com/Azure/azure-sdk-for-go",
	"github.com/MSOpenTech/azure-sdk-for-go":       "github.com/Azure/azure-sdk-for-go",
	"gopkg.in/Azure/azure-sdk-for-go":              "github.com/Azure/azure-sdk-for-go",
	"gopkg.in/azure/azure-sdk-for-go":              "github.com/Azure/azure-sdk-for-go",
	"github.com/masslessparticle/azure-sdk-for-go": "github.com/Azure/azure-sdk-for-go",
	"github.com/aliyun/alibaba-cloud-sdk-go":       "github.com/Azure/azure-sdk-for-go",
	"github.com/johnstairs/azure-sdk-for-go	": "github.com/Azure/azure-sdk-for-go",
	"github.com/shopify/sarama": "github.com/Shopify/sarama",
}

// knownAlternativeFor returns the module that the given module path is an alternative to,
// or the empty string if there is no such module.
//
// It consults the knownAlternatives map, ignoring version suffixes.
func knownAlternativeFor(modulePath string) string {
	key, _, ok := module.SplitPathVersion(modulePath)
	if !ok {
		return ""
	}
	return knownAlternatives[key]
}
<commit_msg>internal/fetch: add more known alternative modules<commit_after>// Copyright 2022 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package fetch

import "golang.org/x/mod/module"

// knownAlternatives lists module paths that are known to be forks of other
// modules.
// For example, github.com/msopentech/azure-sdk-for-go
// is an alternative to github.com/Azure/azure-sdk-for-go.
// Map keys are case-sensitive and should not include a final major version
// like "/v3" or ".v3" for gopkg.in paths.
//
// When a module has a go.mod file, we can detect alternatives by comparing the
// module path with the path in the go.mod file. This list is for modules
// without go.mod files.
var knownAlternatives = map[string]string{
	"github.com/Azure/Azure-sdk-for-go":                "github.com/Azure/azure-sdk-for-go",
	"github.com/azure/azure-sdk-for-go":                "github.com/Azure/azure-sdk-for-go",
	"github.com/evenh/azure-sdk-for-go":                "github.com/Azure/azure-sdk-for-go",
	"github.com/msopentech/azure-sdk-for-go":           "github.com/Azure/azure-sdk-for-go",
	"github.com/MSOpenTech/azure-sdk-for-go":           "github.com/Azure/azure-sdk-for-go",
	"github.com/scott-the-programmer/azure-sdk-for-go": "github.com/Azure/azure-sdk-for-go",
	"gopkg.in/Azure/azure-sdk-for-go":                  "github.com/Azure/azure-sdk-for-go",
	"gopkg.in/azure/azure-sdk-for-go":                  "github.com/Azure/azure-sdk-for-go",
	"github.com/masslessparticle/azure-sdk-for-go":     "github.com/Azure/azure-sdk-for-go",
	"github.com/aliyun/alibaba-cloud-sdk-go":           "github.com/Azure/azure-sdk-for-go",
	"github.com/johnstairs/azure-sdk-for-go	": "github.com/Azure/azure-sdk-for-go",
	"github.com/shopify/sarama": "github.com/Shopify/sarama",
}

// knownAlternativeFor returns the module that the given module path is an alternative to,
// or the empty string if there is no such module.
//
// It consults the knownAlternatives map, ignoring version suffixes.
func knownAlternativeFor(modulePath string) string {
	key, _, ok := module.SplitPathVersion(modulePath)
	if !ok {
		return ""
	}
	return knownAlternatives[key]
}
<|endoftext|>


<commit_before><commit_msg>initial commit<commit_after><|endoftext|>


<commit_before>// Copyright 2013 Travis Keep. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or
// at http://opensource.org/licenses/BSD-3-Clause.

// Package gohue controls hue lights.
package gohue

import (
  "bytes"
  "encoding/json"
  "fmt"
  "github.com/keep94/gohue/json_structs"
  "github.com/keep94/tasks"
  "io"
  "net/http"
  "net/url"
  "time"
)

const (
  // Bright represents the brightest setting
  Bright = uint8(255)

  // Dim represents the dimmest setting.
  Dim = uint8(0)
)

var (
  // Pointer to true
  TruePtr = &trueVal

  // Pointer to false
  FalsePtr = &falseVal
)

var (
  Red = NewColor(0.675, 0.322, Bright)
  Green = NewColor(0.4077, 0.5154, Bright)
  Blue = NewColor(0.167, 0.04, Bright)
  Yellow = Red.Blend(Green, 0.5)
  Magenta = Blue.Blend(Red, 0.5)
  Cyan = Blue.Blend(Green, 0.5)
  Purple = NewColor(0.2522, 0.0882, Bright)
  White = NewColor(0.3848, 0.3629, Bright)
  Pink = NewColor(0.55, 0.3394, Bright)
  Orange = Red.Blend(Yellow, 0.5)
)

var (
  trueVal = true
  falseVal = false
)

const (
  maxu16 = float64(10000.0)
)

// Color represents a particular color and brightness. Programs using Colors
// should typically store and pass them as values, not pointers.
type Color struct {
  x uint16
  y uint16
  bri uint8
}

// NewColor returns a new Color. x and y are the coordinates of the color
// in the color XY space; brightness is the brightness where 255 is brightest
// and 0 is dimmest.
func NewColor(x, y float64, brightness uint8) Color {
  return Color{x: uint16(x * maxu16 + 0.5), y: uint16(y * maxu16 + 0.5), bri: brightness}
}

// NewColorPtr works like NewColor but returns a pointer.
func NewColorPtr(x, y float64, brightness uint8) *Color {
  return &Color{x: uint16(x * maxu16 + 0.5), y: uint16(y * maxu16 + 0.5), bri: brightness}
}

func (c Color) String() string {
  return fmt.Sprintf("(%.4f, %.4f) Bri: %d", c.X(), c.Y(), c.bri)
}

// X returns the X value of this Color.
func (c Color) X() float64 {
  return float64(c.x) / maxu16
}

// Y returns the Y value of this Color.
func (c Color) Y() float64 {
  return float64(c.y) / maxu16
}

// Brightness returns the brightness of this color.
func (c Color) Brightness() uint8 {
  return c.bri
}

// WithBrightness returns a color like this one with specified brightness.
func (c Color) WithBrightness(bri uint8) Color {
  return Color{x: c.x, y: c.y, bri: bri}
}

// Blend blends this color with another color returning the blended Color.
// ratio=0 means use only this color; ratio=1 means use only the other color.
func (c Color) Blend(other Color, ratio float64) Color {
  invratio := 1.0 - ratio
  return NewColor(
      c.X() * invratio + other.X() * ratio,
      c.Y() * invratio + other.Y() * ratio,
      uint8(float64(c.bri) * invratio + float64(other.bri) * ratio))
}

// LightProperies represents the properties of a light.
type LightProperties struct {
  // C is the Color. nil means leave the color as-is
  C *Color

  // On is true if light is on or false if it is off. If nil,
  // it means leave the on/off state as is.
  On *bool
}

// Context represents a connection with a hue bridge.
type Context struct {

  // The userId / developer Id. See hue documentation.
  UserId string

  // The private ip address of the hue bridge.
  IpAddress string
}

// Set sets the properties of a light. lightId is the ID of the light to set.
// 0 means all lights.
func (c *Context) Set(lightId int, properties *LightProperties) (response []byte, err error) {
  jsonMap := make(map[string]interface{})
  if properties.C != nil {
    jsonMap["bri"] = properties.C.Brightness()
    jsonMap["xy"] = []float64{properties.C.X(), properties.C.Y()}
  }
  if properties.On != nil {
    jsonMap["on"] = *properties.On
  }
  var reqBuffer []byte
  if reqBuffer, err = json.Marshal(jsonMap); err != nil {
    return
  }
  var url *url.URL
  if url, err = c.lightUrl(lightId); err != nil {
    return
  }
  request := &http.Request{
      Method: "PUT",
      URL: url,
      ContentLength: int64(len(reqBuffer)),
      Body: simpleReadCloser{bytes.NewReader(reqBuffer)},
  }
  var client http.Client
  var resp *http.Response
  if resp, err = client.Do(request); err != nil {
    return
  }
  defer resp.Body.Close()
  var respBuffer bytes.Buffer
  if _, err = respBuffer.ReadFrom(resp.Body); err != nil {
    return
  }
  return respBuffer.Bytes(), nil
}

// Get gets the properties of a light. lightId is the ID of the light.
func (c *Context) Get(lightId int) (properties *LightProperties, err error) {
  var url *url.URL
  if url, err = c.getLightUrl(lightId); err != nil {
    return
  }
  request := &http.Request{
      Method: "GET",
      URL: url,
  }
  var client http.Client
  var resp *http.Response
  if resp, err = client.Do(request); err != nil {
    return
  }
  defer resp.Body.Close()
  jsonDecoder := json.NewDecoder(resp.Body)
  var jsonProps json_structs.LightState
  if err = jsonDecoder.Decode(&jsonProps); err != nil {
    return
  }
  if jsonProps.State != nil && len(jsonProps.State.XY) == 2 {
    state := jsonProps.State
    var on *bool
    if state.On {
      on = TruePtr
    } else {
      on = FalsePtr
    }
    jsonColor := state.XY
    properties = &LightProperties{
        C: NewColorPtr(jsonColor[0], jsonColor[1], state.Bri),
        On: on}
  }
  return
}

func (c *Context) getLightUrl(id int) (*url.URL, error) {
  return url.Parse(fmt.Sprintf("http://%s/api/%s/lights/%d", c.IpAddress, c.UserId, id))
}

func (c *Context) lightUrl(id int) (*url.URL, error) {
  if id == 0 {
    return c.allUrl()
  }
  return url.Parse(fmt.Sprintf("http://%s/api/%s/lights/%d/state", c.IpAddress, c.UserId, id))
}

func (c *Context) allUrl() (*url.URL, error) {
  return url.Parse(fmt.Sprintf("http://%s/api/%s/groups/0/action", c.IpAddress, c.UserId))
}

// ColorDuration specifies the color a light should be a certain duration
// into a gradient.
type ColorDuration struct {

  // The color the light should be.
  C Color

  // The Duration into the gradient.
  D time.Duration
}

// Interface Setter sets the properties of a light. lightId is the ID of the
// light to set. 0 means all lights.
type Setter interface {
  Set(lightId int, properties *LightProperties) (response []byte, err error)
}

// Gradient represents a change in colors over time.
type Gradient struct {

  // The desired color at certain durations into the gradient. The specified
  // durations must be in increasing order.
  Cds []ColorDuration

  // Light color is refreshed this often.
  Refresh time.Duration
}

// Action represents some action to the lights.
// Callers should set exactly one of the
// Parallel, Series, G, C, or Sleep fields. The other fields compliment
// these fields.
type Action struct {

  // The light bulb ids. Empty means the default set of lights. For child
  // actions, the default set of lights is the parent's set of lights;
  // for top-level actions, the default set of lights are the light bulb
  // ids passed to AsTask. Lights applies to Parallel, Series, G, or C fields.
  Lights []int

  // Repeat this many times. 0 or negative means do once.
  // May be used with Parallel, Series, G, C, or Sleep fields.
  Repeat int

  // The Gradient
  G *Gradient

  // The single color
  C *Color

  // If true, light is turned on. May be used with G or C fields.
  On bool

  // If true, light is turned off. May be used with G or C fields.
  Off bool

  // Sleep sleeps this duration
  Sleep time.Duration

  // Actions to be done in series
  Series []*Action

  // Actions to be done in parallel
  Parallel []*Action
}

// AsTask returns a Task from this instance. setter is what changes the
// lightbulb. lights is the default set of lights empty means all lights.
func (a *Action) AsTask(setter Setter, lights []int) tasks.Task {
  if a.Repeat < 2 {
    return a.asTask(setter, lights)
  }
  return tasks.RepeatingTask(a.asTask(setter, lights), a.Repeat)
}

func (a *Action) asTask(setter Setter, lights []int) tasks.Task {
  if len(a.Lights) > 0 {
    lights = a.Lights
  }
  if len(a.Parallel) > 0 {
    parallelTasks := make([]tasks.Task, len(a.Parallel))
    for i := range parallelTasks {
      parallelTasks[i] = a.Parallel[i].AsTask(setter, lights)
    }
    return tasks.ParallelTasks(parallelTasks...)
  }
  if len(a.Series) > 0 {
    seriesTasks := make([]tasks.Task, len(a.Series))
    for i := range seriesTasks {
      seriesTasks[i] = a.Series[i].AsTask(setter, lights)
    }
    return tasks.SeriesTasks(seriesTasks...)
  }
  if a.G != nil {
    if len(a.G.Cds) == 0 || a.G.Cds[0].D != 0 {
      panic("D of first ColorDuration element must be 0.")
    }
    return tasks.TaskFunc(func(e *tasks.Execution) {
      a.doGradient(setter, lights, e)
    })
  }
  if a.C != nil || a.On || a.Off {
    return tasks.TaskFunc(func(e *tasks.Execution) {
      a.doOnOff(setter, lights, e)
    })
  }
  return tasks.TaskFunc(func(e *tasks.Execution) {
    e.Sleep(a.Sleep)
  })
}

func (a *Action) doOnOff(setter Setter, lights []int, e *tasks.Execution) {
  var properties LightProperties
  if a.On {
    properties.On = TruePtr
  } else if a.Off {
    properties.On = FalsePtr
  }
  properties.C = a.C
  multiSet(e, setter, lights, &properties)
}

func (a *Action) doGradient(setter Setter, lights []int, e *tasks.Execution) {
  startTime := e.Now()
  var currentD time.Duration
  var properties LightProperties
  if a.On {
    properties.On = TruePtr
  }
  idx := 1
  for idx < len(a.G.Cds) {
    if currentD > a.G.Cds[idx].D {
      idx++
      continue
    }
    acolor := a.G.Cds[idx - 1].C.Blend(
        a.G.Cds[idx].C,
        float64(currentD - a.G.Cds[idx - 1].D) / float64(a.G.Cds[idx].D - a.G.Cds[idx - 1].D))
    properties.C = &acolor
    multiSet(e, setter, lights, &properties)
    properties.On = nil

    // If we have already reached the end of the gradient, just return
    // immediately.
    if currentD == a.G.Cds[len(a.G.Cds) - 1].D {
      return
    }
    if e.Error() != nil {
      return
    }
    if !e.Sleep(a.G.Refresh) {
      return
    }
    currentD = e.Now().Sub(startTime)
  }
  properties.C = &a.G.Cds[len(a.G.Cds) - 1].C
  multiSet(e, setter, lights, &properties)
}

type simpleReadCloser struct {
  io.Reader
}

func (s simpleReadCloser) Close() error {
  return nil
}

func multiSet(
    e *tasks.Execution,
    setter Setter,
    lights []int,
    properties *LightProperties) {
  if len(lights) == 0 {
    if _, err := setter.Set(0, properties); err != nil {
      e.SetError(err)
      return
    }
  } else {
    for _, light := range lights {
      if _, err := setter.Set(light, properties); err != nil {
        e.SetError(err)
        return
      }
    }
  }
}
<commit_msg>Improve documentation on Action type.<commit_after>// Copyright 2013 Travis Keep. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or
// at http://opensource.org/licenses/BSD-3-Clause.

// Package gohue controls hue lights.
package gohue

import (
  "bytes"
  "encoding/json"
  "fmt"
  "github.com/keep94/gohue/json_structs"
  "github.com/keep94/tasks"
  "io"
  "net/http"
  "net/url"
  "time"
)

const (
  // Bright represents the brightest setting
  Bright = uint8(255)

  // Dim represents the dimmest setting.
  Dim = uint8(0)
)

var (
  // Pointer to true
  TruePtr = &trueVal

  // Pointer to false
  FalsePtr = &falseVal
)

var (
  Red = NewColor(0.675, 0.322, Bright)
  Green = NewColor(0.4077, 0.5154, Bright)
  Blue = NewColor(0.167, 0.04, Bright)
  Yellow = Red.Blend(Green, 0.5)
  Magenta = Blue.Blend(Red, 0.5)
  Cyan = Blue.Blend(Green, 0.5)
  Purple = NewColor(0.2522, 0.0882, Bright)
  White = NewColor(0.3848, 0.3629, Bright)
  Pink = NewColor(0.55, 0.3394, Bright)
  Orange = Red.Blend(Yellow, 0.5)
)

var (
  trueVal = true
  falseVal = false
)

const (
  maxu16 = float64(10000.0)
)

// Color represents a particular color and brightness. Programs using Colors
// should typically store and pass them as values, not pointers.
type Color struct {
  x uint16
  y uint16
  bri uint8
}

// NewColor returns a new Color. x and y are the coordinates of the color
// in the color XY space; brightness is the brightness where 255 is brightest
// and 0 is dimmest.
func NewColor(x, y float64, brightness uint8) Color {
  return Color{x: uint16(x * maxu16 + 0.5), y: uint16(y * maxu16 + 0.5), bri: brightness}
}

// NewColorPtr works like NewColor but returns a pointer.
func NewColorPtr(x, y float64, brightness uint8) *Color {
  return &Color{x: uint16(x * maxu16 + 0.5), y: uint16(y * maxu16 + 0.5), bri: brightness}
}

func (c Color) String() string {
  return fmt.Sprintf("(%.4f, %.4f) Bri: %d", c.X(), c.Y(), c.bri)
}

// X returns the X value of this Color.
func (c Color) X() float64 {
  return float64(c.x) / maxu16
}

// Y returns the Y value of this Color.
func (c Color) Y() float64 {
  return float64(c.y) / maxu16
}

// Brightness returns the brightness of this color.
func (c Color) Brightness() uint8 {
  return c.bri
}

// WithBrightness returns a color like this one with specified brightness.
func (c Color) WithBrightness(bri uint8) Color {
  return Color{x: c.x, y: c.y, bri: bri}
}

// Blend blends this color with another color returning the blended Color.
// ratio=0 means use only this color; ratio=1 means use only the other color.
func (c Color) Blend(other Color, ratio float64) Color {
  invratio := 1.0 - ratio
  return NewColor(
      c.X() * invratio + other.X() * ratio,
      c.Y() * invratio + other.Y() * ratio,
      uint8(float64(c.bri) * invratio + float64(other.bri) * ratio))
}

// LightProperies represents the properties of a light.
type LightProperties struct {
  // C is the Color. nil means leave the color as-is
  C *Color

  // On is true if light is on or false if it is off. If nil,
  // it means leave the on/off state as is.
  On *bool
}

// Context represents a connection with a hue bridge.
type Context struct {

  // The userId / developer Id. See hue documentation.
  UserId string

  // The private ip address of the hue bridge.
  IpAddress string
}

// Set sets the properties of a light. lightId is the ID of the light to set.
// 0 means all lights.
func (c *Context) Set(lightId int, properties *LightProperties) (response []byte, err error) {
  jsonMap := make(map[string]interface{})
  if properties.C != nil {
    jsonMap["bri"] = properties.C.Brightness()
    jsonMap["xy"] = []float64{properties.C.X(), properties.C.Y()}
  }
  if properties.On != nil {
    jsonMap["on"] = *properties.On
  }
  var reqBuffer []byte
  if reqBuffer, err = json.Marshal(jsonMap); err != nil {
    return
  }
  var url *url.URL
  if url, err = c.lightUrl(lightId); err != nil {
    return
  }
  request := &http.Request{
      Method: "PUT",
      URL: url,
      ContentLength: int64(len(reqBuffer)),
      Body: simpleReadCloser{bytes.NewReader(reqBuffer)},
  }
  var client http.Client
  var resp *http.Response
  if resp, err = client.Do(request); err != nil {
    return
  }
  defer resp.Body.Close()
  var respBuffer bytes.Buffer
  if _, err = respBuffer.ReadFrom(resp.Body); err != nil {
    return
  }
  return respBuffer.Bytes(), nil
}

// Get gets the properties of a light. lightId is the ID of the light.
func (c *Context) Get(lightId int) (properties *LightProperties, err error) {
  var url *url.URL
  if url, err = c.getLightUrl(lightId); err != nil {
    return
  }
  request := &http.Request{
      Method: "GET",
      URL: url,
  }
  var client http.Client
  var resp *http.Response
  if resp, err = client.Do(request); err != nil {
    return
  }
  defer resp.Body.Close()
  jsonDecoder := json.NewDecoder(resp.Body)
  var jsonProps json_structs.LightState
  if err = jsonDecoder.Decode(&jsonProps); err != nil {
    return
  }
  if jsonProps.State != nil && len(jsonProps.State.XY) == 2 {
    state := jsonProps.State
    var on *bool
    if state.On {
      on = TruePtr
    } else {
      on = FalsePtr
    }
    jsonColor := state.XY
    properties = &LightProperties{
        C: NewColorPtr(jsonColor[0], jsonColor[1], state.Bri),
        On: on}
  }
  return
}

func (c *Context) getLightUrl(id int) (*url.URL, error) {
  return url.Parse(fmt.Sprintf("http://%s/api/%s/lights/%d", c.IpAddress, c.UserId, id))
}

func (c *Context) lightUrl(id int) (*url.URL, error) {
  if id == 0 {
    return c.allUrl()
  }
  return url.Parse(fmt.Sprintf("http://%s/api/%s/lights/%d/state", c.IpAddress, c.UserId, id))
}

func (c *Context) allUrl() (*url.URL, error) {
  return url.Parse(fmt.Sprintf("http://%s/api/%s/groups/0/action", c.IpAddress, c.UserId))
}

// ColorDuration specifies the color a light should be a certain duration
// into a gradient.
type ColorDuration struct {

  // The color the light should be.
  C Color

  // The Duration into the gradient.
  D time.Duration
}

// Interface Setter sets the properties of a light. lightId is the ID of the
// light to set. 0 means all lights.
type Setter interface {
  Set(lightId int, properties *LightProperties) (response []byte, err error)
}

// Gradient represents a change in colors over time.
type Gradient struct {

  // The desired color at certain durations into the gradient. The specified
  // durations must be in increasing order.
  Cds []ColorDuration

  // Light color is refreshed this often.
  Refresh time.Duration
}

// Action represents some action to the lights.
// Callers should set exactly one of the
// Parallel, Series, G, C, On, Off, or Sleep fields. The one exception is that
// On can be used with G or C. The other fields compliment these fields.
type Action struct {

  // The light bulb ids. Empty means the default set of lights. For child
  // actions, the default set of lights is the parent's set of lights;
  // for top-level actions, the default set of lights are the light bulb
  // ids passed to AsTask.
  Lights []int

  // Repeat this many times. 0 or negative means do once.
  Repeat int

  // The Gradient
  G *Gradient

  // The single color
  C *Color

  // If true, light(s) are turned on. May be used along with G or C fields to
  // ensure light(s) are on.
  On bool

  // If true, light(s) are turned off.
  Off bool

  // Sleep sleeps this duration
  Sleep time.Duration

  // Actions to be done in series
  Series []*Action

  // Actions to be done in parallel
  Parallel []*Action
}

// AsTask returns a Task from this instance. setter is what changes the
// lightbulb. lights is the default set of lights empty means all lights.
func (a *Action) AsTask(setter Setter, lights []int) tasks.Task {
  if a.Repeat < 2 {
    return a.asTask(setter, lights)
  }
  return tasks.RepeatingTask(a.asTask(setter, lights), a.Repeat)
}

func (a *Action) asTask(setter Setter, lights []int) tasks.Task {
  if len(a.Lights) > 0 {
    lights = a.Lights
  }
  if len(a.Parallel) > 0 {
    parallelTasks := make([]tasks.Task, len(a.Parallel))
    for i := range parallelTasks {
      parallelTasks[i] = a.Parallel[i].AsTask(setter, lights)
    }
    return tasks.ParallelTasks(parallelTasks...)
  }
  if len(a.Series) > 0 {
    seriesTasks := make([]tasks.Task, len(a.Series))
    for i := range seriesTasks {
      seriesTasks[i] = a.Series[i].AsTask(setter, lights)
    }
    return tasks.SeriesTasks(seriesTasks...)
  }
  if a.G != nil {
    if len(a.G.Cds) == 0 || a.G.Cds[0].D != 0 {
      panic("D of first ColorDuration element must be 0.")
    }
    return tasks.TaskFunc(func(e *tasks.Execution) {
      a.doGradient(setter, lights, e)
    })
  }
  if a.C != nil || a.On || a.Off {
    return tasks.TaskFunc(func(e *tasks.Execution) {
      a.doOnOff(setter, lights, e)
    })
  }
  return tasks.TaskFunc(func(e *tasks.Execution) {
    e.Sleep(a.Sleep)
  })
}

func (a *Action) doOnOff(setter Setter, lights []int, e *tasks.Execution) {
  var properties LightProperties
  if a.On {
    properties.On = TruePtr
  } else if a.Off {
    properties.On = FalsePtr
  }
  properties.C = a.C
  multiSet(e, setter, lights, &properties)
}

func (a *Action) doGradient(setter Setter, lights []int, e *tasks.Execution) {
  startTime := e.Now()
  var currentD time.Duration
  var properties LightProperties
  if a.On {
    properties.On = TruePtr
  }
  idx := 1
  for idx < len(a.G.Cds) {
    if currentD > a.G.Cds[idx].D {
      idx++
      continue
    }
    acolor := a.G.Cds[idx - 1].C.Blend(
        a.G.Cds[idx].C,
        float64(currentD - a.G.Cds[idx - 1].D) / float64(a.G.Cds[idx].D - a.G.Cds[idx - 1].D))
    properties.C = &acolor
    multiSet(e, setter, lights, &properties)
    properties.On = nil

    // If we have already reached the end of the gradient, just return
    // immediately.
    if currentD == a.G.Cds[len(a.G.Cds) - 1].D {
      return
    }
    if e.Error() != nil {
      return
    }
    if !e.Sleep(a.G.Refresh) {
      return
    }
    currentD = e.Now().Sub(startTime)
  }
  properties.C = &a.G.Cds[len(a.G.Cds) - 1].C
  multiSet(e, setter, lights, &properties)
}

type simpleReadCloser struct {
  io.Reader
}

func (s simpleReadCloser) Close() error {
  return nil
}

func multiSet(
    e *tasks.Execution,
    setter Setter,
    lights []int,
    properties *LightProperties) {
  if len(lights) == 0 {
    if _, err := setter.Set(0, properties); err != nil {
      e.SetError(err)
      return
    }
  } else {
    for _, light := range lights {
      if _, err := setter.Set(light, properties); err != nil {
        e.SetError(err)
        return
      }
    }
  }
}
<|endoftext|>


<commit_before>package connection_test

import (
	"net"
	"testing"

	"github.com/cenkalti/rain/internal/connection"
	"github.com/cenkalti/rain/internal/protocol"
)

var addr = &net.TCPAddr{
	IP:   net.IPv4(127, 0, 0, 1),
	Port: 5000,
}

var (
	ext1     = [8]byte{0x0A}
	ext2     = [8]byte{0x0B}
	id1      = protocol.PeerID{0x0C}
	id2      = protocol.PeerID{0x0D}
	infoHash = protocol.InfoHash{0x0E}
)

func TestUnencrypted(t *testing.T) {
	l, err := net.ListenTCP("tcp", addr)
	if err != nil {
		t.Fatal(err)
	}
	done := make(chan struct{})
	go func() {
		conn, cipher, ext, id, err := connection.Dial(addr, false, false, ext1, infoHash, id1)
		if err != nil {
			t.Fatal(err)
		}
		if conn == nil {
			t.Errorf("conn: %s", conn)
		}
		if cipher != 0 {
			t.Errorf("cipher: %d", cipher)
		}
		if ext != ext2 {
			t.Errorf("ext: %s", ext)
		}
		if id != id2 {
			t.Errorf("id: %s", id)
		}
		close(done)
	}()
	conn, err := l.Accept()
	if err != nil {
		t.Fatal(err)
	}
	cipher, ext, ih, id, err := connection.Accept(conn, false, nil, func(ih protocol.InfoHash) bool { return ih == infoHash }, ext2, id2)
	<-done
	if err != nil {
		t.Fatal(err)
	}
	if cipher != 0 {
		t.Errorf("cipher: %d", cipher)
	}
	if ext != ext1 {
		t.Errorf("ext: %s", ext)
	}
	if ih != infoHash {
		t.Errorf("ih: %s", ih)
	}
	if id != id1 {
		t.Errorf("id: %s", id)
	}
}
<commit_msg>add connection encryption test<commit_after>package connection_test

import (
	"net"
	"testing"

	"github.com/cenkalti/mse"

	"github.com/cenkalti/rain/internal/connection"
	"github.com/cenkalti/rain/internal/protocol"
)

var addr = &net.TCPAddr{
	IP:   net.IPv4(127, 0, 0, 1),
	Port: 5000,
}

var (
	ext1     = [8]byte{0x0A}
	ext2     = [8]byte{0x0B}
	id1      = protocol.PeerID{0x0C}
	id2      = protocol.PeerID{0x0D}
	infoHash = protocol.InfoHash{0x0E}
	sKeyHash = mse.HashSKey(infoHash[:])
)

func TestUnencrypted(t *testing.T) {
	l, err := net.ListenTCP("tcp", addr)
	if err != nil {
		t.Fatal(err)
	}
	defer l.Close()
	done := make(chan struct{})
	go func() {
		conn, cipher, ext, id, err := connection.Dial(addr, false, false, ext1, infoHash, id1)
		if err != nil {
			t.Fatal(err)
		}
		if conn == nil {
			t.Errorf("conn: %s", conn)
		}
		if cipher != 0 {
			t.Errorf("cipher: %d", cipher)
		}
		if ext != ext2 {
			t.Errorf("ext: %s", ext)
		}
		if id != id2 {
			t.Errorf("id: %s", id)
		}
		close(done)
	}()
	conn, err := l.Accept()
	if err != nil {
		t.Fatal(err)
	}
	cipher, ext, ih, id, err := connection.Accept(conn, false, nil, func(ih protocol.InfoHash) bool { return ih == infoHash }, ext2, id2)
	<-done
	if err != nil {
		t.Fatal(err)
	}
	if cipher != 0 {
		t.Errorf("cipher: %d", cipher)
	}
	if ext != ext1 {
		t.Errorf("ext: %s", ext)
	}
	if ih != infoHash {
		t.Errorf("ih: %s", ih)
	}
	if id != id1 {
		t.Errorf("id: %s", id)
	}
}

func TestEncrypted(t *testing.T) {
	l, err := net.ListenTCP("tcp", addr)
	if err != nil {
		t.Fatal(err)
	}
	defer l.Close()
	done := make(chan struct{})
	go func() {
		conn, cipher, ext, id, err := connection.Dial(addr, true, false, ext1, infoHash, id1)
		if err != nil {
			t.Fatal(err)
		}
		if conn == nil {
			t.Errorf("conn: %s", conn)
		}
		if cipher != mse.RC4 {
			t.Errorf("cipher: %d", cipher)
		}
		if ext != ext2 {
			t.Errorf("ext: %s", ext)
		}
		if id != id2 {
			t.Errorf("id: %s", id)
		}
		close(done)
	}()
	conn, err := l.Accept()
	if err != nil {
		t.Fatal(err)
	}
	cipher, ext, ih, id, err := connection.Accept(
		conn, false,
		func(h [20]byte) (sKey []byte) {
			if h == sKeyHash {
				return infoHash[:]
			}
			return nil
		},
		func(ih protocol.InfoHash) bool { return ih == infoHash },
		ext2, id2)
	<-done
	if err != nil {
		t.Fatal(err)
	}
	if cipher != mse.RC4 {
		t.Errorf("cipher: %d", cipher)
	}
	if ext != ext1 {
		t.Errorf("ext: %s", ext)
	}
	if ih != infoHash {
		t.Errorf("ih: %s", ih)
	}
	if id != id1 {
		t.Errorf("id: %s", id)
	}
}
<|endoftext|>


<commit_before>package gogen

import (
	"github.com/sirkon/ldetool/internal/generator/gogen/internal/srcobj"
)

/* decode_int8 sample - other templates are the same
tmp ={{ .Source }};
if tmpInt, err = strconv.ParseInt(*(*string)(unsafe.Pointer(&tmp)), 10, {{ .Bits }}); err != nil {
    return false, fmt.Errorf("Error parsing as `{{ Type }}` for field {{ .Dest }}: %s", err)
}
p.{{Dest}} = int8(tmpInt)
*/

func (g *Generator) prepNumeric() {
	g.regImport("", "strconv")
	g.regImport("", "fmt")
	g.regVar("err", "error")
}

func (g *Generator) prepInt() {
	g.prepNumeric()
	g.regVar("tmpInt", "int64")
}

func (g *Generator) prepUint() {
	g.prepNumeric()
	g.regVar("tmpUint", "uint64")
}

func (g *Generator) prepFloat() {
	g.prepNumeric()
	g.regVar("tmpFloat", "float64")
}

func (g *Generator) decode(src srcobj.Source, tmp, gotype, dest, decoder string, params ...srcobj.Source) {
	g.prepNumeric()

	var p []srcobj.Source
	if g.useString {
		//g.regVar("tmpChar", "byte")
		//g.body.Append(srcobj.Assign("tmpChar", src))
		//g.body.Append(srcobj.Raw("\n"))
		//src = srcobj.Raw("tmpChar")
		p = []srcobj.Source{src}
	} else {
		g.regImport("", "unsafe")
		p = []srcobj.Source{
			srcobj.Deref(
				srcobj.NewCall(
					"(*string)",
					srcobj.NewCall(
						"unsafe.Pointer",
						srcobj.Ref(src),
					),
				),
			),
		}
	}
	p = append(p, params...)
	g.body.Append(
		srcobj.If{
			Expr: srcobj.OperatorSemicolon(
				srcobj.OperatorAssign(
					srcobj.OperatorComma(
						srcobj.Raw(tmp),
						srcobj.Raw("err"),
					),
					srcobj.NewCall(
						decoder, // func name, strconv.ParseInt, for instance
						p...,
					),
				),
				srcobj.OperatorNEq(srcobj.Raw("err"), srcobj.Raw("nil")),
			),
			Then: srcobj.ReturnError("Cannot parse `%s`: %s", srcobj.Stringify(src), srcobj.Raw("err")),
		},
	)
	g.body.Append(srcobj.LineAssign{
		Receiver: dest,
		Expr:     srcobj.NewCall(gotype, srcobj.Raw(tmp)),
	})
}

func (g *Generator) decodeInt(src srcobj.Source, dest string) {
	g.prepInt()
	g.decode(src, "tmpInt", "int", dest, "strconv.ParseInt", srcobj.Literal(10), srcobj.Literal(64))
}

func (g *Generator) decodeInt8(src srcobj.Source, dest string) {
	g.prepInt()
	g.decode(src, "tmpInt", "int8", dest, "strconv.ParseInt", srcobj.Literal(10), srcobj.Literal(8))
}

func (g *Generator) decodeInt16(src srcobj.Source, dest string) {
	g.prepInt()
	g.decode(src, "tmpInt", "int16", dest, "strconv.ParseInt", srcobj.Literal(10), srcobj.Literal(16))
}

func (g *Generator) decodeInt32(src srcobj.Source, dest string) {
	g.prepInt()
	g.decode(src, "tmpInt", "int32", dest, "strconv.ParseInt", srcobj.Literal(10), srcobj.Literal(32))
}

func (g *Generator) decodeInt64(src srcobj.Source, dest string) {
	g.prepInt()
	g.decode(src, "tmpInt", "int64", dest, "strconv.ParseInt", srcobj.Literal(10), srcobj.Literal(64))
}

func (g *Generator) decodeUint(src srcobj.Source, dest string) {
	g.prepUint()
	g.decode(src, "tmpUint", "uint", dest, "strconv.ParseUint", srcobj.Literal(10), srcobj.Literal(64))
}

func (g *Generator) decodeUint8(src srcobj.Source, dest string) {
	g.prepUint()
	g.decode(src, "tmpUint", "uint8", dest, "strconv.ParseUint", srcobj.Literal(10), srcobj.Literal(8))
}

func (g *Generator) decodeUint16(src srcobj.Source, dest string) {
	g.prepUint()
	g.decode(src, "tmpUint", "uint16", dest, "strconv.ParseUint", srcobj.Literal(10), srcobj.Literal(16))
}

func (g *Generator) decodeUint32(src srcobj.Source, dest string) {
	g.prepUint()
	g.decode(src, "tmpUint", "uint32", dest, "strconv.ParseUint", srcobj.Literal(10), srcobj.Literal(32))
}

func (g *Generator) decodeUint64(src srcobj.Source, dest string) {
	g.prepUint()
	g.decode(src, "tmpUint", "uint64", dest, "strconv.ParseUint", srcobj.Literal(10), srcobj.Literal(64))
}

func (g *Generator) decodeHex(src srcobj.Source, dest string) {
	g.prepUint()
	g.decode(src, "tmpUint", "uint", dest, "strconv.ParseUint", srcobj.Literal(16), srcobj.Literal(64))
}

func (g *Generator) decodeHex8(src srcobj.Source, dest string) {
	g.prepUint()
	g.decode(src, "tmpUint", "uint8", dest, "strconv.ParseUint", srcobj.Literal(16), srcobj.Literal(8))
}

func (g *Generator) decodeHex16(src srcobj.Source, dest string) {
	g.prepUint()
	g.decode(src, "tmpUint", "uint16", dest, "strconv.ParseUint", srcobj.Literal(16), srcobj.Literal(16))
}

func (g *Generator) decodeHex32(src srcobj.Source, dest string) {
	g.prepUint()
	g.decode(src, "tmpUint", "uint32", dest, "strconv.ParseUint", srcobj.Literal(16), srcobj.Literal(32))
}

func (g *Generator) decodeHex64(src srcobj.Source, dest string) {
	g.prepUint()
	g.decode(src, "tmpUint", "uint64", dest, "strconv.ParseUint", srcobj.Literal(16), srcobj.Literal(64))
}

func (g *Generator) decodeOct(src srcobj.Source, dest string) {
	g.prepUint()
	g.decode(src, "tmpUint", "uint", dest, "strconv.ParseUint", srcobj.Literal(8), srcobj.Literal(64))
}

func (g *Generator) decodeOct8(src srcobj.Source, dest string) {
	g.prepUint()
	g.decode(src, "tmpUint", "uint8", dest, "strconv.ParseUint", srcobj.Literal(8), srcobj.Literal(8))
}

func (g *Generator) decodeOct16(src srcobj.Source, dest string) {
	g.prepUint()
	g.decode(src, "tmpUint", "uint16", dest, "strconv.ParseUint", srcobj.Literal(8), srcobj.Literal(8))
}

func (g *Generator) decodeOct32(src srcobj.Source, dest string) {
	g.prepUint()
	g.decode(src, "tmpUint", "uint32", dest, "strconv.ParseUint", srcobj.Literal(8), srcobj.Literal(32))
}

func (g *Generator) decodeOct64(src srcobj.Source, dest string) {
	g.prepUint()
	g.decode(src, "tmpUint", "uint64", dest, "strconv.ParseUint", srcobj.Literal(8), srcobj.Literal(64))
}

func (g *Generator) prepDec() {
	g.regVar("err", "error")
	g.regImport("", "github.com/sirkon/decconv")
}

func (g *Generator) decodeSmallDecimal(src srcobj.Source, dest, decoder string, precision, scale int) {
	p := []srcobj.Source{
		srcobj.Literal(precision),
		srcobj.Literal(scale),
	}
	if g.useString {
		p = append(p, srcobj.NewCall("[]byte", src))
	} else {
		p = append(p, src)
	}
	g.body.Append(
		srcobj.If{
			Expr: srcobj.OperatorSemicolon(
				srcobj.OperatorAssign(
					srcobj.OperatorComma(
						srcobj.Raw(dest),
						srcobj.Raw("err"),
					),
					srcobj.NewCall(
						decoder, // func name, strconv.ParseInt, for instance
						p...,
					),
				),
				srcobj.OperatorNEq(srcobj.Raw("err"), srcobj.Raw("nil")),
			),
			Then: srcobj.ReturnError("Cannot parse `%s`: %s", srcobj.Stringify(src), srcobj.Raw("err")),
		},
	)
}

func (g *Generator) decodeDec32(src srcobj.Source, dest string, precision, scale int) {
	g.decodeSmallDecimal(src, dest, "decconv.Decode32", precision, scale)
}

func (g *Generator) decodeDec64(src srcobj.Source, dest string, precision, scale int) {
	g.decodeSmallDecimal(src, dest, "decconv.Decode64", precision, scale)
}

func (g *Generator) decodeDec128(src srcobj.Source, dest string, precision, scale int) {
	g.prepDec()
	p := []srcobj.Source{
		srcobj.Literal(precision),
		srcobj.Literal(scale),
	}
	if g.useString {
		p = append(p, srcobj.NewCall("[]byte", src))
	} else {
		p = append(p, src)
	}
	g.body.Append(
		srcobj.If{
			Expr: srcobj.OperatorSemicolon(
				srcobj.OperatorAssign(
					srcobj.OperatorComma(
						srcobj.OperatorComma(
							srcobj.Raw(dest+".Lo"),
							srcobj.Raw(dest+".Hi"),
						),
						srcobj.Raw("err"),
					),
					srcobj.NewCall(
						"decconv.Decode128", // func name, strconv.ParseInt, for instance
						p...,
					),
				),
				srcobj.OperatorNEq(srcobj.Raw("err"), srcobj.Raw("nil")),
			),
			Then: srcobj.ReturnError("Cannot parse `%s`: %s", srcobj.Stringify(src), srcobj.Raw("err")),
		},
	)
}

func (g *Generator) decodeFloat32(src srcobj.Source, dest string) {
	g.prepFloat()
	g.decode(src, "tmpFloat", "float32", dest, "strconv.ParseFloat", srcobj.Literal(32))
}

func (g *Generator) decodeFloat64(src srcobj.Source, dest string) {
	g.prepFloat()
	g.decode(src, "tmpFloat", "float64", dest, "strconv.ParseFloat", srcobj.Literal(64))
}

func (g *Generator) decodeString(src srcobj.Source, dest string) {
	g.body.Append(srcobj.LineAssign{
		Receiver: dest,
		Expr:     src,
	})
}
<commit_msg>fix: fmt import for decimals<commit_after>package gogen

import (
	"github.com/sirkon/ldetool/internal/generator/gogen/internal/srcobj"
)

/* decode_int8 sample - other templates are the same
tmp ={{ .Source }};
if tmpInt, err = strconv.ParseInt(*(*string)(unsafe.Pointer(&tmp)), 10, {{ .Bits }}); err != nil {
    return false, fmt.Errorf("Error parsing as `{{ Type }}` for field {{ .Dest }}: %s", err)
}
p.{{Dest}} = int8(tmpInt)
*/

func (g *Generator) prepNumeric() {
	g.regImport("", "strconv")
	g.regImport("", "fmt")
	g.regVar("err", "error")
}

func (g *Generator) prepInt() {
	g.prepNumeric()
	g.regVar("tmpInt", "int64")
}

func (g *Generator) prepUint() {
	g.prepNumeric()
	g.regVar("tmpUint", "uint64")
}

func (g *Generator) prepFloat() {
	g.prepNumeric()
	g.regVar("tmpFloat", "float64")
}

func (g *Generator) decode(src srcobj.Source, tmp, gotype, dest, decoder string, params ...srcobj.Source) {
	g.prepNumeric()

	var p []srcobj.Source
	if g.useString {
		//g.regVar("tmpChar", "byte")
		//g.body.Append(srcobj.Assign("tmpChar", src))
		//g.body.Append(srcobj.Raw("\n"))
		//src = srcobj.Raw("tmpChar")
		p = []srcobj.Source{src}
	} else {
		g.regImport("", "unsafe")
		p = []srcobj.Source{
			srcobj.Deref(
				srcobj.NewCall(
					"(*string)",
					srcobj.NewCall(
						"unsafe.Pointer",
						srcobj.Ref(src),
					),
				),
			),
		}
	}
	p = append(p, params...)
	g.body.Append(
		srcobj.If{
			Expr: srcobj.OperatorSemicolon(
				srcobj.OperatorAssign(
					srcobj.OperatorComma(
						srcobj.Raw(tmp),
						srcobj.Raw("err"),
					),
					srcobj.NewCall(
						decoder, // func name, strconv.ParseInt, for instance
						p...,
					),
				),
				srcobj.OperatorNEq(srcobj.Raw("err"), srcobj.Raw("nil")),
			),
			Then: srcobj.ReturnError("Cannot parse `%s`: %s", srcobj.Stringify(src), srcobj.Raw("err")),
		},
	)
	g.body.Append(srcobj.LineAssign{
		Receiver: dest,
		Expr:     srcobj.NewCall(gotype, srcobj.Raw(tmp)),
	})
}

func (g *Generator) decodeInt(src srcobj.Source, dest string) {
	g.prepInt()
	g.decode(src, "tmpInt", "int", dest, "strconv.ParseInt", srcobj.Literal(10), srcobj.Literal(64))
}

func (g *Generator) decodeInt8(src srcobj.Source, dest string) {
	g.prepInt()
	g.decode(src, "tmpInt", "int8", dest, "strconv.ParseInt", srcobj.Literal(10), srcobj.Literal(8))
}

func (g *Generator) decodeInt16(src srcobj.Source, dest string) {
	g.prepInt()
	g.decode(src, "tmpInt", "int16", dest, "strconv.ParseInt", srcobj.Literal(10), srcobj.Literal(16))
}

func (g *Generator) decodeInt32(src srcobj.Source, dest string) {
	g.prepInt()
	g.decode(src, "tmpInt", "int32", dest, "strconv.ParseInt", srcobj.Literal(10), srcobj.Literal(32))
}

func (g *Generator) decodeInt64(src srcobj.Source, dest string) {
	g.prepInt()
	g.decode(src, "tmpInt", "int64", dest, "strconv.ParseInt", srcobj.Literal(10), srcobj.Literal(64))
}

func (g *Generator) decodeUint(src srcobj.Source, dest string) {
	g.prepUint()
	g.decode(src, "tmpUint", "uint", dest, "strconv.ParseUint", srcobj.Literal(10), srcobj.Literal(64))
}

func (g *Generator) decodeUint8(src srcobj.Source, dest string) {
	g.prepUint()
	g.decode(src, "tmpUint", "uint8", dest, "strconv.ParseUint", srcobj.Literal(10), srcobj.Literal(8))
}

func (g *Generator) decodeUint16(src srcobj.Source, dest string) {
	g.prepUint()
	g.decode(src, "tmpUint", "uint16", dest, "strconv.ParseUint", srcobj.Literal(10), srcobj.Literal(16))
}

func (g *Generator) decodeUint32(src srcobj.Source, dest string) {
	g.prepUint()
	g.decode(src, "tmpUint", "uint32", dest, "strconv.ParseUint", srcobj.Literal(10), srcobj.Literal(32))
}

func (g *Generator) decodeUint64(src srcobj.Source, dest string) {
	g.prepUint()
	g.decode(src, "tmpUint", "uint64", dest, "strconv.ParseUint", srcobj.Literal(10), srcobj.Literal(64))
}

func (g *Generator) decodeHex(src srcobj.Source, dest string) {
	g.prepUint()
	g.decode(src, "tmpUint", "uint", dest, "strconv.ParseUint", srcobj.Literal(16), srcobj.Literal(64))
}

func (g *Generator) decodeHex8(src srcobj.Source, dest string) {
	g.prepUint()
	g.decode(src, "tmpUint", "uint8", dest, "strconv.ParseUint", srcobj.Literal(16), srcobj.Literal(8))
}

func (g *Generator) decodeHex16(src srcobj.Source, dest string) {
	g.prepUint()
	g.decode(src, "tmpUint", "uint16", dest, "strconv.ParseUint", srcobj.Literal(16), srcobj.Literal(16))
}

func (g *Generator) decodeHex32(src srcobj.Source, dest string) {
	g.prepUint()
	g.decode(src, "tmpUint", "uint32", dest, "strconv.ParseUint", srcobj.Literal(16), srcobj.Literal(32))
}

func (g *Generator) decodeHex64(src srcobj.Source, dest string) {
	g.prepUint()
	g.decode(src, "tmpUint", "uint64", dest, "strconv.ParseUint", srcobj.Literal(16), srcobj.Literal(64))
}

func (g *Generator) decodeOct(src srcobj.Source, dest string) {
	g.prepUint()
	g.decode(src, "tmpUint", "uint", dest, "strconv.ParseUint", srcobj.Literal(8), srcobj.Literal(64))
}

func (g *Generator) decodeOct8(src srcobj.Source, dest string) {
	g.prepUint()
	g.decode(src, "tmpUint", "uint8", dest, "strconv.ParseUint", srcobj.Literal(8), srcobj.Literal(8))
}

func (g *Generator) decodeOct16(src srcobj.Source, dest string) {
	g.prepUint()
	g.decode(src, "tmpUint", "uint16", dest, "strconv.ParseUint", srcobj.Literal(8), srcobj.Literal(8))
}

func (g *Generator) decodeOct32(src srcobj.Source, dest string) {
	g.prepUint()
	g.decode(src, "tmpUint", "uint32", dest, "strconv.ParseUint", srcobj.Literal(8), srcobj.Literal(32))
}

func (g *Generator) decodeOct64(src srcobj.Source, dest string) {
	g.prepUint()
	g.decode(src, "tmpUint", "uint64", dest, "strconv.ParseUint", srcobj.Literal(8), srcobj.Literal(64))
}

func (g *Generator) prepDec() {
	g.regVar("err", "error")
	g.regImport("", "fmt")
	g.regImport("", "github.com/sirkon/decconv")
}

func (g *Generator) decodeSmallDecimal(src srcobj.Source, dest, decoder string, precision, scale int) {
	g.regImport("", "fmt")
	p := []srcobj.Source{
		srcobj.Literal(precision),
		srcobj.Literal(scale),
	}
	if g.useString {
		p = append(p, srcobj.NewCall("[]byte", src))
	} else {
		p = append(p, src)
	}
	g.body.Append(
		srcobj.If{
			Expr: srcobj.OperatorSemicolon(
				srcobj.OperatorAssign(
					srcobj.OperatorComma(
						srcobj.Raw(dest),
						srcobj.Raw("err"),
					),
					srcobj.NewCall(
						decoder, // func name, strconv.ParseInt, for instance
						p...,
					),
				),
				srcobj.OperatorNEq(srcobj.Raw("err"), srcobj.Raw("nil")),
			),
			Then: srcobj.ReturnError("Cannot parse `%s`: %s", srcobj.Stringify(src), srcobj.Raw("err")),
		},
	)
}

func (g *Generator) decodeDec32(src srcobj.Source, dest string, precision, scale int) {
	g.decodeSmallDecimal(src, dest, "decconv.Decode32", precision, scale)
}

func (g *Generator) decodeDec64(src srcobj.Source, dest string, precision, scale int) {
	g.decodeSmallDecimal(src, dest, "decconv.Decode64", precision, scale)
}

func (g *Generator) decodeDec128(src srcobj.Source, dest string, precision, scale int) {
	g.prepDec()
	p := []srcobj.Source{
		srcobj.Literal(precision),
		srcobj.Literal(scale),
	}
	if g.useString {
		p = append(p, srcobj.NewCall("[]byte", src))
	} else {
		p = append(p, src)
	}
	g.body.Append(
		srcobj.If{
			Expr: srcobj.OperatorSemicolon(
				srcobj.OperatorAssign(
					srcobj.OperatorComma(
						srcobj.OperatorComma(
							srcobj.Raw(dest+".Lo"),
							srcobj.Raw(dest+".Hi"),
						),
						srcobj.Raw("err"),
					),
					srcobj.NewCall(
						"decconv.Decode128", // func name, strconv.ParseInt, for instance
						p...,
					),
				),
				srcobj.OperatorNEq(srcobj.Raw("err"), srcobj.Raw("nil")),
			),
			Then: srcobj.ReturnError("Cannot parse `%s`: %s", srcobj.Stringify(src), srcobj.Raw("err")),
		},
	)
}

func (g *Generator) decodeFloat32(src srcobj.Source, dest string) {
	g.prepFloat()
	g.decode(src, "tmpFloat", "float32", dest, "strconv.ParseFloat", srcobj.Literal(32))
}

func (g *Generator) decodeFloat64(src srcobj.Source, dest string) {
	g.prepFloat()
	g.decode(src, "tmpFloat", "float64", dest, "strconv.ParseFloat", srcobj.Literal(64))
}

func (g *Generator) decodeString(src srcobj.Source, dest string) {
	g.body.Append(srcobj.LineAssign{
		Receiver: dest,
		Expr:     src,
	})
}
<|endoftext|>


<commit_before>package lfqueue

import (
	"fmt"
	"log"
	"math/rand"
	"runtime"
	"sync"
	"testing"
	"time"
)

func TestLfQueueConsistency(t *testing.T) {
	max := 1000000
	c := 100
	var wg sync.WaitGroup
	wg.Add(1)
	q := NewLockfreeQueue()
	go func() {
		i := 0
		seen := make(map[string]string)
		for {
			r := q.Pop()
			if r == nil {
				runtime.Gosched()

				continue
			}
			i++
			s := r.(string)
			_, present := seen[s]
			if present {
				log.Printf("item have already been seen %v", s)
				t.FailNow()
			}
			seen[s] = s

			if i == max {
				wg.Done()
				return
			}
		}
	}()

	for j := 0; j < c; j++ {
		jj := j
		cmax := max / c
		go func() {
			for i := 0; i < cmax; i++ {
				if rand.Intn(10) == 0 {
					time.Sleep(time.Duration(rand.Intn(1000)))
				}
				q.Push(fmt.Sprintf("%v %v", jj, i))
			}
		}()
	}

	wg.Wait()
	time.Sleep(500 * time.Millisecond)
	//queue should be empty
	for i := 0; i < 100; i++ {
		r := q.Pop()
		if r != nil {
			log.Printf("unexpected result %+v", r)
			t.FailNow()
		}
	}
}
<commit_msg>force build<commit_after>package lfqueue

import (
	"fmt"
	"log"
	"math/rand"
	"runtime"
	"sync"
	"testing"
	"time"
)

func TestLfQueueConsistency(t *testing.T) {
	max := 1000000
	c := 100
	var wg sync.WaitGroup
	wg.Add(1)
	q := NewLockfreeQueue()
	go func() {
		i := 0
		seen := make(map[string]string)
		for {
			r := q.Pop()
			if r == nil {
				runtime.Gosched()

				continue
			}
			i++
			s := r.(string)
			_, present := seen[s]
			if present {
				log.Printf("duplicate item %v", s)
				t.FailNow()
			}
			seen[s] = s

			if i == max {
				wg.Done()
				return
			}
		}
	}()

	for j := 0; j < c; j++ {
		jj := j
		cmax := max / c
		go func() {
			for i := 0; i < cmax; i++ {
				if rand.Intn(10) == 0 {
					time.Sleep(time.Duration(rand.Intn(1000)))
				}
				q.Push(fmt.Sprintf("%v %v", jj, i))
			}
		}()
	}

	wg.Wait()
	time.Sleep(500 * time.Millisecond)
	//queue should be empty
	for i := 0; i < 100; i++ {
		r := q.Pop()
		if r != nil {
			log.Printf("unexpected result %+v", r)
			t.FailNow()
		}
	}
}
<|endoftext|>


<commit_before>package acceptance

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/pivotal-cf-experimental/pivnet-resource/concourse"
	"github.com/pivotal-cf-experimental/pivnet-resource/logger"
	"github.com/pivotal-cf-experimental/pivnet-resource/pivnet"

	"testing"
)

var (
	inPath    string
	checkPath string
	outPath   string

	endpoint string

	productSlug        string
	pivnetAPIToken     string
	awsAccessKeyID     string
	awsSecretAccessKey string
	pivnetRegion       string
	pivnetBucketName   string
	s3FilepathPrefix   string

	pivnetClient pivnet.Client
)

func TestAcceptance(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Acceptance Suite")
}

var _ = BeforeSuite(func() {
	var err error
	By("Getting product slug from environment variables")
	productSlug = os.Getenv("PRODUCT_SLUG")
	Expect(productSlug).NotTo(BeEmpty(), "$PRODUCT_SLUG must be provided")

	By("Getting API token from environment variables")
	pivnetAPIToken = os.Getenv("API_TOKEN")
	Expect(pivnetAPIToken).NotTo(BeEmpty(), "$API_TOKEN must be provided")

	By("Getting aws access key id from environment variables")
	awsAccessKeyID = os.Getenv("AWS_ACCESS_KEY_ID")
	Expect(awsAccessKeyID).NotTo(BeEmpty(), "$AWS_ACCESS_KEY_ID must be provided")

	By("Getting aws secret access key from environment variables")
	awsSecretAccessKey = os.Getenv("AWS_SECRET_ACCESS_KEY")
	Expect(awsSecretAccessKey).NotTo(BeEmpty(), "$AWS_SECRET_ACCESS_KEY must be provided")

	By("Getting pivnet region from environment variables")
	pivnetRegion = os.Getenv("PIVNET_S3_REGION")
	Expect(pivnetRegion).NotTo(BeEmpty(), "$PIVNET_S3_REGION must be provided")

	By("Getting pivnet bucket name from environment variables")
	pivnetBucketName = os.Getenv("PIVNET_BUCKET_NAME")
	Expect(pivnetBucketName).NotTo(BeEmpty(), "$PIVNET_BUCKET_NAME must be provided")

	By("Getting s3 filepath prefix from environment variables")
	s3FilepathPrefix = os.Getenv("S3_FILEPATH_PREFIX")
	Expect(s3FilepathPrefix).NotTo(BeEmpty(), "$S3_FILEPATH_PREFIX must be provided")

	By("Getting endpoint from environment variables")
	endpoint = os.Getenv("PIVNET_ENDPOINT")
	Expect(endpoint).NotTo(BeEmpty(), "$PIVNET_ENDPOINT must be provided")

	By("Compiling check binary")
	checkPath, err = gexec.Build("github.com/pivotal-cf-experimental/pivnet-resource/cmd/check", "-race")
	Expect(err).NotTo(HaveOccurred())

	By("Compiling out binary")
	outPath, err = gexec.Build("github.com/pivotal-cf-experimental/pivnet-resource/cmd/out", "-race")
	Expect(err).NotTo(HaveOccurred())

	By("Compiling in binary")
	inPath, err = gexec.Build("github.com/pivotal-cf-experimental/pivnet-resource/cmd/in", "-race")
	Expect(err).NotTo(HaveOccurred())

	By("Copying s3-out to compilation location")
	originalS3OutPath := os.Getenv("S3_OUT_LOCATION")
	Expect(originalS3OutPath).ToNot(BeEmpty(), "$S3_OUT_LOCATION must be provided")
	_, err = os.Stat(originalS3OutPath)
	Expect(err).NotTo(HaveOccurred())
	s3OutPath := filepath.Join(path.Dir(outPath), "s3-out")
	copyFileContents(originalS3OutPath, s3OutPath)
	Expect(err).NotTo(HaveOccurred())

	By("Ensuring copy of s3-out is executable")
	err = os.Chmod(s3OutPath, os.ModePerm)
	Expect(err).NotTo(HaveOccurred())

	By("Creating pivnet client (for out-of-band operations)")
	testLogger := logger.NewLogger(GinkgoWriter)

	clientConfig := pivnet.NewClientConfig{
		Endpoint:  endpoint,
		Token:     pivnetAPIToken,
		UserAgent: "pivnet-resource/integration-test",
	}
	pivnetClient = pivnet.NewClient(clientConfig, testLogger)
})

var _ = AfterSuite(func() {
	gexec.CleanupBuildArtifacts()
})

func getReleases(productSlug string) []pivnet.Release {
	productURL := fmt.Sprintf(
		"%s/api/v2/products/%s/releases",
		endpoint,
		productSlug,
	)

	req, err := http.NewRequest("GET", productURL, nil)
	Expect(err).NotTo(HaveOccurred())

	req.Header.Add("Authorization", fmt.Sprintf("Token %s", pivnetAPIToken))

	resp, err := http.DefaultClient.Do(req)
	Expect(err).NotTo(HaveOccurred())
	Expect(resp.StatusCode).To(Equal(http.StatusOK))

	response := pivnet.Response{}
	err = json.NewDecoder(resp.Body).Decode(&response)
	Expect(err).NotTo(HaveOccurred())

	return response.Releases
}

func getProductVersions(productSlug string) []string {
	var versions []string
	for _, release := range getReleases(productSlug) {
		versions = append(versions, string(release.Version))
	}

	return versions
}

func getPivnetRelease(productSlug, productVersion string) pivnet.Release {
	for _, release := range getReleases(productSlug) {
		if release.Version == productVersion {
			return release
		}
	}
	Fail(fmt.Sprintf(
		"Could not find release for productSlug: %s and productVersion: %s",
		productSlug,
		productVersion,
	))
	// We won't get here
	return pivnet.Release{}
}

func deletePivnetRelease(productSlug, productVersion string) {
	pivnetRelease := getPivnetRelease(productSlug, productVersion)
	releaseID := pivnetRelease.ID
	Expect(releaseID).NotTo(Equal(0))

	productURL := fmt.Sprintf(
		"%s/api/v2/products/%s/releases/%d",
		endpoint,
		productSlug,
		releaseID,
	)

	req, err := http.NewRequest("DELETE", productURL, nil)
	Expect(err).NotTo(HaveOccurred())

	req.Header.Add("Authorization", fmt.Sprintf("Token %s", pivnetAPIToken))

	resp, err := http.DefaultClient.Do(req)
	Expect(err).NotTo(HaveOccurred())
	Expect(resp.StatusCode).To(Equal(http.StatusNoContent))
}

func getProductFiles(productSlug string) []pivnet.ProductFile {
	productURL := fmt.Sprintf(
		"%s/api/v2/products/%s/product_files",
		endpoint,
		productSlug,
	)

	req, err := http.NewRequest("GET", productURL, nil)
	Expect(err).NotTo(HaveOccurred())

	req.Header.Add("Authorization", fmt.Sprintf("Token %s", pivnetAPIToken))

	resp, err := http.DefaultClient.Do(req)
	Expect(err).NotTo(HaveOccurred())
	Expect(resp.StatusCode).To(Equal(http.StatusOK))

	response := pivnet.ProductFiles{}
	err = json.NewDecoder(resp.Body).Decode(&response)
	Expect(err).NotTo(HaveOccurred())

	return response.ProductFiles
}

func getUserGroups(productSlug string, releaseID int) []pivnet.UserGroup {
	userGroupsURL := fmt.Sprintf(
		"%s/api/v2/products/%s/releases/%d/user_groups",
		endpoint,
		productSlug,
		releaseID,
	)

	req, err := http.NewRequest("GET", userGroupsURL, nil)
	Expect(err).NotTo(HaveOccurred())

	req.Header.Add("Authorization", fmt.Sprintf("Token %s", pivnetAPIToken))

	resp, err := http.DefaultClient.Do(req)
	Expect(err).NotTo(HaveOccurred())
	Expect(resp.StatusCode).To(Equal(http.StatusOK))

	response := pivnet.UserGroups{}
	err = json.NewDecoder(resp.Body).Decode(&response)
	Expect(err).NotTo(HaveOccurred())

	return response.UserGroups
}

// copyFileContents copies the contents of the file named src to the file named
// by dst. The file will be created if it does not already exist. If the
// destination file exists, all it's contents will be replaced by the contents
// of the source file.
// See http://stackoverflow.com/questions/21060945/simple-way-to-copy-a-file-in-golang
func copyFileContents(src, dst string) (err error) {
	in, err := os.Open(src)
	if err != nil {
		return
	}
	defer in.Close()
	out, err := os.Create(dst)
	if err != nil {
		return
	}
	defer func() {
		cerr := out.Close()
		if err == nil {
			err = cerr
		}
	}()
	if _, err = io.Copy(out, in); err != nil {
		return
	}
	err = out.Sync()
	return
}

func sanitize(contents string) string {
	output := contents
	output = strings.Replace(output, pivnetAPIToken, "***sanitized-api-token***", -1)
	output = strings.Replace(output, awsAccessKeyID, "***sanitized-aws-access-key-id***", -1)
	output = strings.Replace(output, awsSecretAccessKey, "***sanitized-aws-secret-access-key***", -1)
	return output
}

func run(command *exec.Cmd, stdinContents []byte) *gexec.Session {
	fmt.Fprintf(GinkgoWriter, "input: %s\n", sanitize(string(stdinContents)))

	stdin, err := command.StdinPipe()
	Expect(err).ShouldNot(HaveOccurred())

	session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
	Expect(err).NotTo(HaveOccurred())

	_, err = io.WriteString(stdin, string(stdinContents))
	Expect(err).ShouldNot(HaveOccurred())

	err = stdin.Close()
	Expect(err).ShouldNot(HaveOccurred())

	return session
}

func metadataValueForKey(metadata []concourse.Metadata, name string) (string, error) {
	for _, i := range metadata {
		if i.Name == name {
			return i.Value, nil
		}
	}
	return "", fmt.Errorf("name not found: %s", name)
}
<commit_msg>Sanitize acceptance test output.<commit_after>package acceptance

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"path"
	"path/filepath"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/pivotal-cf-experimental/pivnet-resource/concourse"
	"github.com/pivotal-cf-experimental/pivnet-resource/logger"
	"github.com/pivotal-cf-experimental/pivnet-resource/pivnet"
	"github.com/pivotal-cf-experimental/pivnet-resource/sanitizer"

	"testing"
)

var (
	inPath    string
	checkPath string
	outPath   string

	endpoint string

	productSlug        string
	pivnetAPIToken     string
	awsAccessKeyID     string
	awsSecretAccessKey string
	pivnetRegion       string
	pivnetBucketName   string
	s3FilepathPrefix   string

	pivnetClient pivnet.Client
)

func TestAcceptance(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Acceptance Suite")
}

var _ = BeforeSuite(func() {
	var err error
	By("Getting product slug from environment variables")
	productSlug = os.Getenv("PRODUCT_SLUG")
	Expect(productSlug).NotTo(BeEmpty(), "$PRODUCT_SLUG must be provided")

	By("Getting API token from environment variables")
	pivnetAPIToken = os.Getenv("API_TOKEN")
	Expect(pivnetAPIToken).NotTo(BeEmpty(), "$API_TOKEN must be provided")

	By("Getting aws access key id from environment variables")
	awsAccessKeyID = os.Getenv("AWS_ACCESS_KEY_ID")
	Expect(awsAccessKeyID).NotTo(BeEmpty(), "$AWS_ACCESS_KEY_ID must be provided")

	By("Getting aws secret access key from environment variables")
	awsSecretAccessKey = os.Getenv("AWS_SECRET_ACCESS_KEY")
	Expect(awsSecretAccessKey).NotTo(BeEmpty(), "$AWS_SECRET_ACCESS_KEY must be provided")

	By("Getting pivnet region from environment variables")
	pivnetRegion = os.Getenv("PIVNET_S3_REGION")
	Expect(pivnetRegion).NotTo(BeEmpty(), "$PIVNET_S3_REGION must be provided")

	By("Getting pivnet bucket name from environment variables")
	pivnetBucketName = os.Getenv("PIVNET_BUCKET_NAME")
	Expect(pivnetBucketName).NotTo(BeEmpty(), "$PIVNET_BUCKET_NAME must be provided")

	By("Getting s3 filepath prefix from environment variables")
	s3FilepathPrefix = os.Getenv("S3_FILEPATH_PREFIX")
	Expect(s3FilepathPrefix).NotTo(BeEmpty(), "$S3_FILEPATH_PREFIX must be provided")

	By("Getting endpoint from environment variables")
	endpoint = os.Getenv("PIVNET_ENDPOINT")
	Expect(endpoint).NotTo(BeEmpty(), "$PIVNET_ENDPOINT must be provided")

	By("Compiling check binary")
	checkPath, err = gexec.Build("github.com/pivotal-cf-experimental/pivnet-resource/cmd/check", "-race")
	Expect(err).NotTo(HaveOccurred())

	By("Compiling out binary")
	outPath, err = gexec.Build("github.com/pivotal-cf-experimental/pivnet-resource/cmd/out", "-race")
	Expect(err).NotTo(HaveOccurred())

	By("Compiling in binary")
	inPath, err = gexec.Build("github.com/pivotal-cf-experimental/pivnet-resource/cmd/in", "-race")
	Expect(err).NotTo(HaveOccurred())

	By("Copying s3-out to compilation location")
	originalS3OutPath := os.Getenv("S3_OUT_LOCATION")
	Expect(originalS3OutPath).ToNot(BeEmpty(), "$S3_OUT_LOCATION must be provided")
	_, err = os.Stat(originalS3OutPath)
	Expect(err).NotTo(HaveOccurred())
	s3OutPath := filepath.Join(path.Dir(outPath), "s3-out")
	copyFileContents(originalS3OutPath, s3OutPath)
	Expect(err).NotTo(HaveOccurred())

	By("Ensuring copy of s3-out is executable")
	err = os.Chmod(s3OutPath, os.ModePerm)
	Expect(err).NotTo(HaveOccurred())

	By("Sanitizing acceptance test output")
	sanitized := map[string]string{
		pivnetAPIToken:     "***sanitized-api-token***",
		awsAccessKeyID:     "***sanitized-aws-access-key-id***",
		awsSecretAccessKey: "***sanitized-aws-secret-access-key***",
	}
	sanitizer := sanitizer.NewSanitizer(sanitized, GinkgoWriter)
	GinkgoWriter = sanitizer

	By("Creating pivnet client (for out-of-band operations)")
	testLogger := logger.NewLogger(GinkgoWriter)

	clientConfig := pivnet.NewClientConfig{
		Endpoint:  endpoint,
		Token:     pivnetAPIToken,
		UserAgent: "pivnet-resource/integration-test",
	}
	pivnetClient = pivnet.NewClient(clientConfig, testLogger)
})

var _ = AfterSuite(func() {
	gexec.CleanupBuildArtifacts()
})

func getReleases(productSlug string) []pivnet.Release {
	productURL := fmt.Sprintf(
		"%s/api/v2/products/%s/releases",
		endpoint,
		productSlug,
	)

	req, err := http.NewRequest("GET", productURL, nil)
	Expect(err).NotTo(HaveOccurred())

	req.Header.Add("Authorization", fmt.Sprintf("Token %s", pivnetAPIToken))

	resp, err := http.DefaultClient.Do(req)
	Expect(err).NotTo(HaveOccurred())
	Expect(resp.StatusCode).To(Equal(http.StatusOK))

	response := pivnet.Response{}
	err = json.NewDecoder(resp.Body).Decode(&response)
	Expect(err).NotTo(HaveOccurred())

	return response.Releases
}

func getProductVersions(productSlug string) []string {
	var versions []string
	for _, release := range getReleases(productSlug) {
		versions = append(versions, string(release.Version))
	}

	return versions
}

func getPivnetRelease(productSlug, productVersion string) pivnet.Release {
	for _, release := range getReleases(productSlug) {
		if release.Version == productVersion {
			return release
		}
	}
	Fail(fmt.Sprintf(
		"Could not find release for productSlug: %s and productVersion: %s",
		productSlug,
		productVersion,
	))
	// We won't get here
	return pivnet.Release{}
}

func deletePivnetRelease(productSlug, productVersion string) {
	pivnetRelease := getPivnetRelease(productSlug, productVersion)
	releaseID := pivnetRelease.ID
	Expect(releaseID).NotTo(Equal(0))

	productURL := fmt.Sprintf(
		"%s/api/v2/products/%s/releases/%d",
		endpoint,
		productSlug,
		releaseID,
	)

	req, err := http.NewRequest("DELETE", productURL, nil)
	Expect(err).NotTo(HaveOccurred())

	req.Header.Add("Authorization", fmt.Sprintf("Token %s", pivnetAPIToken))

	resp, err := http.DefaultClient.Do(req)
	Expect(err).NotTo(HaveOccurred())
	Expect(resp.StatusCode).To(Equal(http.StatusNoContent))
}

func getProductFiles(productSlug string) []pivnet.ProductFile {
	productURL := fmt.Sprintf(
		"%s/api/v2/products/%s/product_files",
		endpoint,
		productSlug,
	)

	req, err := http.NewRequest("GET", productURL, nil)
	Expect(err).NotTo(HaveOccurred())

	req.Header.Add("Authorization", fmt.Sprintf("Token %s", pivnetAPIToken))

	resp, err := http.DefaultClient.Do(req)
	Expect(err).NotTo(HaveOccurred())
	Expect(resp.StatusCode).To(Equal(http.StatusOK))

	response := pivnet.ProductFiles{}
	err = json.NewDecoder(resp.Body).Decode(&response)
	Expect(err).NotTo(HaveOccurred())

	return response.ProductFiles
}

func getUserGroups(productSlug string, releaseID int) []pivnet.UserGroup {
	userGroupsURL := fmt.Sprintf(
		"%s/api/v2/products/%s/releases/%d/user_groups",
		endpoint,
		productSlug,
		releaseID,
	)

	req, err := http.NewRequest("GET", userGroupsURL, nil)
	Expect(err).NotTo(HaveOccurred())

	req.Header.Add("Authorization", fmt.Sprintf("Token %s", pivnetAPIToken))

	resp, err := http.DefaultClient.Do(req)
	Expect(err).NotTo(HaveOccurred())
	Expect(resp.StatusCode).To(Equal(http.StatusOK))

	response := pivnet.UserGroups{}
	err = json.NewDecoder(resp.Body).Decode(&response)
	Expect(err).NotTo(HaveOccurred())

	return response.UserGroups
}

// copyFileContents copies the contents of the file named src to the file named
// by dst. The file will be created if it does not already exist. If the
// destination file exists, all it's contents will be replaced by the contents
// of the source file.
// See http://stackoverflow.com/questions/21060945/simple-way-to-copy-a-file-in-golang
func copyFileContents(src, dst string) (err error) {
	in, err := os.Open(src)
	if err != nil {
		return
	}
	defer in.Close()
	out, err := os.Create(dst)
	if err != nil {
		return
	}
	defer func() {
		cerr := out.Close()
		if err == nil {
			err = cerr
		}
	}()
	if _, err = io.Copy(out, in); err != nil {
		return
	}
	err = out.Sync()
	return
}

func run(command *exec.Cmd, stdinContents []byte) *gexec.Session {
	fmt.Fprintf(GinkgoWriter, "input: %s\n", stdinContents)

	stdin, err := command.StdinPipe()
	Expect(err).ShouldNot(HaveOccurred())

	session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
	Expect(err).NotTo(HaveOccurred())

	_, err = io.WriteString(stdin, string(stdinContents))
	Expect(err).ShouldNot(HaveOccurred())

	err = stdin.Close()
	Expect(err).ShouldNot(HaveOccurred())

	return session
}

func metadataValueForKey(metadata []concourse.Metadata, name string) (string, error) {
	for _, i := range metadata {
		if i.Name == name {
			return i.Value, nil
		}
	}
	return "", fmt.Errorf("name not found: %s", name)
}
<|endoftext|>