<commit_before>package job

import (
	"bytes"
	"encoding/gob"
	"errors"
	"fmt"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/ajvb/kala/utils/iso8601"

	log "github.com/Sirupsen/logrus"
	"github.com/nu7hatch/gouuid"
)

var (
	RFC3339WithoutTimezone = "2006-01-02T15:04:05"

	ErrInvalidJob       = errors.New("Invalid Local Job. Job's must contain a Name and a Command field")
	ErrInvalidRemoteJob = errors.New("Invalid Remote Job. Job's must contain a Name and a url field")
	ErrInvalidJobType   = errors.New("Invalid Job type. Types supported: 0 for local and 1 for remote")
)

type Job struct {
	Name string `json:"name"`
	Id   string `json:"id"`

	// Command to run
	// e.g. "bash /path/to/my/script.sh"
	Command string `json:"command"`

	// Email of the owner of this job
	// e.g. "admin@example.com"
	Owner string `json:"owner"`

	// Is this job disabled?
	Disabled bool `json:"disabled"`

	// Jobs that are dependent upon this one will be run after this job runs.
	DependentJobs []string `json:"dependent_jobs"`

	// List of ids of jobs that this job is dependent upon.
	ParentJobs []string `json:"parent_jobs"`

	// ISO 8601 String
	// e.g. "R/2014-03-08T20:00:00.000Z/PT2H"
	Schedule     string `json:"schedule"`
	scheduleTime time.Time
	// ISO 8601 Duration struct, used for scheduling
	// job after each run.
	delayDuration *iso8601.Duration

	// Number of times to schedule this job after the
	// first run.
	timesToRepeat int64

	// Number of times to retry on failed attempt for each run.
	Retries uint `json:"retries"`

	// Duration in which it is safe to retry the Job.
	Epsilon         string `json:"epsilon"`
	epsilonDuration *iso8601.Duration

	jobTimer  *time.Timer
	NextRunAt time.Time `json:"next_run_at"`

	// Meta data about successful and failed runs.
	Metadata Metadata `json:"metadata"`

	// Type of the job
	JobType jobType `json:"type"`

	// Custom properties for the remote job type
	RemoteProperties RemoteProperties `json:"remote_properties"`

	// Collection of Job Stats
	Stats []*JobStat `json:"stats"`

	lock sync.RWMutex

	// Says if a job has been executed right numbers of time
	// and should not been executed again in the future
	IsDone bool `json:"is_done"`
}

type jobType int

const (
	LocalJob jobType = iota
	RemoteJob
)

// RemoteProperties Custom properties for the remote job type
type RemoteProperties struct {
	Url    string `json:"url"`
	Method string `json:"method"`

	// A body to attach to the http request
	Body string `json:"body"`

	// A list of headers to add to http request (e.g. [{"key": "charset", "value": "UTF-8"}])
	Headers http.Header `json:"headers"`

	// A timeout property for the http request in seconds
	Timeout int `json:"timeout"`

	// A list of expected response codes (e.g. [200, 201])
	ExpectedResponseCodes []int `json:"expected_response_codes"`
}

type Metadata struct {
	SuccessCount     uint      `json:"success_count"`
	LastSuccess      time.Time `json:"last_success"`
	ErrorCount       uint      `json:"error_count"`
	LastError        time.Time `json:"last_error"`
	LastAttemptedRun time.Time `json:"last_attempted_run"`
}

// Bytes returns the byte representation of the Job.
func (j Job) Bytes() ([]byte, error) {
	buff := new(bytes.Buffer)
	enc := gob.NewEncoder(buff)
	err := enc.Encode(j)
	if err != nil {
		return nil, err
	}
	return buff.Bytes(), nil
}

// NewFromBytes returns a Job instance from a byte representation.
func NewFromBytes(b []byte) (*Job, error) {
	j := &Job{}

	buf := bytes.NewBuffer(b)
	err := gob.NewDecoder(buf).Decode(j)
	if err != nil {
		return nil, err
	}

	return j, nil
}

// Init fills in the protected fields and parses the iso8601 notation.
// It also adds the job to the Cache
func (j *Job) Init(cache JobCache) error {
	j.lock.Lock()
	defer j.lock.Unlock()

	//validate job type and params
	err := j.validation()
	if err != nil {
		return err
	}

	u4, err := uuid.NewV4()
	if err != nil {
		log.Errorf("Error occured when generating uuid: %s", err)
		return err
	}
	j.Id = u4.String()

	// Add Job to the cache.
	err = cache.Set(j)
	if err != nil {
		return err
	}

	if len(j.ParentJobs) != 0 {
		// Add new job to parent jobs
		for _, p := range j.ParentJobs {
			parentJob, err := cache.Get(p)
			if err != nil {
				return err
			}
			parentJob.DependentJobs = append(parentJob.DependentJobs, j.Id)
		}

		return nil
	}

	// TODO: Delete from cache after running.
	if j.Schedule == "" {
		// If schedule is empty, its a one-off job.
		go j.Run(cache)
		return nil
	}

	j.lock.Unlock()
	err = j.InitDelayDuration(true)
	j.lock.Lock()
	if err != nil {
		j.lock.Unlock()
		cache.Delete(j.Id)
		j.lock.Lock()
		return err
	}

	j.lock.Unlock()
	j.StartWaiting(cache)

	j.lock.Lock()

	return nil
}

// InitDelayDuration is used to parsed the iso8601 Schedule notation into its relevent fields in the Job struct.
// If checkTime is true, then it will return an error if the Scheduled time has passed.
func (j *Job) InitDelayDuration(checkTime bool) error {
	j.lock.Lock()
	defer j.lock.Unlock()

	var err error
	splitTime := strings.Split(j.Schedule, "/")
	if len(splitTime) != 3 {
		return fmt.Errorf(
			"Schedule not formatted correctly. Should look like: R/2014-03-08T20:00:00Z/PT2H",
		)
	}

	// Handle Repeat Amount
	if splitTime[0] == "R" {
		// Repeat forever
		j.timesToRepeat = -1
	} else {
		j.timesToRepeat, err = strconv.ParseInt(strings.Split(splitTime[0], "R")[1], 10, 0)
		if err != nil {
			log.Errorf("Error converting timesToRepeat to an int: %s", err)
			return err
		}
	}
	log.Debugf("timesToRepeat: %d", j.timesToRepeat)

	j.scheduleTime, err = time.Parse(time.RFC3339, splitTime[1])
	if err != nil {
		j.scheduleTime, err = time.Parse(RFC3339WithoutTimezone, splitTime[1])
		if err != nil {
			log.Errorf("Error converting scheduleTime to a time.Time: %s", err)
			return err
		}
	}
	if checkTime {
		if (time.Duration(j.scheduleTime.UnixNano() - time.Now().UnixNano())) < 0 {
			return fmt.Errorf("Schedule time has passed on Job with id of %s", j.Id)
		}
	}
	log.Debugf("Schedule Time: %s", j.scheduleTime)

	if j.timesToRepeat != 0 {
		j.delayDuration, err = iso8601.FromString(splitTime[2])
		if err != nil {
			log.Errorf("Error converting delayDuration to a iso8601.Duration: %s", err)
			return err
		}
		log.Debugf("Delay Duration: %s", j.delayDuration.ToDuration())
	}

	if j.Epsilon != "" {
		j.epsilonDuration, err = iso8601.FromString(j.Epsilon)
		if err != nil {
			log.Errorf("Error converting j.Epsilon to iso8601.Duration: %s", err)
			return err
		}
	}

	return nil
}

// StartWaiting begins a timer for when it should execute the Jobs .Run() method.
func (j *Job) StartWaiting(cache JobCache) {
	waitDuration := j.GetWaitDuration()

	j.lock.Lock()
	defer j.lock.Unlock()

	log.Infof("Job Scheduled to run in: %s", waitDuration)

	j.NextRunAt = time.Now().Add(waitDuration)

	jobRun := func() { j.Run(cache) }
	j.jobTimer = time.AfterFunc(waitDuration, jobRun)
}

func (j *Job) GetWaitDuration() time.Duration {
	j.lock.RLock()
	defer j.lock.RUnlock()

	waitDuration := time.Duration(j.scheduleTime.UnixNano() - time.Now().UnixNano())

	if waitDuration < 0 {
		if j.timesToRepeat == 0 {
			return 0
		}

		if j.Metadata.LastAttemptedRun.IsZero() {
			waitDuration = j.delayDuration.ToDuration()
		} else {
			lastRun := j.Metadata.LastAttemptedRun
			// Needs to be recalculated each time because of Months.
			lastRun = lastRun.Add(j.delayDuration.ToDuration())
			waitDuration = lastRun.Sub(time.Now())
		}
	}

	return waitDuration
}

// Disable stops the job from running by stopping its jobTimer. It also sets Job.Disabled to true,
// which is reflected in the UI.
func (j *Job) Disable() {
	j.lock.Lock()
	defer j.lock.Unlock()

	if j.jobTimer != nil {
		j.jobTimer.Stop()
	}
	j.Disabled = true
}

func (j *Job) Enable(cache JobCache) {
	j.lock.Lock()
	defer j.lock.Unlock()

	if j.jobTimer != nil && j.Disabled {
		go j.StartWaiting(cache)
	}
	j.Disabled = false
}

// DeleteFromParentJobs goes through and deletes the current job from any parent jobs.
func (j *Job) DeleteFromParentJobs(cache JobCache) error {
	j.lock.Lock()
	defer j.lock.Unlock()

	for _, p := range j.ParentJobs {
		parentJob, err := cache.Get(p)

		if err != nil {
			return err
		}

		parentJob.lock.Lock()

		ndx := 0
		for i, id := range parentJob.DependentJobs {
			if id == j.Id {
				ndx = i
				break
			}
		}
		parentJob.DependentJobs = append(
			parentJob.DependentJobs[:ndx], parentJob.DependentJobs[ndx+1:]...,
		)

		parentJob.lock.Unlock()
	}

	return nil
}

// DeleteFromDependentJobs
func (j *Job) DeleteFromDependentJobs(cache JobCache) error {
	j.lock.RLock()
	defer j.lock.RUnlock()

	for _, id := range j.DependentJobs {
		childJob, err := cache.Get(id)
		if err != nil {
			return err
		}

		// If there are no other parent jobs, delete this job.
		if len(childJob.ParentJobs) == 1 {
			cache.Delete(childJob.Id)
			continue
		}

		childJob.lock.Lock()

		ndx := 0
		for i, id := range childJob.ParentJobs {
			if id == j.Id {
				ndx = i
				break
			}
		}
		childJob.ParentJobs = append(
			childJob.ParentJobs[:ndx], childJob.ParentJobs[ndx+1:]...,
		)

		childJob.lock.Unlock()

	}

	return nil
}

func (j *Job) Run(cache JobCache) {
	// Schedule next run
	j.lock.RLock()
	jobRunner := &JobRunner{job: j, meta: j.Metadata}
	j.lock.RUnlock()
	newStat, newMeta, err := jobRunner.Run(cache)
	if err != nil {
		log.Errorf("Error running job: %s", err)
	}

	j.lock.Lock()
	j.Metadata = newMeta
	if newStat != nil {
		j.Stats = append(j.Stats, newStat)
	}

	if j.ShouldStartWaiting() {
		go j.StartWaiting(cache)
	} else {
		j.IsDone = true
	}

	j.lock.Unlock()
}

func (j *Job) StopTimer() {
	j.lock.Lock()
	defer j.lock.Unlock()

	if j.jobTimer != nil {
		j.jobTimer.Stop()
	}
}

func (j *Job) RunCmd() error {
	j.lock.RLock()
	defer j.lock.RUnlock()

	jobRunner := &JobRunner{job: j}
	return jobRunner.runCmd()
}

func (j *Job) hasFixedRepetitions() bool {
	if j.timesToRepeat != -1 {
		return true
	}
	return false
}

func (j *Job) ShouldStartWaiting() bool {
	if j.Disabled {
		return false
	}

	if j.hasFixedRepetitions() && int(j.timesToRepeat) < len(j.Stats) {
		return false
	}
	return true
}

func (j *Job) validation() error {
	var err error
	if j.JobType == LocalJob && (j.Name == "" || j.Command == "") {
		err = ErrInvalidJob
	} else if j.JobType == RemoteJob && (j.Name == "" || j.RemoteProperties.Url == "") {
		err = ErrInvalidRemoteJob
	} else if j.JobType != LocalJob && j.JobType != RemoteJob {
		err = ErrInvalidJobType
	} else {
		return nil
	}
	log.Errorf(err.Error())
	return err
}
<commit_msg>If cache was not deleted for one-off jobs return nil so kala can startup properly.<commit_after>package job

import (
	"bytes"
	"encoding/gob"
	"errors"
	"fmt"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/ajvb/kala/utils/iso8601"

	log "github.com/Sirupsen/logrus"
	"github.com/nu7hatch/gouuid"
)

var (
	RFC3339WithoutTimezone = "2006-01-02T15:04:05"

	ErrInvalidJob       = errors.New("Invalid Local Job. Job's must contain a Name and a Command field")
	ErrInvalidRemoteJob = errors.New("Invalid Remote Job. Job's must contain a Name and a url field")
	ErrInvalidJobType   = errors.New("Invalid Job type. Types supported: 0 for local and 1 for remote")
)

type Job struct {
	Name string `json:"name"`
	Id   string `json:"id"`

	// Command to run
	// e.g. "bash /path/to/my/script.sh"
	Command string `json:"command"`

	// Email of the owner of this job
	// e.g. "admin@example.com"
	Owner string `json:"owner"`

	// Is this job disabled?
	Disabled bool `json:"disabled"`

	// Jobs that are dependent upon this one will be run after this job runs.
	DependentJobs []string `json:"dependent_jobs"`

	// List of ids of jobs that this job is dependent upon.
	ParentJobs []string `json:"parent_jobs"`

	// ISO 8601 String
	// e.g. "R/2014-03-08T20:00:00.000Z/PT2H"
	Schedule     string `json:"schedule"`
	scheduleTime time.Time
	// ISO 8601 Duration struct, used for scheduling
	// job after each run.
	delayDuration *iso8601.Duration

	// Number of times to schedule this job after the
	// first run.
	timesToRepeat int64

	// Number of times to retry on failed attempt for each run.
	Retries uint `json:"retries"`

	// Duration in which it is safe to retry the Job.
	Epsilon         string `json:"epsilon"`
	epsilonDuration *iso8601.Duration

	jobTimer  *time.Timer
	NextRunAt time.Time `json:"next_run_at"`

	// Meta data about successful and failed runs.
	Metadata Metadata `json:"metadata"`

	// Type of the job
	JobType jobType `json:"type"`

	// Custom properties for the remote job type
	RemoteProperties RemoteProperties `json:"remote_properties"`

	// Collection of Job Stats
	Stats []*JobStat `json:"stats"`

	lock sync.RWMutex

	// Says if a job has been executed right numbers of time
	// and should not been executed again in the future
	IsDone bool `json:"is_done"`
}

type jobType int

const (
	LocalJob jobType = iota
	RemoteJob
)

// RemoteProperties Custom properties for the remote job type
type RemoteProperties struct {
	Url    string `json:"url"`
	Method string `json:"method"`

	// A body to attach to the http request
	Body string `json:"body"`

	// A list of headers to add to http request (e.g. [{"key": "charset", "value": "UTF-8"}])
	Headers http.Header `json:"headers"`

	// A timeout property for the http request in seconds
	Timeout int `json:"timeout"`

	// A list of expected response codes (e.g. [200, 201])
	ExpectedResponseCodes []int `json:"expected_response_codes"`
}

type Metadata struct {
	SuccessCount     uint      `json:"success_count"`
	LastSuccess      time.Time `json:"last_success"`
	ErrorCount       uint      `json:"error_count"`
	LastError        time.Time `json:"last_error"`
	LastAttemptedRun time.Time `json:"last_attempted_run"`
}

// Bytes returns the byte representation of the Job.
func (j Job) Bytes() ([]byte, error) {
	buff := new(bytes.Buffer)
	enc := gob.NewEncoder(buff)
	err := enc.Encode(j)
	if err != nil {
		return nil, err
	}
	return buff.Bytes(), nil
}

// NewFromBytes returns a Job instance from a byte representation.
func NewFromBytes(b []byte) (*Job, error) {
	j := &Job{}

	buf := bytes.NewBuffer(b)
	err := gob.NewDecoder(buf).Decode(j)
	if err != nil {
		return nil, err
	}

	return j, nil
}

// Init fills in the protected fields and parses the iso8601 notation.
// It also adds the job to the Cache
func (j *Job) Init(cache JobCache) error {
	j.lock.Lock()
	defer j.lock.Unlock()

	//validate job type and params
	err := j.validation()
	if err != nil {
		return err
	}

	u4, err := uuid.NewV4()
	if err != nil {
		log.Errorf("Error occured when generating uuid: %s", err)
		return err
	}
	j.Id = u4.String()

	// Add Job to the cache.
	err = cache.Set(j)
	if err != nil {
		return err
	}

	if len(j.ParentJobs) != 0 {
		// Add new job to parent jobs
		for _, p := range j.ParentJobs {
			parentJob, err := cache.Get(p)
			if err != nil {
				return err
			}
			parentJob.DependentJobs = append(parentJob.DependentJobs, j.Id)
		}

		return nil
	}

	// TODO: Delete from cache after running.
	if j.Schedule == "" {
		// If schedule is empty, its a one-off job.
		go j.Run(cache)
		return nil
	}

	j.lock.Unlock()
	err = j.InitDelayDuration(true)
	j.lock.Lock()
	if err != nil {
		j.lock.Unlock()
		cache.Delete(j.Id)
		j.lock.Lock()
		return err
	}

	j.lock.Unlock()
	j.StartWaiting(cache)

	j.lock.Lock()

	return nil
}

// InitDelayDuration is used to parsed the iso8601 Schedule notation into its relevent fields in the Job struct.
// If checkTime is true, then it will return an error if the Scheduled time has passed.
func (j *Job) InitDelayDuration(checkTime bool) error {
	j.lock.Lock()
	defer j.lock.Unlock()

	if j.Schedule == "" {
		return nil
	}

	var err error
	splitTime := strings.Split(j.Schedule, "/")
	if len(splitTime) != 3 {
		return fmt.Errorf(
			"Schedule not formatted correctly. Should look like: R/2014-03-08T20:00:00Z/PT2H",
		)
	}

	// Handle Repeat Amount
	if splitTime[0] == "R" {
		// Repeat forever
		j.timesToRepeat = -1
	} else {
		j.timesToRepeat, err = strconv.ParseInt(strings.Split(splitTime[0], "R")[1], 10, 0)
		if err != nil {
			log.Errorf("Error converting timesToRepeat to an int: %s", err)
			return err
		}
	}
	log.Debugf("timesToRepeat: %d", j.timesToRepeat)

	j.scheduleTime, err = time.Parse(time.RFC3339, splitTime[1])
	if err != nil {
		j.scheduleTime, err = time.Parse(RFC3339WithoutTimezone, splitTime[1])
		if err != nil {
			log.Errorf("Error converting scheduleTime to a time.Time: %s", err)
			return err
		}
	}
	if checkTime {
		if (time.Duration(j.scheduleTime.UnixNano() - time.Now().UnixNano())) < 0 {
			return fmt.Errorf("Schedule time has passed on Job with id of %s", j.Id)
		}
	}
	log.Debugf("Schedule Time: %s", j.scheduleTime)

	if j.timesToRepeat != 0 {
		j.delayDuration, err = iso8601.FromString(splitTime[2])
		if err != nil {
			log.Errorf("Error converting delayDuration to a iso8601.Duration: %s", err)
			return err
		}
		log.Debugf("Delay Duration: %s", j.delayDuration.ToDuration())
	}

	if j.Epsilon != "" {
		j.epsilonDuration, err = iso8601.FromString(j.Epsilon)
		if err != nil {
			log.Errorf("Error converting j.Epsilon to iso8601.Duration: %s", err)
			return err
		}
	}

	return nil
}

// StartWaiting begins a timer for when it should execute the Jobs .Run() method.
func (j *Job) StartWaiting(cache JobCache) {
	waitDuration := j.GetWaitDuration()

	j.lock.Lock()
	defer j.lock.Unlock()

	log.Infof("Job Scheduled to run in: %s", waitDuration)

	j.NextRunAt = time.Now().Add(waitDuration)

	jobRun := func() { j.Run(cache) }
	j.jobTimer = time.AfterFunc(waitDuration, jobRun)
}

func (j *Job) GetWaitDuration() time.Duration {
	j.lock.RLock()
	defer j.lock.RUnlock()

	waitDuration := time.Duration(j.scheduleTime.UnixNano() - time.Now().UnixNano())

	if waitDuration < 0 {
		if j.timesToRepeat == 0 {
			return 0
		}

		if j.Metadata.LastAttemptedRun.IsZero() {
			waitDuration = j.delayDuration.ToDuration()
		} else {
			lastRun := j.Metadata.LastAttemptedRun
			// Needs to be recalculated each time because of Months.
			lastRun = lastRun.Add(j.delayDuration.ToDuration())
			waitDuration = lastRun.Sub(time.Now())
		}
	}

	return waitDuration
}

// Disable stops the job from running by stopping its jobTimer. It also sets Job.Disabled to true,
// which is reflected in the UI.
func (j *Job) Disable() {
	j.lock.Lock()
	defer j.lock.Unlock()

	if j.jobTimer != nil {
		j.jobTimer.Stop()
	}
	j.Disabled = true
}

func (j *Job) Enable(cache JobCache) {
	j.lock.Lock()
	defer j.lock.Unlock()

	if j.jobTimer != nil && j.Disabled {
		go j.StartWaiting(cache)
	}
	j.Disabled = false
}

// DeleteFromParentJobs goes through and deletes the current job from any parent jobs.
func (j *Job) DeleteFromParentJobs(cache JobCache) error {
	j.lock.Lock()
	defer j.lock.Unlock()

	for _, p := range j.ParentJobs {
		parentJob, err := cache.Get(p)

		if err != nil {
			return err
		}

		parentJob.lock.Lock()

		ndx := 0
		for i, id := range parentJob.DependentJobs {
			if id == j.Id {
				ndx = i
				break
			}
		}
		parentJob.DependentJobs = append(
			parentJob.DependentJobs[:ndx], parentJob.DependentJobs[ndx+1:]...,
		)

		parentJob.lock.Unlock()
	}

	return nil
}

// DeleteFromDependentJobs
func (j *Job) DeleteFromDependentJobs(cache JobCache) error {
	j.lock.RLock()
	defer j.lock.RUnlock()

	for _, id := range j.DependentJobs {
		childJob, err := cache.Get(id)
		if err != nil {
			return err
		}

		// If there are no other parent jobs, delete this job.
		if len(childJob.ParentJobs) == 1 {
			cache.Delete(childJob.Id)
			continue
		}

		childJob.lock.Lock()

		ndx := 0
		for i, id := range childJob.ParentJobs {
			if id == j.Id {
				ndx = i
				break
			}
		}
		childJob.ParentJobs = append(
			childJob.ParentJobs[:ndx], childJob.ParentJobs[ndx+1:]...,
		)

		childJob.lock.Unlock()

	}

	return nil
}

func (j *Job) Run(cache JobCache) {
	// Schedule next run
	j.lock.RLock()
	jobRunner := &JobRunner{job: j, meta: j.Metadata}
	j.lock.RUnlock()
	newStat, newMeta, err := jobRunner.Run(cache)
	if err != nil {
		log.Errorf("Error running job: %s", err)
	}

	j.lock.Lock()
	j.Metadata = newMeta
	if newStat != nil {
		j.Stats = append(j.Stats, newStat)
	}

	if j.ShouldStartWaiting() {
		go j.StartWaiting(cache)
	} else {
		j.IsDone = true
	}

	j.lock.Unlock()
}

func (j *Job) StopTimer() {
	j.lock.Lock()
	defer j.lock.Unlock()

	if j.jobTimer != nil {
		j.jobTimer.Stop()
	}
}

func (j *Job) RunCmd() error {
	j.lock.RLock()
	defer j.lock.RUnlock()

	jobRunner := &JobRunner{job: j}
	return jobRunner.runCmd()
}

func (j *Job) hasFixedRepetitions() bool {
	if j.timesToRepeat != -1 {
		return true
	}
	return false
}

func (j *Job) ShouldStartWaiting() bool {
	if j.Disabled {
		return false
	}

	if j.hasFixedRepetitions() && int(j.timesToRepeat) < len(j.Stats) {
		return false
	}
	return true
}

func (j *Job) validation() error {
	var err error
	if j.JobType == LocalJob && (j.Name == "" || j.Command == "") {
		err = ErrInvalidJob
	} else if j.JobType == RemoteJob && (j.Name == "" || j.RemoteProperties.Url == "") {
		err = ErrInvalidRemoteJob
	} else if j.JobType != LocalJob && j.JobType != RemoteJob {
		err = ErrInvalidJobType
	} else {
		return nil
	}
	log.Errorf(err.Error())
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"io"
	"os"
	"path/filepath"
	"strings"

	fmt "github.com/jhunt/go-ansi"

	"github.com/starkandwayne/shield/plugin"
)

func main() {
	p := FSPlugin{
		Name:    "Local Filesystem Plugin",
		Author:  "Stark & Wayne",
		Version: "1.0.0",
		Features: plugin.PluginFeatures{
			Target: "yes",
			Store:  "no",
		},
		Example: `
{
  "base_dir" : "/path/to/backup"   # REQUIRED

  "include"  : "*.txt",            # UNIX glob of files to include in backup
  "exclude"  : "*.o"               # ... and another for what to exclude
}
`,
		Defaults: `
{
}
`,

		Fields: []plugin.Field{
			plugin.Field{
				Mode:     "target",
				Name:     "base_dir",
				Type:     "abspath",
				Title:    "Base Directory",
				Help:     "Absolute path of the directory to backup.",
				Example:  "/srv/www/htdocs",
				Required: true,
			},
			plugin.Field{
				Mode:  "target",
				Name:  "include",
				Type:  "string",
				Title: "Files to Include",
				Help:  "Only files that match this pattern will be included in the backup archive.  If not specified, all files will be included.",
			},
			plugin.Field{
				Mode:  "target",
				Name:  "exclude",
				Type:  "abspath",
				Title: "Files to Exclude",
				Help:  "Files that match this pattern will be excluded from the backup archive.  If not specified, no files will be excluded.",
			},
			plugin.Field{
				Mode:  "target",
				Name:  "strict",
				Type:  "bool",
				Title: "Strict Mode",
				Help:  "If files go missing while walking the directory, consider that a fatal error.",
			},
		},
	}

	plugin.DEBUG("fs plugin starting up...")
	plugin.Run(p)
}

type FSPlugin plugin.PluginInfo

type FSConfig struct {
	Include  string
	Exclude  string
	BasePath string
	Strict   bool
}

func (cfg *FSConfig) Match(path string) bool {
	if cfg.Exclude != "" {
		if ok, err := filepath.Match(cfg.Exclude, path); ok && err == nil {
			return false
		}
	}
	if cfg.Include != "" {
		if ok, err := filepath.Match(cfg.Include, path); ok && err == nil {
			return true
		}
		return false
	}
	return true
}

func (p FSPlugin) Meta() plugin.PluginInfo {
	return plugin.PluginInfo(p)
}

func getFSConfig(endpoint plugin.ShieldEndpoint) (*FSConfig, error) {
	include, err := endpoint.StringValueDefault("include", "")
	if err != nil {
		return nil, err
	}

	exclude, err := endpoint.StringValueDefault("exclude", "")
	if err != nil {
		return nil, err
	}

	base_dir, err := endpoint.StringValue("base_dir")
	if err != nil {
		return nil, err
	}

	strict, err := endpoint.BooleanValueDefault("strict", false)
	if err != nil {
		return nil, err
	}

	return &FSConfig{
		Include:  include,
		Exclude:  exclude,
		BasePath: base_dir,
		Strict:   strict,
	}, nil
}

func (p FSPlugin) Validate(endpoint plugin.ShieldEndpoint) error {
	var (
		s    string
		b    bool
		err  error
		fail bool
	)

	b, err = endpoint.BooleanValueDefault("strict", false)
	if err != nil {
		fmt.Printf("@R{\u2717 strict    %s}\n", err)
		fail = true
	} else if b {
		fmt.Printf("@G{\u2713 strict}    @C{yes} - files that go missing are considered an error\n")
	} else {
		fmt.Printf("@G{\u2713 strict}    @C{no} (default)\n")
	}

	s, err = endpoint.StringValue("base_dir")
	if err != nil {
		fmt.Printf("@R{\u2717 base_dir  %s}\n", err)
		fail = true
	} else {
		fmt.Printf("@G{\u2713 base_dir}  files in @C{%s} will be backed up\n", s)
	}

	s, err = endpoint.StringValueDefault("include", "")
	if err != nil {
		fmt.Printf("@R{\u2717 include   %s}\n", err)
		fail = true
	} else if s == "" {
		fmt.Printf("@G{\u2713 include}   all files will be included\n")
	} else {
		fmt.Printf("@G{\u2713 include}   only files matching @C{%s} will be backed up\n", s)
	}

	s, err = endpoint.StringValueDefault("exclude", "")
	if err != nil {
		fmt.Printf("@R{\u2717 base_dir  %s}\n", err)
		fail = true
	} else if s == "" {
		fmt.Printf("@G{\u2713 exclude}   no files will be excluded\n")
	} else {
		fmt.Printf("@G{\u2713 exclude}   files matching @C{%s} will be skipped\n", s)
	}

	if fail {
		return fmt.Errorf("fs: invalid configuration")
	}
	return nil
}

func (p FSPlugin) Backup(endpoint plugin.ShieldEndpoint) error {
	cfg, err := getFSConfig(endpoint)
	if err != nil {
		return err
	}

	archive := tar.NewWriter(os.Stdout)
	n := 0
	walker := func(path string, info os.FileInfo, err error) error {
		baseRelative := strings.TrimPrefix(strings.Replace(path, cfg.BasePath, "", 1), "/")
		if baseRelative == "" { /* musta been cfg.BasePath or cfg.BasePath + '/' */
			return nil
		}

		fmt.Fprintf(os.Stderr, " - found '%s' ... ", path)
		if info == nil {
			if _, ok := err.(*os.PathError); !cfg.Strict && ok {
				fmt.Fprintf(os.Stderr, "no longer exists; skipping.\n")
				return nil
			} else {
				fmt.Fprintf(os.Stderr, "FAILED\n")
				return fmt.Errorf("failed to walk %s: %s", path, err)
			}
		}

		if !cfg.Match(info.Name()) {
			fmt.Fprintf(os.Stderr, "ignoring (per include/exclude)\n")
			if info.IsDir() {
				return filepath.SkipDir
			}
			return nil
		}
		n += 1
		fmt.Fprintf(os.Stderr, "ok\n")

		link := ""
		if info.Mode()&os.ModeType == os.ModeSymlink {
			link, err = os.Readlink(path)
			if err != nil {
				return err
			}
		}
		header, err := tar.FileInfoHeader(info, link)
		if err != nil {
			return err
		}

		header.Name = baseRelative
		if err := archive.WriteHeader(header); err != nil {
			return err
		}

		if info.Mode().IsDir() || link != "" {
			return nil
		}

		if info.Mode().IsRegular() {
			f, err := os.Open(path)
			if err != nil {
				return err
			}
			io.Copy(archive, f)
			return nil
		}

		return fmt.Errorf("unable to archive special file '%s'", path)
	}

	fmt.Fprintf(os.Stderr, "backing up files in '%s'...\n", cfg.BasePath)
	if err := filepath.Walk(cfg.BasePath, walker); err != nil {
		return err
	}
	fmt.Fprintf(os.Stderr, "done; found %d files / directories to archive...\n\n", n)

	return archive.Close()
}

func (p FSPlugin) Restore(endpoint plugin.ShieldEndpoint) error {
	cfg, err := getFSConfig(endpoint)
	if err != nil {
		return err
	}

	if err := os.MkdirAll(cfg.BasePath, 0777); err != nil {
		return err
	}

	n := 0
	archive := tar.NewReader(os.Stdin)
	for {
		header, err := archive.Next()
		if err != nil {
			if err == io.EOF {
				return nil
			}
			return err
		}

		info := header.FileInfo()
		path := fmt.Sprintf("%s/%s", cfg.BasePath, header.Name)
		n += 1
		fmt.Fprintf(os.Stderr, " - restoring '%s'... ", path)
		if info.Mode().IsDir() {
			if err := os.MkdirAll(path, 0777); err != nil {
				fmt.Fprintf(os.Stderr, "FAILED (could not create directory)\n")
				return err
			}
			fmt.Fprintf(os.Stderr, "created directory\n")

		} else if info.Mode().IsRegular() {
			f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, info.Mode())
			if err != nil {
				fmt.Fprintf(os.Stderr, "FAILED (could not create new file)\n")
				return err
			}
			if _, err := io.Copy(f, archive); err != nil {
				fmt.Fprintf(os.Stderr, "FAILED (could not copy data to disk)\n")
				return err
			}
			fmt.Fprintf(os.Stderr, "created file\n")

		} else {
			fmt.Fprintf(os.Stderr, "FAILED (not a regular file or a directory)\n")
			return fmt.Errorf("unable to unpack special file '%s'", path)
		}

		/* put things back the way they were... */
		if err := os.Chtimes(path, header.AccessTime, header.ModTime); err != nil {
			fmt.Fprintf(os.Stderr, "FAILED (could not set atime / mtime / ctime)\n")
			return err
		}
		if err := os.Chown(path, header.Uid, header.Gid); err != nil {
			fmt.Fprintf(os.Stderr, "FAILED (could not set user ownership)\n")
			return err
		}
		if err := os.Chmod(path, info.Mode()); err != nil {
			fmt.Fprintf(os.Stderr, "FAILED (could not set group ownership)\n")
			return err
		}

		fmt.Fprintf(os.Stderr, "ok\n")
	}

	fmt.Fprintf(os.Stderr, "done; restored %d files / directories...\n\n", n)
	return nil
}

func (p FSPlugin) Store(endpoint plugin.ShieldEndpoint) (string, int64, error) {
	return "", 0, plugin.UNIMPLEMENTED
}

func (p FSPlugin) Retrieve(endpoint plugin.ShieldEndpoint, file string) error {
	return plugin.UNIMPLEMENTED
}

func (p FSPlugin) Purge(endpoint plugin.ShieldEndpoint, file string) error {
	return plugin.UNIMPLEMENTED
}
<commit_msg>fs: exclude value is a string, not an abspath<commit_after>package main

import (
	"archive/tar"
	"io"
	"os"
	"path/filepath"
	"strings"

	fmt "github.com/jhunt/go-ansi"

	"github.com/starkandwayne/shield/plugin"
)

func main() {
	p := FSPlugin{
		Name:    "Local Filesystem Plugin",
		Author:  "Stark & Wayne",
		Version: "1.0.0",
		Features: plugin.PluginFeatures{
			Target: "yes",
			Store:  "no",
		},
		Example: `
{
  "base_dir" : "/path/to/backup"   # REQUIRED

  "include"  : "*.txt",            # UNIX glob of files to include in backup
  "exclude"  : "*.o"               # ... and another for what to exclude
}
`,
		Defaults: `
{
}
`,

		Fields: []plugin.Field{
			plugin.Field{
				Mode:     "target",
				Name:     "base_dir",
				Type:     "abspath",
				Title:    "Base Directory",
				Help:     "Absolute path of the directory to backup.",
				Example:  "/srv/www/htdocs",
				Required: true,
			},
			plugin.Field{
				Mode:  "target",
				Name:  "include",
				Type:  "string",
				Title: "Files to Include",
				Help:  "Only files that match this pattern will be included in the backup archive.  If not specified, all files will be included.",
			},
			plugin.Field{
				Mode:  "target",
				Name:  "exclude",
				Type:  "string",
				Title: "Files to Exclude",
				Help:  "Files that match this pattern will be excluded from the backup archive.  If not specified, no files will be excluded.",
			},
			plugin.Field{
				Mode:  "target",
				Name:  "strict",
				Type:  "bool",
				Title: "Strict Mode",
				Help:  "If files go missing while walking the directory, consider that a fatal error.",
			},
		},
	}

	plugin.DEBUG("fs plugin starting up...")
	plugin.Run(p)
}

type FSPlugin plugin.PluginInfo

type FSConfig struct {
	Include  string
	Exclude  string
	BasePath string
	Strict   bool
}

func (cfg *FSConfig) Match(path string) bool {
	if cfg.Exclude != "" {
		if ok, err := filepath.Match(cfg.Exclude, path); ok && err == nil {
			return false
		}
	}
	if cfg.Include != "" {
		if ok, err := filepath.Match(cfg.Include, path); ok && err == nil {
			return true
		}
		return false
	}
	return true
}

func (p FSPlugin) Meta() plugin.PluginInfo {
	return plugin.PluginInfo(p)
}

func getFSConfig(endpoint plugin.ShieldEndpoint) (*FSConfig, error) {
	include, err := endpoint.StringValueDefault("include", "")
	if err != nil {
		return nil, err
	}

	exclude, err := endpoint.StringValueDefault("exclude", "")
	if err != nil {
		return nil, err
	}

	base_dir, err := endpoint.StringValue("base_dir")
	if err != nil {
		return nil, err
	}

	strict, err := endpoint.BooleanValueDefault("strict", false)
	if err != nil {
		return nil, err
	}

	return &FSConfig{
		Include:  include,
		Exclude:  exclude,
		BasePath: base_dir,
		Strict:   strict,
	}, nil
}

func (p FSPlugin) Validate(endpoint plugin.ShieldEndpoint) error {
	var (
		s    string
		b    bool
		err  error
		fail bool
	)

	b, err = endpoint.BooleanValueDefault("strict", false)
	if err != nil {
		fmt.Printf("@R{\u2717 strict    %s}\n", err)
		fail = true
	} else if b {
		fmt.Printf("@G{\u2713 strict}    @C{yes} - files that go missing are considered an error\n")
	} else {
		fmt.Printf("@G{\u2713 strict}    @C{no} (default)\n")
	}

	s, err = endpoint.StringValue("base_dir")
	if err != nil {
		fmt.Printf("@R{\u2717 base_dir  %s}\n", err)
		fail = true
	} else {
		fmt.Printf("@G{\u2713 base_dir}  files in @C{%s} will be backed up\n", s)
	}

	s, err = endpoint.StringValueDefault("include", "")
	if err != nil {
		fmt.Printf("@R{\u2717 include   %s}\n", err)
		fail = true
	} else if s == "" {
		fmt.Printf("@G{\u2713 include}   all files will be included\n")
	} else {
		fmt.Printf("@G{\u2713 include}   only files matching @C{%s} will be backed up\n", s)
	}

	s, err = endpoint.StringValueDefault("exclude", "")
	if err != nil {
		fmt.Printf("@R{\u2717 base_dir  %s}\n", err)
		fail = true
	} else if s == "" {
		fmt.Printf("@G{\u2713 exclude}   no files will be excluded\n")
	} else {
		fmt.Printf("@G{\u2713 exclude}   files matching @C{%s} will be skipped\n", s)
	}

	if fail {
		return fmt.Errorf("fs: invalid configuration")
	}
	return nil
}

func (p FSPlugin) Backup(endpoint plugin.ShieldEndpoint) error {
	cfg, err := getFSConfig(endpoint)
	if err != nil {
		return err
	}

	archive := tar.NewWriter(os.Stdout)
	n := 0
	walker := func(path string, info os.FileInfo, err error) error {
		baseRelative := strings.TrimPrefix(strings.Replace(path, cfg.BasePath, "", 1), "/")
		if baseRelative == "" { /* musta been cfg.BasePath or cfg.BasePath + '/' */
			return nil
		}

		fmt.Fprintf(os.Stderr, " - found '%s' ... ", path)
		if info == nil {
			if _, ok := err.(*os.PathError); !cfg.Strict && ok {
				fmt.Fprintf(os.Stderr, "no longer exists; skipping.\n")
				return nil
			} else {
				fmt.Fprintf(os.Stderr, "FAILED\n")
				return fmt.Errorf("failed to walk %s: %s", path, err)
			}
		}

		if !cfg.Match(info.Name()) {
			fmt.Fprintf(os.Stderr, "ignoring (per include/exclude)\n")
			if info.IsDir() {
				return filepath.SkipDir
			}
			return nil
		}
		n += 1
		fmt.Fprintf(os.Stderr, "ok\n")

		link := ""
		if info.Mode()&os.ModeType == os.ModeSymlink {
			link, err = os.Readlink(path)
			if err != nil {
				return err
			}
		}
		header, err := tar.FileInfoHeader(info, link)
		if err != nil {
			return err
		}

		header.Name = baseRelative
		if err := archive.WriteHeader(header); err != nil {
			return err
		}

		if info.Mode().IsDir() || link != "" {
			return nil
		}

		if info.Mode().IsRegular() {
			f, err := os.Open(path)
			if err != nil {
				return err
			}
			io.Copy(archive, f)
			return nil
		}

		return fmt.Errorf("unable to archive special file '%s'", path)
	}

	fmt.Fprintf(os.Stderr, "backing up files in '%s'...\n", cfg.BasePath)
	if err := filepath.Walk(cfg.BasePath, walker); err != nil {
		return err
	}
	fmt.Fprintf(os.Stderr, "done; found %d files / directories to archive...\n\n", n)

	return archive.Close()
}

func (p FSPlugin) Restore(endpoint plugin.ShieldEndpoint) error {
	cfg, err := getFSConfig(endpoint)
	if err != nil {
		return err
	}

	if err := os.MkdirAll(cfg.BasePath, 0777); err != nil {
		return err
	}

	n := 0
	archive := tar.NewReader(os.Stdin)
	for {
		header, err := archive.Next()
		if err != nil {
			if err == io.EOF {
				return nil
			}
			return err
		}

		info := header.FileInfo()
		path := fmt.Sprintf("%s/%s", cfg.BasePath, header.Name)
		n += 1
		fmt.Fprintf(os.Stderr, " - restoring '%s'... ", path)
		if info.Mode().IsDir() {
			if err := os.MkdirAll(path, 0777); err != nil {
				fmt.Fprintf(os.Stderr, "FAILED (could not create directory)\n")
				return err
			}
			fmt.Fprintf(os.Stderr, "created directory\n")

		} else if info.Mode().IsRegular() {
			f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, info.Mode())
			if err != nil {
				fmt.Fprintf(os.Stderr, "FAILED (could not create new file)\n")
				return err
			}
			if _, err := io.Copy(f, archive); err != nil {
				fmt.Fprintf(os.Stderr, "FAILED (could not copy data to disk)\n")
				return err
			}
			fmt.Fprintf(os.Stderr, "created file\n")

		} else {
			fmt.Fprintf(os.Stderr, "FAILED (not a regular file or a directory)\n")
			return fmt.Errorf("unable to unpack special file '%s'", path)
		}

		/* put things back the way they were... */
		if err := os.Chtimes(path, header.AccessTime, header.ModTime); err != nil {
			fmt.Fprintf(os.Stderr, "FAILED (could not set atime / mtime / ctime)\n")
			return err
		}
		if err := os.Chown(path, header.Uid, header.Gid); err != nil {
			fmt.Fprintf(os.Stderr, "FAILED (could not set user ownership)\n")
			return err
		}
		if err := os.Chmod(path, info.Mode()); err != nil {
			fmt.Fprintf(os.Stderr, "FAILED (could not set group ownership)\n")
			return err
		}

		fmt.Fprintf(os.Stderr, "ok\n")
	}

	fmt.Fprintf(os.Stderr, "done; restored %d files / directories...\n\n", n)
	return nil
}

func (p FSPlugin) Store(endpoint plugin.ShieldEndpoint) (string, int64, error) {
	return "", 0, plugin.UNIMPLEMENTED
}

func (p FSPlugin) Retrieve(endpoint plugin.ShieldEndpoint, file string) error {
	return plugin.UNIMPLEMENTED
}

func (p FSPlugin) Purge(endpoint plugin.ShieldEndpoint, file string) error {
	return plugin.UNIMPLEMENTED
}
<|endoftext|>


<commit_before>package gautomator

import (
	"fmt"
	"github.com/gonum/matrix/mat64" // Matrix
	"io"
	"time"
)

// A task is an action executed by a module
type Task struct {
	Id     int      `json:"id"`
	Origin string   `json:"origin"`
	Name   string   `json:"name"` //the task name
	Node   string   `json:"node"` // The node name
	Module string   `json:"module"`
	Args   []string `json:"args"`
	Status int      `json:"status"` //-3: queued
	// -2 Advertized (infored that the dependencies are done)
	// -1: running
	// >=0 : return code
	StartTime      time.Time `json:"startTime"`
	EndTime        time.Time `json:"endTime"`
	TaskCanRunChan chan bool // true: run, false: wait
}

// This is the structure corresponding to the "dot-graph" of a task list
// We store the nodes in a map
// The index is the source node
type TaskGraphStructure struct {
	Tasks           map[int]*Task
	DegreeMatrix    *mat64.Dense
	AdjacencyMatrix *mat64.Dense // Row id is the map id of the source task
	// Col id is the map id of the destination task
}

func (this *TaskGraphStructure) PrintAdjacencyMatrix() {
	rowSize, colSize := this.AdjacencyMatrix.Dims()
	fmt.Printf("  ")
	for c := 0; c < colSize; c++ {
		fmt.Printf("%v ", this.Tasks[c].Name)
	}
	fmt.Printf("\n")
	for r := 0; r < rowSize; r++ {
		fmt.Printf("%v ", this.Tasks[r].Name)
		for c := 0; c < colSize; c++ {
			fmt.Printf("%v ", this.AdjacencyMatrix.At(r, c))
		}
		fmt.Printf("\n")
	}
}

func (this *TaskGraphStructure) PrintDegreeMatrix() {
	rowSize, colSize := this.DegreeMatrix.Dims()
	for r := 0; r < rowSize; r++ {
		for c := 0; c < colSize; c++ {
			fmt.Printf("%v ", this.DegreeMatrix.At(r, c))
		}
		fmt.Printf("\n")
	}
}

func NewTask() *Task {
	return &Task{
		-1,
		"null",
		"null",
		"null",
		"dummy",
		make([]string, 1),
		-3,
		time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC),
		time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC),
		make(chan bool),
	}

}
func NewTaskGraphStructure() *TaskGraphStructure {
	return &TaskGraphStructure{
		make(map[int]*Task, 0),
		mat64.NewDense(0, 0, nil),
		mat64.NewDense(0, 0, nil),
	}
}

// Returns a combination of the current structure
// and the one passed as argument
func (this *TaskGraphStructure) AugmentTaskStructure(taskStructure *TaskGraphStructure) *TaskGraphStructure {
	// merging adjacency matrix
	initialRowLen, initialColLen := this.AdjacencyMatrix.Dims()
	addedRowLen, addedColLen := taskStructure.AdjacencyMatrix.Dims()
	this.AdjacencyMatrix = mat64.DenseCopyOf(this.AdjacencyMatrix.Grow(addedRowLen, addedColLen))
	//a, b := this.AdjacencyMatrix.Dims()
	for r := 0; r < initialRowLen+addedRowLen; r++ {
		for c := 0; c < initialColLen+addedColLen; c++ {
			switch {
			case r < initialRowLen && c < initialColLen:
				// If we are in the original matrix: do nothing
			case r < initialRowLen && c > initialColLen:
				// If outside, put some zero
				this.AdjacencyMatrix.Set(r, c, float64(0))
			case r > initialRowLen && c < initialColLen:
				// If outside, put some zero
				this.AdjacencyMatrix.Set(r, c, float64(0))
			case r >= initialRowLen && c >= initialColLen:
				// Add the new matrix
				this.AdjacencyMatrix.Set(r, c, taskStructure.AdjacencyMatrix.At(r-initialRowLen, c-initialColLen))
			}
		}
	}
	// merging degree matrix
	initialRowLen, initialColLen = this.DegreeMatrix.Dims()
	addedRowLen, addedColLen = taskStructure.DegreeMatrix.Dims()
	this.DegreeMatrix = mat64.DenseCopyOf(this.DegreeMatrix.Grow(addedRowLen, addedColLen))
	for r := 0; r < initialRowLen+addedRowLen; r++ {
		for c := 0; c < initialColLen+addedColLen; c++ {
			switch {
			case r < initialRowLen && c < initialColLen:
				// If we are in the original matrix: do nothing
			case r < initialRowLen && c > initialColLen:
				// If outside, set zero
				this.DegreeMatrix.Set(r, c, float64(0))
			case r > initialRowLen && c < initialColLen:
				// If outside, set zero
				this.DegreeMatrix.Set(r, c, float64(0))
			case r >= initialRowLen && c >= initialColLen:
				// Add the new matrix
				this.DegreeMatrix.Set(r, c, taskStructure.DegreeMatrix.At(r-initialRowLen, c-initialColLen))
			}
		}
	}
	actualSize := len(this.Tasks)
	for i, task := range taskStructure.Tasks {
		task.Id = actualSize + i
		this.Tasks[actualSize+i] = task
	}
	return this
}

func (this *TaskGraphStructure) getTaskFromName(name string) []int {
	indexA := make([]int,1)
	indexA[0] = -1
	for _ , task := range this.Tasks {
		if task.Name == name {
		    if indexA[0] == -1 {
			indexA = append(indexA[1:],task.Id)
		    } else {
			indexA = append(indexA,task.Id)
		    }
		}
	}
	return indexA
}

func colSum(matrix *mat64.Dense, colId int) float64 {
	row, _ := matrix.Dims()
	sum := float64(0)
	for r := 0; r < row; r++ {
		sum += matrix.At(r, colId)
	}
	return sum
}

func rowSum(matrix *mat64.Dense, rowId int) float64 {
	_, col := matrix.Dims()
	sum := float64(0)
	for c := 0; c < col; c++ {
		sum += matrix.At(rowId, c)
	}
	return sum
}

// the aim of this function is to find if a task has a subdefinition (aka an origin) and change it
// Example:
// imagine the graphs
// digraph bla {
//    a -> b;
//    b -> c;
// }
// digraph b {
//    alpha -> gamma;
// }
// then alpha and beta will have "b" as Origin.
// therefore we should add a link in the AdjacencyMatix and in the DegreeMatrix
func (this *TaskGraphStructure) Relink() *TaskGraphStructure {
	// IN this array we store the row,col on which we set 1
	backup := make(map[string][]int, 0)
	_, col := this.AdjacencyMatrix.Dims()
	for _, task := range this.Tasks {
		if colSum(this.AdjacencyMatrix, task.Id) == 0 {
			id := this.getTaskFromName(task.Origin)
			// TODO There should be only one task, otherwise display an error
			if id[0] != -1 {
				// Task is a meta task
				this.Tasks[id[0]].Module = "meta"
				this.AdjacencyMatrix.Set(id[0], task.Id, float64(1))
				backup[task.Origin] = append(backup[task.Origin], id[0], task.Id)
			}
		}
		if rowSum(this.AdjacencyMatrix, task.Id) == 0 {
			id := this.getTaskFromName(task.Origin)
			// TODO There should be only one task, otherwise display an error
			if id[0] != -1 {
				for c := 0; c < col; c++ {
					add := true
					for counter := 0; counter < len(backup[task.Origin])-1; counter += 2 {
						if backup[task.Origin][counter] == id[0] && backup[task.Origin][counter+1] == c {
							add = false
						}
					}
					if add == true && this.Tasks[c].Origin != task.Origin {
						this.AdjacencyMatrix.Set(task.Id, c, this.AdjacencyMatrix.At(task.Id, c)+this.AdjacencyMatrix.At(id[0], c))
					}
				}
			}
		}
	}
	//TODO: complete the degreematrix
	return this
}

// Duplicate the task "id"
// Returns the id of the new task and the whole structure
func (this *TaskGraphStructure) DuplicateTask(name string) []int {
    /*
	row, _ := this.AdjacencyMatrix.Dims()
	// Add the task to the list
	Ids := this.getTaskFromName(name)
	id := origin.Id
	newId := row
	newTask := origin
	newTask.Id = newId
	log.Println("Step3")
	this.Tasks[newId] = newTask
	// Adjust the AdjacencyMatrix
	log.Println("Step4")
	this.AdjacencyMatrix = mat64.DenseCopyOf(this.AdjacencyMatrix.Grow(1, 1))
	// Copy the row 'id' to row 'newId'
	log.Println("Step5")
	for r := 0; r < newId; r++ {
	    log.Printf("Step6: r:%v id=%v newId=%v",r,id,newId)
		this.AdjacencyMatrix.Set(r, newId, this.AdjacencyMatrix.At(r, id))
	}
	// Copy the col 'id' to col 'newId'
	for c := 0; c < newId; c++ {
		log.Println("Step7")
		this.AdjacencyMatrix.Set(newId, c, this.AdjacencyMatrix.At(id, c))
	}
	log.Println("Step8")
	return newId
	*/
	return nil
}

// This function print the dot file associated with the graph
func (this *TaskGraphStructure) PrintDot(w io.Writer) {
	fmt.Fprintln(w, "digraph G {")
	// Writing node definition
	for _, task := range this.Tasks {
		fmt.Fprintf(w, "\t\"%v\" [\n", task.Id)
		fmt.Fprintf(w, "\t\tid = \"%v\"\n", task.Id)
		if task.Module == "meta" {
			fmt.Fprintln(w, "\t\tshape=diamond")
			fmt.Fprintf(w, "\t\tlabel=\"%v\"", task.Name)
		} else {
			fmt.Fprintf(w, "\t\tlabel = \"<name>%v|<node>%v|<module>%v\"\n", task.Name, task.Node, task.Module)
			fmt.Fprintf(w, "\t\tshape = \"record\"\n")
		}
		fmt.Fprintf(w, "\t];\n")
	}
	row, col := this.AdjacencyMatrix.Dims()
	for r := 0; r < row; r++ {
		for c := 0; c < col; c++ {
			if this.AdjacencyMatrix.At(r, c) == 1 {
				fmt.Fprintf(w, "\t%v -> %v\n", this.Tasks[r].Id, this.Tasks[c].Id)
			}
		}
	}
	fmt.Fprintln(w, "}")
}

// Return a structure of all the task with the given origin
func (this *TaskGraphStructure) GetSubstructure(origin string) *TaskGraphStructure {
	subTaskStructure := NewTaskGraphStructure()
	index := 0
	tasksToExtract := make(map[int]*Task, 0)
	for _, task := range this.Tasks {
		if task.Origin == origin {
			//fmt.Printf("Adding %v(%v) at index:%v\n", task.Name, task.Id, index)
			tasksToExtract[index] = task
			index += 1
		}
	}
	// Create the matrix of the correct size
	size := len(tasksToExtract)
	if size > 0 {
		subTaskStructure.AdjacencyMatrix = mat64.NewDense(size, size, nil)
		subTaskStructure.DegreeMatrix = mat64.NewDense(size, size, nil)
		for i := 0; i < size; i++ {
			task := tasksToExtract[i]
			//fmt.Printf("Task with ID:%v and name:%v will have id:%v\n", task.Id, task.Name, i)
			// Construct the AdjacencyMatrix line by line
			for col := 0; col < size; col++ {
				task2 := tasksToExtract[col]
				//fmt.Printf("Setting %v,%v with value from %v,%v\n", i, col, task.Id, task2.Id)
				subTaskStructure.AdjacencyMatrix.Set(i, col, this.AdjacencyMatrix.At(task.Id, task2.Id))
			}
			subTaskStructure.DegreeMatrix.Set(i, i, this.DegreeMatrix.At(task.Id, task.Id))
			subTaskStructure.Tasks[i] = NewTask()
			subTaskStructure.Tasks[i].Name = task.Name
			subTaskStructure.Tasks[i].Module = task.Module
			subTaskStructure.Tasks[i].Args = task.Args
			subTaskStructure.Tasks[i].Origin = task.Origin
			subTaskStructure.Tasks[i].Id = i
		}
		//subTaskStructure.PrintAdjacencyMatrix()
		return subTaskStructure
	} else {
		return nil
	}
}
<commit_msg>Duplicate task to be tested<commit_after>package gautomator

import (
	"fmt"
	"github.com/gonum/matrix/mat64" // Matrix
	"io"
	"time"
)

// A task is an action executed by a module
type Task struct {
	Id     int      `json:"id"`
	Origin string   `json:"origin"`
	Name   string   `json:"name"` //the task name
	Node   string   `json:"node"` // The node name
	Module string   `json:"module"`
	Args   []string `json:"args"`
	Status int      `json:"status"` //-3: queued
	// -2 Advertized (infored that the dependencies are done)
	// -1: running
	// >=0 : return code
	StartTime      time.Time `json:"startTime"`
	EndTime        time.Time `json:"endTime"`
	TaskCanRunChan chan bool // true: run, false: wait
}

// This is the structure corresponding to the "dot-graph" of a task list
// We store the nodes in a map
// The index is the source node
type TaskGraphStructure struct {
	Tasks           map[int]*Task
	DegreeMatrix    *mat64.Dense
	AdjacencyMatrix *mat64.Dense // Row id is the map id of the source task
	// Col id is the map id of the destination task
}

func (this *TaskGraphStructure) PrintAdjacencyMatrix() {
	rowSize, colSize := this.AdjacencyMatrix.Dims()
	fmt.Printf("  ")
	for c := 0; c < colSize; c++ {
		fmt.Printf("%v ", this.Tasks[c].Name)
	}
	fmt.Printf("\n")
	for r := 0; r < rowSize; r++ {
		fmt.Printf("%v ", this.Tasks[r].Name)
		for c := 0; c < colSize; c++ {
			fmt.Printf("%v ", this.AdjacencyMatrix.At(r, c))
		}
		fmt.Printf("\n")
	}
}

func (this *TaskGraphStructure) PrintDegreeMatrix() {
	rowSize, colSize := this.DegreeMatrix.Dims()
	for r := 0; r < rowSize; r++ {
		for c := 0; c < colSize; c++ {
			fmt.Printf("%v ", this.DegreeMatrix.At(r, c))
		}
		fmt.Printf("\n")
	}
}

func NewTask() *Task {
	return &Task{
		-1,
		"null",
		"null",
		"null",
		"dummy",
		make([]string, 1),
		-3,
		time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC),
		time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC),
		make(chan bool),
	}

}
func NewTaskGraphStructure() *TaskGraphStructure {
	return &TaskGraphStructure{
		make(map[int]*Task, 0),
		mat64.NewDense(0, 0, nil),
		mat64.NewDense(0, 0, nil),
	}
}

// Returns a combination of the current structure
// and the one passed as argument
func (this *TaskGraphStructure) AugmentTaskStructure(taskStructure *TaskGraphStructure) *TaskGraphStructure {
	// merging adjacency matrix
	initialRowLen, initialColLen := this.AdjacencyMatrix.Dims()
	addedRowLen, addedColLen := taskStructure.AdjacencyMatrix.Dims()
	this.AdjacencyMatrix = mat64.DenseCopyOf(this.AdjacencyMatrix.Grow(addedRowLen, addedColLen))
	//a, b := this.AdjacencyMatrix.Dims()
	for r := 0; r < initialRowLen+addedRowLen; r++ {
		for c := 0; c < initialColLen+addedColLen; c++ {
			switch {
			case r < initialRowLen && c < initialColLen:
				// If we are in the original matrix: do nothing
			case r < initialRowLen && c > initialColLen:
				// If outside, put some zero
				this.AdjacencyMatrix.Set(r, c, float64(0))
			case r > initialRowLen && c < initialColLen:
				// If outside, put some zero
				this.AdjacencyMatrix.Set(r, c, float64(0))
			case r >= initialRowLen && c >= initialColLen:
				// Add the new matrix
				this.AdjacencyMatrix.Set(r, c, taskStructure.AdjacencyMatrix.At(r-initialRowLen, c-initialColLen))
			}
		}
	}
	// merging degree matrix
	initialRowLen, initialColLen = this.DegreeMatrix.Dims()
	addedRowLen, addedColLen = taskStructure.DegreeMatrix.Dims()
	this.DegreeMatrix = mat64.DenseCopyOf(this.DegreeMatrix.Grow(addedRowLen, addedColLen))
	for r := 0; r < initialRowLen+addedRowLen; r++ {
		for c := 0; c < initialColLen+addedColLen; c++ {
			switch {
			case r < initialRowLen && c < initialColLen:
				// If we are in the original matrix: do nothing
			case r < initialRowLen && c > initialColLen:
				// If outside, set zero
				this.DegreeMatrix.Set(r, c, float64(0))
			case r > initialRowLen && c < initialColLen:
				// If outside, set zero
				this.DegreeMatrix.Set(r, c, float64(0))
			case r >= initialRowLen && c >= initialColLen:
				// Add the new matrix
				this.DegreeMatrix.Set(r, c, taskStructure.DegreeMatrix.At(r-initialRowLen, c-initialColLen))
			}
		}
	}
	actualSize := len(this.Tasks)
	for i, task := range taskStructure.Tasks {
		task.Id = actualSize + i
		this.Tasks[actualSize+i] = task
	}
	return this
}

func (this *TaskGraphStructure) getTaskFromName(name string) []int {
	indexA := make([]int,1)
	indexA[0] = -1
	for _ , task := range this.Tasks {
		if task.Name == name {
		    if indexA[0] == -1 {
			indexA = append(indexA[1:],task.Id)
		    } else {
			indexA = append(indexA,task.Id)
		    }
		}
	}
	return indexA
}

func colSum(matrix *mat64.Dense, colId int) float64 {
	row, _ := matrix.Dims()
	sum := float64(0)
	for r := 0; r < row; r++ {
		sum += matrix.At(r, colId)
	}
	return sum
}

func rowSum(matrix *mat64.Dense, rowId int) float64 {
	_, col := matrix.Dims()
	sum := float64(0)
	for c := 0; c < col; c++ {
		sum += matrix.At(rowId, c)
	}
	return sum
}

// the aim of this function is to find if a task has a subdefinition (aka an origin) and change it
// Example:
// imagine the graphs
// digraph bla {
//    a -> b;
//    b -> c;
// }
// digraph b {
//    alpha -> gamma;
// }
// then alpha and beta will have "b" as Origin.
// therefore we should add a link in the AdjacencyMatix and in the DegreeMatrix
func (this *TaskGraphStructure) Relink() *TaskGraphStructure {
	// IN this array we store the row,col on which we set 1
	backup := make(map[string][]int, 0)
	_, col := this.AdjacencyMatrix.Dims()
	for _, task := range this.Tasks {
		if colSum(this.AdjacencyMatrix, task.Id) == 0 {
			id := this.getTaskFromName(task.Origin)
			// TODO There should be only one task, otherwise display an error
			if id[0] != -1 {
				// Task is a meta task
				this.Tasks[id[0]].Module = "meta"
				this.AdjacencyMatrix.Set(id[0], task.Id, float64(1))
				backup[task.Origin] = append(backup[task.Origin], id[0], task.Id)
			}
		}
		if rowSum(this.AdjacencyMatrix, task.Id) == 0 {
			id := this.getTaskFromName(task.Origin)
			// TODO There should be only one task, otherwise display an error
			if id[0] != -1 {
				for c := 0; c < col; c++ {
					add := true
					for counter := 0; counter < len(backup[task.Origin])-1; counter += 2 {
						if backup[task.Origin][counter] == id[0] && backup[task.Origin][counter+1] == c {
							add = false
						}
					}
					if add == true && this.Tasks[c].Origin != task.Origin {
						this.AdjacencyMatrix.Set(task.Id, c, this.AdjacencyMatrix.At(task.Id, c)+this.AdjacencyMatrix.At(id[0], c))
					}
				}
			}
		}
	}
	//TODO: complete the degreematrix
	return this
}

// Duplicate the task "id"
// Returns the id of the new task and the whole structure
func (this *TaskGraphStructure) DuplicateTask(name string) []int {
	newIds := make([]int,0)
	Ids := this.getTaskFromName(name)
	for id := range Ids {
	    if id != -1 {
		newId, _ := this.AdjacencyMatrix.Dims()
		newIds = append(newIds,newId)
		newTask := NewTask()
		newTask.Id = newId
		newTask.Origin = this.Tasks[id].Origin
		newTask.Module = this.Tasks[id].Module
		newTask.Node = this.Tasks[id].Node
		newTask.Args = this.Tasks[id].Args
		newTask.Status = this.Tasks[id].Status
		this.AdjacencyMatrix = mat64.DenseCopyOf(this.AdjacencyMatrix.Grow(1, 1))
		for r := 0; r < newId; r++ {
		    this.AdjacencyMatrix.Set(r, newId, this.AdjacencyMatrix.At(r, id))
		}
		// Copy the col 'id' to col 'newId'
		for c := 0; c < newId; c++ {
		    this.AdjacencyMatrix.Set(newId, c, this.AdjacencyMatrix.At(id, c))
		}
	    }
	}
	return newIds
}

// This function print the dot file associated with the graph
func (this *TaskGraphStructure) PrintDot(w io.Writer) {
	fmt.Fprintln(w, "digraph G {")
	// Writing node definition
	for _, task := range this.Tasks {
		fmt.Fprintf(w, "\t\"%v\" [\n", task.Id)
		fmt.Fprintf(w, "\t\tid = \"%v\"\n", task.Id)
		if task.Module == "meta" {
			fmt.Fprintln(w, "\t\tshape=diamond")
			fmt.Fprintf(w, "\t\tlabel=\"%v\"", task.Name)
		} else {
			fmt.Fprintf(w, "\t\tlabel = \"<name>%v|<node>%v|<module>%v\"\n", task.Name, task.Node, task.Module)
			fmt.Fprintf(w, "\t\tshape = \"record\"\n")
		}
		fmt.Fprintf(w, "\t];\n")
	}
	row, col := this.AdjacencyMatrix.Dims()
	for r := 0; r < row; r++ {
		for c := 0; c < col; c++ {
			if this.AdjacencyMatrix.At(r, c) == 1 {
				fmt.Fprintf(w, "\t%v -> %v\n", this.Tasks[r].Id, this.Tasks[c].Id)
			}
		}
	}
	fmt.Fprintln(w, "}")
}

// Return a structure of all the task with the given origin
func (this *TaskGraphStructure) GetSubstructure(origin string) *TaskGraphStructure {
	subTaskStructure := NewTaskGraphStructure()
	index := 0
	tasksToExtract := make(map[int]*Task, 0)
	for _, task := range this.Tasks {
		if task.Origin == origin {
			//fmt.Printf("Adding %v(%v) at index:%v\n", task.Name, task.Id, index)
			tasksToExtract[index] = task
			index += 1
		}
	}
	// Create the matrix of the correct size
	size := len(tasksToExtract)
	if size > 0 {
		subTaskStructure.AdjacencyMatrix = mat64.NewDense(size, size, nil)
		subTaskStructure.DegreeMatrix = mat64.NewDense(size, size, nil)
		for i := 0; i < size; i++ {
			task := tasksToExtract[i]
			//fmt.Printf("Task with ID:%v and name:%v will have id:%v\n", task.Id, task.Name, i)
			// Construct the AdjacencyMatrix line by line
			for col := 0; col < size; col++ {
				task2 := tasksToExtract[col]
				//fmt.Printf("Setting %v,%v with value from %v,%v\n", i, col, task.Id, task2.Id)
				subTaskStructure.AdjacencyMatrix.Set(i, col, this.AdjacencyMatrix.At(task.Id, task2.Id))
			}
			subTaskStructure.DegreeMatrix.Set(i, i, this.DegreeMatrix.At(task.Id, task.Id))
			subTaskStructure.Tasks[i] = NewTask()
			subTaskStructure.Tasks[i].Name = task.Name
			subTaskStructure.Tasks[i].Module = task.Module
			subTaskStructure.Tasks[i].Args = task.Args
			subTaskStructure.Tasks[i].Origin = task.Origin
			subTaskStructure.Tasks[i].Id = i
		}
		//subTaskStructure.PrintAdjacencyMatrix()
		return subTaskStructure
	} else {
		return nil
	}
}
<|endoftext|>


<commit_before>package encoding

import (
	"bytes"
	"encoding/base32"
	"encoding/base64"
	"encoding/gob"
	"encoding/json"
	"encoding/xml"
	"runtime"
	fuzz "github.com/AdaLogics/go-fuzz-headers"
)

func FuzzEncoding(data []byte) int {
	f := fuzz.NewConsumer(data)
	decType, err := f.GetInt()
	if err != nil {
		return 0
	}
	b1, err := f.GetBytes()
	if err != nil {
		return 0
	}
	b2, err := f.GetBytes()
	if err != nil {
		return 0
	}
	defer func() {
		if r := recover(); r != nil {
		}
		runtime.GC()
	}()
	switch decType%5 {
	case 0:
		e, err := f.GetString()
		if err != nil || len(e) != 32 {
			return 0
		}
		enc := base32.NewEncoding(e)
		d := base32.NewDecoder(enc, bytes.NewReader(b1))
		_, _ = d.Read(b2)
		return 1
	case 1:
		e, err := f.GetString()
		if err != nil || len(e) != 64 {
			return 0
		}
		for i := 0; i < len(e); i++ {
			if e[i] == '\n' || e[i] == '\r' {
				return 0
			}
		}
		enc := base64.NewEncoding(e)
		d := base64.NewDecoder(enc, bytes.NewReader(b1))
		_, _ = d.Read(b2)
		return 1
	case 2:
		d := gob.NewDecoder(bytes.NewReader(b1))
		_ = d.Decode(b2)
		return 1
	case 3:
		d := json.NewDecoder(bytes.NewReader(b1))
		_ = d.Decode(b2)
		return 1
	case 4:
		d := xml.NewDecoder(bytes.NewReader(b1))
		_, _ = d.Token()
		return 1
	}
	return 1
}
<commit_msg>golang: refactor encoding fuzzer (#8841)<commit_after>package encoding

import (
	"bytes"
	"encoding/base32"
	"encoding/base64"
	"encoding/gob"
	"encoding/json"
	"encoding/xml"
	fuzz "github.com/AdaLogics/go-fuzz-headers"
)

func FuzzEncoding(data []byte) int {
	f := fuzz.NewConsumer(data)
	decType, err := f.GetInt()
	if err != nil {
		return 0
	}
	b1, err := f.GetBytes()
	if err != nil {
		return 0
	}
	switch decType % 5 {
	case 0:
		e, err := f.GetString()
		if err != nil || len(e) != 32 {
			return 0
		}
		enc := base32.NewEncoding(e)
		d := base32.NewDecoder(enc, bytes.NewReader(b1))
		dbuf := make([]byte, enc.DecodedLen(len(e)))
		_, _ = d.Read(dbuf)
	case 1:
		e, err := f.GetString()
		if err != nil || len(e) != 64 {
			return 0
		}
		for i := 0; i < len(e); i++ {
			if e[i] == '\n' || e[i] == '\r' {
				return 0
			}
		}
		enc := base64.NewEncoding(e)
		d := base64.NewDecoder(enc, bytes.NewReader(b1))
		dbuf := make([]byte, enc.DecodedLen(len(e)))
		_, _ = d.Read(dbuf)
	case 2:
		b2, err := f.GetBytes()
		if err != nil {
			return 0
		}
		d := gob.NewDecoder(bytes.NewReader(b1))
		_ = d.Decode(b2)
	case 3:
		b2, err := f.GetBytes()
		if err != nil {
			return 0
		}
		d := json.NewDecoder(bytes.NewReader(b1))
		_ = d.Decode(b2)
	case 4:
		d := xml.NewDecoder(bytes.NewReader(b1))
		_, _ = d.Token()
	}
	return 1
}
<|endoftext|>


<commit_before>package tests

import (
	"fmt"
	"github.com/bozaro/tech-db-forum/generated/client"
	"github.com/bozaro/tech-db-forum/generated/client/operations"
	"github.com/bozaro/tech-db-forum/generated/models"
	"math/rand"
	"net/url"
	"sync"
	"sync/atomic"
	"time"
)

type PerfConfig struct {
	UserCount   int
	ForumCount  int
	ThreadCount int
	PostCount   int
	PostBatch   int
	VoteCount   int

	Validate float32
}

func NewPerfConfig() *PerfConfig {
	return &PerfConfig{
		UserCount:   1000,
		ForumCount:  20,
		ThreadCount: 1000,
		PostCount:   1000000,
		PostBatch:   100,
		VoteCount:   25000,
		Validate:    1.0,
	}
}

func FillUsers(perf *Perf, parallel int, timeout time.Time, count int) {
	var need int32 = int32(count)
	c := perf.c
	data := perf.data

	// spawn four worker goroutines
	var wg sync.WaitGroup
	for i := 0; i < parallel; i++ {
		wg.Add(1)
		go func() {
			f := NewFactory()
			for atomic.AddInt32(&need, -1) >= 0 {
				user := f.CreateUser(c, nil)
				data.AddUser(&PUser{
					AboutHash:    Hash(user.About),
					Email:        user.Email,
					FullnameHash: Hash(user.Fullname),
					Nickname:     user.Nickname,
				})
			}
			wg.Done()
		}()
	}

	// wait for the workers to finish
	waitWaitGroup(&wg, timeout)
}

func FillThreads(perf *Perf, parallel int, timeout time.Time, count int) {
	var need int32 = int32(count)
	c := perf.c
	data := perf.data

	// spawn four worker goroutines
	var wg sync.WaitGroup
	for i := 0; i < parallel; i++ {
		wg.Add(1)
		go func() {
			f := NewFactory()
			for atomic.AddInt32(&need, -1) >= 0 {
				author := data.GetUser(-1)
				forum := data.GetForum(-1)
				thread := f.RandomThread()
				if rand.Intn(100) >= 25 {
					thread.Slug = ""
				}
				thread.Author = author.Nickname
				thread.Forum = forum.Slug
				thread = f.CreateThread(c, thread, nil, nil)
				data.AddThread(&PThread{
					ID:          thread.ID,
					Slug:        thread.Slug,
					Author:      author,
					Forum:       forum,
					Voices:      map[*PUser]int32{},
					MessageHash: Hash(thread.Message),
					TitleHash:   Hash(thread.Title),
					Created:     *thread.Created,
				})
			}
			wg.Done()
		}()
	}

	// wait for the workers to finish
	waitWaitGroup(&wg, timeout)
}

func FillPosts(perf *Perf, parallel int, timeout time.Time, count int, batchSize int) {
	var need int32 = int32(count)
	c := perf.c
	data := perf.data

	// spawn four worker goroutines
	var wg sync.WaitGroup
	for i := 0; i < parallel; i++ {
		wg.Add(1)
		go func() {
			f := NewFactory()
			for atomic.AddInt32(&need, -int32(batchSize)) >= 0 {

				batch := make([]*models.Post, 0, batchSize)
				thread := data.GetThread(-1)
				thread.mutex.Lock() // todo: Потом исправить

				parents := data.GetThreadPostsFlat(thread)

				for j := 0; j < batchSize; j++ {
					var parent *PPost
					if (len(parents) > 0) && (rand.Intn(4) == 0) {
						parent = parents[rand.Intn(len(parents))]
					}
					post := f.RandomPost()
					post.Author = data.GetUser(-1).Nickname
					post.Thread = thread.ID
					if parent != nil {
						post.Parent = parent.ID
					}
					batch = append(batch, post)
				}
				for _, post := range f.CreatePosts(c, batch, nil) {
					data.AddPost(&PPost{
						ID:          post.ID,
						Author:      data.GetUserByNickname(post.Author),
						Thread:      thread,
						Parent:      data.GetPostById(post.Parent),
						Created:     *post.Created,
						IsEdited:    false,
						MessageHash: Hash(post.Message),
					})
				}
				thread.mutex.Unlock()
			}
			wg.Done()
		}()
	}

	// wait for the workers to finish
	waitWaitGroup(&wg, timeout)
}

func VoteThreads(perf *Perf, parallel int, timeout time.Time, count int) {
	var need int32 = int32(count)
	c := perf.c
	data := perf.data

	// spawn four worker goroutines
	var wg sync.WaitGroup
	for i := 0; i < parallel; i++ {
		wg.Add(1)
		go func() {
			for atomic.AddInt32(&need, -1) >= 0 {
				user := data.GetUser(-1)

				thread := data.GetThread(-1)
				thread.mutex.Lock() // todo: Потом исправить

				old_voice := thread.Voices[user]
				var new_voice int32
				if old_voice != 0 {
					new_voice = -old_voice
				} else if rand.Intn(8) < 5 {
					new_voice = 1
				} else {
					new_voice = -1
				}
				thread.Voices[user] = new_voice
				thread.Votes += new_voice - old_voice

				result, err := c.Operations.ThreadVote(operations.NewThreadVoteParams().
					WithSlugOrID(fmt.Sprintf("%d", thread.ID)).
					WithVote(&models.Vote{
						Nickname: user.Nickname,
						Voice:    new_voice,
					}).
					WithContext(Expected(200, nil, nil)))
				CheckNil(err)
				if result.Payload.Votes != thread.Votes {
					panic(fmt.Sprintf("Unexpected votes count: %d != %d", result.Payload.Votes, thread.Votes))
				}
				thread.mutex.Unlock()
			}
			wg.Done()
		}()
	}

	// wait for the workers to finish
	waitWaitGroup(&wg, timeout)
}

func waitWaitGroup(wg *sync.WaitGroup, timeout time.Time) bool {
	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()

	delta := timeout.Sub(time.Now())
	if delta <= time.Second*1 {
		delta = time.Second * 1
	}
	select {
	case <-done:
		return true
	case <-time.After(delta):
		log.Panic("Timeout")
		return false
	}
}

func NewPerf(url *url.URL, config *PerfConfig) *Perf {
	transport := CreateTransport(url)
	report := Report{
		OnlyError: true,
		Result:    Success,
	}
	c := client.New(&CheckerTransport{transport, &report}, nil)

	data := NewPerfData(config)
	return &Perf{c: c,
		data:     data,
		validate: config.Validate,
	}
}

func (self *Perf) Fill(threads int, timeout_sec int, config *PerfConfig) {
	f := NewFactory()

	timeout := time.Now().Add(time.Second * time.Duration(timeout_sec))

	log.Infof("Clear data")
	_, err := self.c.Operations.Clear(nil)
	CheckNil(err)

	log.Infof("Creating users (%d threads)", threads)
	FillUsers(self, threads, timeout, config.UserCount)

	log.Info("Creating forums")
	for i := 0; i < config.ForumCount; i++ {
		user := self.data.GetUser(-1)
		forum := f.RandomForum()
		forum.User = user.Nickname
		forum = f.CreateForum(self.c, forum, nil)
		self.data.AddForum(&PForum{
			Slug:      forum.Slug,
			TitleHash: Hash(forum.Title),
			User:      user,
		})
	}

	log.Infof("Creating threads (%d threads)", threads)
	FillThreads(self, threads, timeout, config.ThreadCount)

	log.Infof("Vote threads (%d threads)", threads)
	VoteThreads(self, threads, timeout, config.VoteCount)

	log.Infof("Creating posts (%d threads)", threads)
	FillPosts(self, threads, timeout, config.PostCount, config.PostBatch)

	log.Info("Done")
}
<commit_msg>Добавил вывод прогресса заполнения базы<commit_after>package tests

import (
	"fmt"
	"github.com/bozaro/tech-db-forum/generated/client"
	"github.com/bozaro/tech-db-forum/generated/client/operations"
	"github.com/bozaro/tech-db-forum/generated/models"
	"math/rand"
	"net/url"
	"sync"
	"sync/atomic"
	"time"
)

type PerfConfig struct {
	UserCount   int
	ForumCount  int
	ThreadCount int
	PostCount   int
	PostBatch   int
	VoteCount   int

	Validate float32
}

func NewPerfConfig() *PerfConfig {
	return &PerfConfig{
		UserCount:   1000,
		ForumCount:  20,
		ThreadCount: 1000,
		PostCount:   1000000,
		PostBatch:   100,
		VoteCount:   25000,
		Validate:    1.0,
	}
}

func FillUsers(perf *Perf, parallel int, timeout time.Time, count int) {
	var need int32 = int32(count)
	c := perf.c
	data := perf.data

	// spawn four worker goroutines
	var wg sync.WaitGroup
	for i := 0; i < parallel; i++ {
		wg.Add(1)
		go func() {
			f := NewFactory()
			for atomic.AddInt32(&need, -1) >= 0 {
				user := f.CreateUser(c, nil)
				data.AddUser(&PUser{
					AboutHash:    Hash(user.About),
					Email:        user.Email,
					FullnameHash: Hash(user.Fullname),
					Nickname:     user.Nickname,
				})
			}
			wg.Done()
		}()
	}

	// wait for the workers to finish
	waitWaitGroup(&wg, timeout, count, &need)
}

func FillThreads(perf *Perf, parallel int, timeout time.Time, count int) {
	var need int32 = int32(count)
	c := perf.c
	data := perf.data

	// spawn four worker goroutines
	var wg sync.WaitGroup
	for i := 0; i < parallel; i++ {
		wg.Add(1)
		go func() {
			f := NewFactory()
			for atomic.AddInt32(&need, -1) >= 0 {
				author := data.GetUser(-1)
				forum := data.GetForum(-1)
				thread := f.RandomThread()
				if rand.Intn(100) >= 25 {
					thread.Slug = ""
				}
				thread.Author = author.Nickname
				thread.Forum = forum.Slug
				thread = f.CreateThread(c, thread, nil, nil)
				data.AddThread(&PThread{
					ID:          thread.ID,
					Slug:        thread.Slug,
					Author:      author,
					Forum:       forum,
					Voices:      map[*PUser]int32{},
					MessageHash: Hash(thread.Message),
					TitleHash:   Hash(thread.Title),
					Created:     *thread.Created,
				})
			}
			wg.Done()
		}()
	}

	// wait for the workers to finish
	waitWaitGroup(&wg, timeout, count, &need)
}

func FillPosts(perf *Perf, parallel int, timeout time.Time, count int, batchSize int) {
	var need int32 = int32(count)
	c := perf.c
	data := perf.data

	// spawn four worker goroutines
	var wg sync.WaitGroup
	for i := 0; i < parallel; i++ {
		wg.Add(1)
		go func() {
			f := NewFactory()
			for atomic.AddInt32(&need, -int32(batchSize)) >= 0 {

				batch := make([]*models.Post, 0, batchSize)
				thread := data.GetThread(-1)
				thread.mutex.Lock() // todo: Потом исправить

				parents := data.GetThreadPostsFlat(thread)

				for j := 0; j < batchSize; j++ {
					var parent *PPost
					if (len(parents) > 0) && (rand.Intn(4) == 0) {
						parent = parents[rand.Intn(len(parents))]
					}
					post := f.RandomPost()
					post.Author = data.GetUser(-1).Nickname
					post.Thread = thread.ID
					if parent != nil {
						post.Parent = parent.ID
					}
					batch = append(batch, post)
				}
				for _, post := range f.CreatePosts(c, batch, nil) {
					data.AddPost(&PPost{
						ID:          post.ID,
						Author:      data.GetUserByNickname(post.Author),
						Thread:      thread,
						Parent:      data.GetPostById(post.Parent),
						Created:     *post.Created,
						IsEdited:    false,
						MessageHash: Hash(post.Message),
					})
				}
				thread.mutex.Unlock()
			}
			wg.Done()
		}()
	}

	// wait for the workers to finish
	waitWaitGroup(&wg, timeout, count, &need)
}

func VoteThreads(perf *Perf, parallel int, timeout time.Time, count int) {
	var need int32 = int32(count)
	c := perf.c
	data := perf.data

	// spawn four worker goroutines
	var wg sync.WaitGroup
	for i := 0; i < parallel; i++ {
		wg.Add(1)
		go func() {
			for atomic.AddInt32(&need, -1) >= 0 {
				user := data.GetUser(-1)

				thread := data.GetThread(-1)
				thread.mutex.Lock() // todo: Потом исправить

				old_voice := thread.Voices[user]
				var new_voice int32
				if old_voice != 0 {
					new_voice = -old_voice
				} else if rand.Intn(8) < 5 {
					new_voice = 1
				} else {
					new_voice = -1
				}
				thread.Voices[user] = new_voice
				thread.Votes += new_voice - old_voice

				result, err := c.Operations.ThreadVote(operations.NewThreadVoteParams().
					WithSlugOrID(fmt.Sprintf("%d", thread.ID)).
					WithVote(&models.Vote{
						Nickname: user.Nickname,
						Voice:    new_voice,
					}).
					WithContext(Expected(200, nil, nil)))
				CheckNil(err)
				if result.Payload.Votes != thread.Votes {
					panic(fmt.Sprintf("Unexpected votes count: %d != %d", result.Payload.Votes, thread.Votes))
				}
				thread.mutex.Unlock()
			}
			wg.Done()
		}()
	}

	// wait for the workers to finish
	waitWaitGroup(&wg, timeout, count, &need)
}

func waitWaitGroup(wg *sync.WaitGroup, timeout time.Time, total int, need *int32) bool {
	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()

	logProgress := func() {
		completed := total - int(atomic.LoadInt32(need))
		if completed > total {
			completed = total
		}
		log.Infof("  %d of %d (%.2f%%)", completed, total, 100*float32(completed)/float32(total))
	}

	for {
		select {
		case <-done:
			logProgress()
			return true
		case <-time.After(time.Second * 10):
			logProgress()
		}
		if time.Now().After(timeout) {
			log.Panic("Timeout")
			return false
		}
	}
}

func NewPerf(url *url.URL, config *PerfConfig) *Perf {
	transport := CreateTransport(url)
	report := Report{
		OnlyError: true,
		Result:    Success,
	}
	c := client.New(&CheckerTransport{transport, &report}, nil)

	data := NewPerfData(config)
	return &Perf{c: c,
		data:     data,
		validate: config.Validate,
	}
}

func (self *Perf) Fill(threads int, timeout_sec int, config *PerfConfig) {
	f := NewFactory()

	timeout := time.Now().Add(time.Second * time.Duration(timeout_sec))

	log.Infof("Clear data")
	_, err := self.c.Operations.Clear(nil)
	CheckNil(err)

	log.Infof("Creating users (%d threads)", threads)
	FillUsers(self, threads, timeout, config.UserCount)

	log.Info("Creating forums")
	for i := 0; i < config.ForumCount; i++ {
		user := self.data.GetUser(-1)
		forum := f.RandomForum()
		forum.User = user.Nickname
		forum = f.CreateForum(self.c, forum, nil)
		self.data.AddForum(&PForum{
			Slug:      forum.Slug,
			TitleHash: Hash(forum.Title),
			User:      user,
		})
	}

	log.Infof("Creating threads (%d threads)", threads)
	FillThreads(self, threads, timeout, config.ThreadCount)

	log.Infof("Vote threads (%d threads)", threads)
	VoteThreads(self, threads, timeout, config.VoteCount)

	log.Infof("Creating posts (%d threads)", threads)
	FillPosts(self, threads, timeout, config.PostCount, config.PostBatch)

	log.Info("Done")
}
<|endoftext|>


<commit_before>package atlas

import (
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/hashicorp/atlas-go/archive"
	"github.com/hashicorp/atlas-go/v1"
	"github.com/mitchellh/mapstructure"
	"github.com/mitchellh/packer/common"
	"github.com/mitchellh/packer/packer"
)

const BuildEnvKey = "ATLAS_BUILD_ID"

// Artifacts can return a string for this state key and the post-processor
// will use automatically use this as the type. The user's value overrides
// this if `artifact_type_override` is set to true.
const ArtifactStateType = "atlas.artifact.type"

// Artifacts can return a map[string]string for this state key and this
// post-processor will automatically merge it into the metadata for any
// uploaded artifact versions.
const ArtifactStateMetadata = "atlas.artifact.metadata"

type Config struct {
	common.PackerConfig `mapstructure:",squash"`

	Artifact     string
	Type         string `mapstructure:"artifact_type"`
	TypeOverride bool   `mapstructure:"artifact_type_override"`
	Metadata     map[string]string

	ServerAddr string `mapstructure:"server_address"`
	Token      string

	// This shouldn't ever be set outside of unit tests.
	Test bool `mapstructure:"test"`

	tpl        *packer.ConfigTemplate
	user, name string
	buildId    int
}

type PostProcessor struct {
	config Config
	client *atlas.Client
}

func (p *PostProcessor) Configure(raws ...interface{}) error {
	_, err := common.DecodeConfig(&p.config, raws...)
	if err != nil {
		return err
	}

	p.config.tpl, err = packer.NewConfigTemplate()
	if err != nil {
		return err
	}
	p.config.tpl.UserVars = p.config.PackerUserVars

	templates := map[string]*string{
		"artifact":       &p.config.Artifact,
		"type":           &p.config.Type,
		"server_address": &p.config.ServerAddr,
		"token":          &p.config.Token,
	}

	errs := new(packer.MultiError)
	for key, ptr := range templates {
		*ptr, err = p.config.tpl.Process(*ptr, nil)
		if err != nil {
			errs = packer.MultiErrorAppend(
				errs, fmt.Errorf("Error processing %s: %s", key, err))
		}
	}

	required := map[string]*string{
		"artifact":      &p.config.Artifact,
		"artifact_type": &p.config.Type,
	}

	for key, ptr := range required {
		if *ptr == "" {
			errs = packer.MultiErrorAppend(
				errs, fmt.Errorf("%s must be set", key))
		}
	}

	if len(errs.Errors) > 0 {
		return errs
	}

	p.config.user, p.config.name, err = atlas.ParseSlug(p.config.Artifact)
	if err != nil {
		return err
	}

	// If we have a build ID, save it
	if v := os.Getenv(BuildEnvKey); v != "" {
		raw, err := strconv.ParseInt(v, 0, 0)
		if err != nil {
			return fmt.Errorf(
				"Error parsing build ID: %s", err)
		}

		p.config.buildId = int(raw)
	}

	// Build the client
	p.client = atlas.DefaultClient()
	if p.config.ServerAddr != "" {
		p.client, err = atlas.NewClient(p.config.ServerAddr)
		if err != nil {
			errs = packer.MultiErrorAppend(
				errs, fmt.Errorf("Error initializing client: %s", err))
			return errs
		}
	}
	if p.config.Token != "" {
		p.client.Token = p.config.Token
	}

	if !p.config.Test {
		// Verify the client
		if err := p.client.Verify(); err != nil {
			errs = packer.MultiErrorAppend(
				errs, fmt.Errorf("Error initializing client: %s", err))
			return errs
		}
	}

	return nil
}

func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) {
	if _, err := p.client.Artifact(p.config.user, p.config.name); err != nil {
		if err != atlas.ErrNotFound {
			return nil, false, fmt.Errorf(
				"Error finding artifact: %s", err)
		}

		// Artifact doesn't exist, create it
		ui.Message(fmt.Sprintf("Creating artifact: %s", p.config.Artifact))
		_, err = p.client.CreateArtifact(p.config.user, p.config.name)
		if err != nil {
			return nil, false, fmt.Errorf(
				"Error creating artifact: %s", err)
		}
	}

	opts := &atlas.UploadArtifactOpts{
		User:     p.config.user,
		Name:     p.config.name,
		Type:     p.config.Type,
		ID:       artifact.Id(),
		Metadata: p.metadata(artifact),
		BuildId:  p.config.buildId,
	}

	if fs := artifact.Files(); len(fs) > 0 {
		var archiveOpts archive.ArchiveOpts

		// We have files. We want to compress/upload them. If we have just
		// one file, then we use it as-is. Otherwise, we compress all of
		// them into a single file.
		var path string
		if len(fs) == 1 {
			path = fs[0]
		} else {
			path = longestCommonPrefix(fs)
			if path == "" {
				return nil, false, fmt.Errorf(
					"No common prefix for achiving files: %v", fs)
			}

			// Modify the archive options to only include the files
			// that are in our file list.
			include := make([]string, 0, len(fs))
			for i, f := range fs {
				include[i] = strings.Replace(f, path, "", 1)
			}
			archiveOpts.Include = include
		}

		r, err := archive.CreateArchive(path, &archiveOpts)
		if err != nil {
			return nil, false, fmt.Errorf(
				"Error archiving artifact: %s", err)
		}
		defer r.Close()

		opts.File = r
		opts.FileSize = r.Size
	}

	ui.Message("Uploading artifact version...")
	var av *atlas.ArtifactVersion
	doneCh := make(chan struct{})
	errCh := make(chan error, 1)
	go func() {
		var err error
		av, err = p.client.UploadArtifact(opts)
		if err != nil {
			errCh <- err
			return
		}
		close(doneCh)
	}()

	select {
	case err := <-errCh:
		return nil, false, fmt.Errorf("Error uploading: %s", err)
	case <-doneCh:
	}

	return &Artifact{
		Name:    p.config.Artifact,
		Type:    p.config.Type,
		Version: av.Version,
	}, true, nil
}

func (p *PostProcessor) metadata(artifact packer.Artifact) map[string]string {
	var metadata map[string]string
	metadataRaw := artifact.State(ArtifactStateMetadata)
	if metadataRaw != nil {
		if err := mapstructure.Decode(metadataRaw, &metadata); err != nil {
			panic(err)
		}
	}

	if p.config.Metadata != nil {
		// If we have no extra metadata, just return as-is
		if metadata == nil {
			return p.config.Metadata
		}

		// Merge the metadata
		for k, v := range p.config.Metadata {
			metadata[k] = v
		}
	}

	return metadata
}

func (p *PostProcessor) artifactType(artifact packer.Artifact) string {
	if !p.config.TypeOverride {
		if v := artifact.State(ArtifactStateType); v != nil {
			return v.(string)
		}
	}

	return p.config.Type
}
<commit_msg>atlas post-processor unknown atlas.UploadartifactOpts field BuildId<commit_after>package atlas

import (
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/hashicorp/atlas-go/archive"
	"github.com/hashicorp/atlas-go/v1"
	"github.com/mitchellh/mapstructure"
	"github.com/mitchellh/packer/common"
	"github.com/mitchellh/packer/packer"
)

const BuildEnvKey = "ATLAS_BUILD_ID"

// Artifacts can return a string for this state key and the post-processor
// will use automatically use this as the type. The user's value overrides
// this if `artifact_type_override` is set to true.
const ArtifactStateType = "atlas.artifact.type"

// Artifacts can return a map[string]string for this state key and this
// post-processor will automatically merge it into the metadata for any
// uploaded artifact versions.
const ArtifactStateMetadata = "atlas.artifact.metadata"

type Config struct {
	common.PackerConfig `mapstructure:",squash"`

	Artifact     string
	Type         string `mapstructure:"artifact_type"`
	TypeOverride bool   `mapstructure:"artifact_type_override"`
	Metadata     map[string]string

	ServerAddr string `mapstructure:"server_address"`
	Token      string

	// This shouldn't ever be set outside of unit tests.
	Test bool `mapstructure:"test"`

	tpl        *packer.ConfigTemplate
	user, name string
	buildId    int
}

type PostProcessor struct {
	config Config
	client *atlas.Client
}

func (p *PostProcessor) Configure(raws ...interface{}) error {
	_, err := common.DecodeConfig(&p.config, raws...)
	if err != nil {
		return err
	}

	p.config.tpl, err = packer.NewConfigTemplate()
	if err != nil {
		return err
	}
	p.config.tpl.UserVars = p.config.PackerUserVars

	templates := map[string]*string{
		"artifact":       &p.config.Artifact,
		"type":           &p.config.Type,
		"server_address": &p.config.ServerAddr,
		"token":          &p.config.Token,
	}

	errs := new(packer.MultiError)
	for key, ptr := range templates {
		*ptr, err = p.config.tpl.Process(*ptr, nil)
		if err != nil {
			errs = packer.MultiErrorAppend(
				errs, fmt.Errorf("Error processing %s: %s", key, err))
		}
	}

	required := map[string]*string{
		"artifact":      &p.config.Artifact,
		"artifact_type": &p.config.Type,
	}

	for key, ptr := range required {
		if *ptr == "" {
			errs = packer.MultiErrorAppend(
				errs, fmt.Errorf("%s must be set", key))
		}
	}

	if len(errs.Errors) > 0 {
		return errs
	}

	p.config.user, p.config.name, err = atlas.ParseSlug(p.config.Artifact)
	if err != nil {
		return err
	}

	// If we have a build ID, save it
	if v := os.Getenv(BuildEnvKey); v != "" {
		raw, err := strconv.ParseInt(v, 0, 0)
		if err != nil {
			return fmt.Errorf(
				"Error parsing build ID: %s", err)
		}

		p.config.buildId = int(raw)
	}

	// Build the client
	p.client = atlas.DefaultClient()
	if p.config.ServerAddr != "" {
		p.client, err = atlas.NewClient(p.config.ServerAddr)
		if err != nil {
			errs = packer.MultiErrorAppend(
				errs, fmt.Errorf("Error initializing client: %s", err))
			return errs
		}
	}
	if p.config.Token != "" {
		p.client.Token = p.config.Token
	}

	if !p.config.Test {
		// Verify the client
		if err := p.client.Verify(); err != nil {
			errs = packer.MultiErrorAppend(
				errs, fmt.Errorf("Error initializing client: %s", err))
			return errs
		}
	}

	return nil
}

func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) {
	if _, err := p.client.Artifact(p.config.user, p.config.name); err != nil {
		if err != atlas.ErrNotFound {
			return nil, false, fmt.Errorf(
				"Error finding artifact: %s", err)
		}

		// Artifact doesn't exist, create it
		ui.Message(fmt.Sprintf("Creating artifact: %s", p.config.Artifact))
		_, err = p.client.CreateArtifact(p.config.user, p.config.name)
		if err != nil {
			return nil, false, fmt.Errorf(
				"Error creating artifact: %s", err)
		}
	}

	opts := &atlas.UploadArtifactOpts{
		User:     p.config.user,
		Name:     p.config.name,
		Type:     p.config.Type,
		ID:       artifact.Id(),
		Metadata: p.metadata(artifact),
		BuildID:  p.config.buildId,
	}

	if fs := artifact.Files(); len(fs) > 0 {
		var archiveOpts archive.ArchiveOpts

		// We have files. We want to compress/upload them. If we have just
		// one file, then we use it as-is. Otherwise, we compress all of
		// them into a single file.
		var path string
		if len(fs) == 1 {
			path = fs[0]
		} else {
			path = longestCommonPrefix(fs)
			if path == "" {
				return nil, false, fmt.Errorf(
					"No common prefix for achiving files: %v", fs)
			}

			// Modify the archive options to only include the files
			// that are in our file list.
			include := make([]string, 0, len(fs))
			for i, f := range fs {
				include[i] = strings.Replace(f, path, "", 1)
			}
			archiveOpts.Include = include
		}

		r, err := archive.CreateArchive(path, &archiveOpts)
		if err != nil {
			return nil, false, fmt.Errorf(
				"Error archiving artifact: %s", err)
		}
		defer r.Close()

		opts.File = r
		opts.FileSize = r.Size
	}

	ui.Message("Uploading artifact version...")
	var av *atlas.ArtifactVersion
	doneCh := make(chan struct{})
	errCh := make(chan error, 1)
	go func() {
		var err error
		av, err = p.client.UploadArtifact(opts)
		if err != nil {
			errCh <- err
			return
		}
		close(doneCh)
	}()

	select {
	case err := <-errCh:
		return nil, false, fmt.Errorf("Error uploading: %s", err)
	case <-doneCh:
	}

	return &Artifact{
		Name:    p.config.Artifact,
		Type:    p.config.Type,
		Version: av.Version,
	}, true, nil
}

func (p *PostProcessor) metadata(artifact packer.Artifact) map[string]string {
	var metadata map[string]string
	metadataRaw := artifact.State(ArtifactStateMetadata)
	if metadataRaw != nil {
		if err := mapstructure.Decode(metadataRaw, &metadata); err != nil {
			panic(err)
		}
	}

	if p.config.Metadata != nil {
		// If we have no extra metadata, just return as-is
		if metadata == nil {
			return p.config.Metadata
		}

		// Merge the metadata
		for k, v := range p.config.Metadata {
			metadata[k] = v
		}
	}

	return metadata
}

func (p *PostProcessor) artifactType(artifact packer.Artifact) string {
	if !p.config.TypeOverride {
		if v := artifact.State(ArtifactStateType); v != nil {
			return v.(string)
		}
	}

	return p.config.Type
}
<|endoftext|>