<commit_before>package custom

import (
	"bytes"
	"encoding/hex"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/UnnoTed/fileb0x/compression"
	"github.com/UnnoTed/fileb0x/dir"
	"github.com/UnnoTed/fileb0x/file"
	"github.com/UnnoTed/fileb0x/utils"
	"github.com/bmatcuk/doublestar"
)

// SharedConfig holds needed data from config package
// without causing import cycle
type SharedConfig struct {
	Output      string
	Compression *compression.Gzip
}

// Custom is a set of files with dedicaTed customization
type Custom struct {
	Files  []string
	Base   string
	Prefix string

	Exclude []string
	Replace []Replacer
}

// Parse the files transforming them into a byte string and inserting the file
// into a map of files
func (c *Custom) Parse(files *map[string]*file.File, dirs **dir.Dir, config *SharedConfig) error {
	to := *files
	dirList := *dirs

	var newList []string
	for _, customFile := range c.Files {
		// get files from glob
		list, err := doublestar.Glob(customFile)
		if err != nil {
			return err
		}

		// insert files from glob into the new list
		newList = append(newList, list...)
	}

	// copy new list
	c.Files = newList

	// 0 files in the list
	if len(c.Files) == 0 {
		return errors.New("No files found")
	}

	// loop through files from glob
	for _, customFile := range c.Files {
		// gives error when file doesn't exist
		if !utils.Exists(customFile) {
			return fmt.Errorf("File [%s] doesn't exist", customFile)
		}

		customFile = utils.FixPath(customFile)
		walkErr := filepath.Walk(customFile, func(fpath string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}

			// only files will be processed
			if info.IsDir() {
				return nil
			}

			var fixedPath string
			if c.Prefix != "" || c.Base != "" {
				if strings.HasPrefix(c.Base, "./") {
					c.Base = c.Base[2:]
				}

				if strings.HasPrefix(fpath, c.Base) {
					fixedPath = c.Prefix + fpath[len(c.Base):]
				} else {
					if c.Base != "" {
						fixedPath = c.Prefix + fpath
					}
				}

				fixedPath = utils.FixPath(fixedPath)
			} else {
				fixedPath = utils.FixPath(fpath)
			}

			// check for excluded files
			for _, excludedFile := range c.Exclude {
				m, err := doublestar.Match(c.Prefix+excludedFile, fixedPath)
				if err != nil {
					return err
				}

				if m {
					return nil
				}
			}

			// FIXME
			// prevent including itself (destination file or dir)
			if info.Name() == config.Output {
				return nil
			}
			/*if info.Name() == cfg.Output { ||
			  info.Name() == path.Base(path.Dir(jsonFile)) {
			  return nil
			}*/

			// get file's content
			content, err := ioutil.ReadFile(fpath)
			if err != nil {
				return err
			}

			// loop through replace list
			for _, r := range c.Replace {
				// check if path matches the pattern from property: file
				matched, err := doublestar.Match(c.Prefix+r.File, fixedPath)
				if err != nil {
					return err
				}

				if matched {
					for pattern, word := range r.Replace {
						content = []byte(strings.Replace(string(content), pattern, word, -1))
					}
				}
			}

			// it's way faster to use a buffer as string than use string
			var buf bytes.Buffer
			buf.WriteString(`[]byte("`)
			f := file.NewFile()

			// compress the content
			if config.Compression.Options != nil {
				content, err = config.Compression.Compress(content)
				if err != nil {
					return err
				}
			}

			// it's way faster to loop and slice a string than a byte array
			h := hex.EncodeToString(content)

			// loop through hex string, at each 2 chars
			// it's added into a byte array -> []byte{0x61 ,...}
			for i := 0; i < len(h); i += 2 {
				buf.WriteString(`\x` + h[i:i+2])
			}

			f.Data = buf.String() + `")`
			f.Name = info.Name()
			f.Path = fixedPath

			// insert dir to dirlist so it can be created on b0x's init()
			dirList.Insert(path.Dir(fixedPath))

			// insert file into file list
			to[fixedPath] = f
			return nil
		})

		if walkErr != nil {
			return walkErr
		}
	}

	return nil
}
<commit_msg>Fix "base" option on Windows<commit_after>package custom

import (
	"bytes"
	"encoding/hex"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/UnnoTed/fileb0x/compression"
	"github.com/UnnoTed/fileb0x/dir"
	"github.com/UnnoTed/fileb0x/file"
	"github.com/UnnoTed/fileb0x/utils"
	"github.com/bmatcuk/doublestar"
)

// SharedConfig holds needed data from config package
// without causing import cycle
type SharedConfig struct {
	Output      string
	Compression *compression.Gzip
}

// Custom is a set of files with dedicaTed customization
type Custom struct {
	Files  []string
	Base   string
	Prefix string

	Exclude []string
	Replace []Replacer
}

// Parse the files transforming them into a byte string and inserting the file
// into a map of files
func (c *Custom) Parse(files *map[string]*file.File, dirs **dir.Dir, config *SharedConfig) error {
	to := *files
	dirList := *dirs

	var newList []string
	for _, customFile := range c.Files {
		// get files from glob
		list, err := doublestar.Glob(customFile)
		if err != nil {
			return err
		}

		// insert files from glob into the new list
		newList = append(newList, list...)
	}

	// copy new list
	c.Files = newList

	// 0 files in the list
	if len(c.Files) == 0 {
		return errors.New("No files found")
	}

	// loop through files from glob
	for _, customFile := range c.Files {
		// gives error when file doesn't exist
		if !utils.Exists(customFile) {
			return fmt.Errorf("File [%s] doesn't exist", customFile)
		}

		customFile = utils.FixPath(customFile)
		walkErr := filepath.Walk(customFile, func(fpath string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}

			// only files will be processed
			if info.IsDir() {
				return nil
			}

			fpath = utils.FixPath(fpath)

			var fixedPath string
			if c.Prefix != "" || c.Base != "" {
				c.Base = strings.TrimPrefix(c.Base, "./")

				if strings.HasPrefix(fpath, c.Base) {
					fixedPath = c.Prefix + fpath[len(c.Base):]
				} else {
					if c.Base != "" {
						fixedPath = c.Prefix + fpath
					}
				}

				fixedPath = utils.FixPath(fixedPath)
			} else {
				fixedPath = utils.FixPath(fpath)
			}

			// check for excluded files
			for _, excludedFile := range c.Exclude {
				m, err := doublestar.Match(c.Prefix+excludedFile, fixedPath)
				if err != nil {
					return err
				}

				if m {
					return nil
				}
			}

			// FIXME
			// prevent including itself (destination file or dir)
			if info.Name() == config.Output {
				return nil
			}
			/*if info.Name() == cfg.Output { ||
			  info.Name() == path.Base(path.Dir(jsonFile)) {
			  return nil
			}*/

			// get file's content
			content, err := ioutil.ReadFile(fpath)
			if err != nil {
				return err
			}

			// loop through replace list
			for _, r := range c.Replace {
				// check if path matches the pattern from property: file
				matched, err := doublestar.Match(c.Prefix+r.File, fixedPath)
				if err != nil {
					return err
				}

				if matched {
					for pattern, word := range r.Replace {
						content = []byte(strings.Replace(string(content), pattern, word, -1))
					}
				}
			}

			// it's way faster to use a buffer as string than use string
			var buf bytes.Buffer
			buf.WriteString(`[]byte("`)
			f := file.NewFile()

			// compress the content
			if config.Compression.Options != nil {
				content, err = config.Compression.Compress(content)
				if err != nil {
					return err
				}
			}

			// it's way faster to loop and slice a string than a byte array
			h := hex.EncodeToString(content)

			// loop through hex string, at each 2 chars
			// it's added into a byte array -> []byte{0x61 ,...}
			for i := 0; i < len(h); i += 2 {
				buf.WriteString(`\x` + h[i:i+2])
			}

			f.Data = buf.String() + `")`
			f.Name = info.Name()
			f.Path = fixedPath

			// insert dir to dirlist so it can be created on b0x's init()
			dirList.Insert(path.Dir(fixedPath))

			// insert file into file list
			to[fixedPath] = f
			return nil
		})

		if walkErr != nil {
			return walkErr
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package daemon

import (
	"bytes"
	"fmt"
	"sort"
	gosync "sync"
	"time"

	"github.com/go-kit/kit/log"
	"github.com/pkg/errors"

	"github.com/weaveworks/flux"
	"github.com/weaveworks/flux/cluster"
	"github.com/weaveworks/flux/git"
	"github.com/weaveworks/flux/guid"
	"github.com/weaveworks/flux/history"
	"github.com/weaveworks/flux/job"
	"github.com/weaveworks/flux/policy"
	"github.com/weaveworks/flux/registry"
	"github.com/weaveworks/flux/release"
	"github.com/weaveworks/flux/remote"
	"github.com/weaveworks/flux/update"
)

var ErrUnknownJob = fmt.Errorf("unkown job")

// Combine these things to form Devasta^Wan implementation of
// Platform.
type Daemon struct {
	V              string
	Cluster        cluster.Cluster
	Manifests      cluster.Manifests
	Registry       registry.Registry
	Repo           git.Repo
	Checkout       *git.Checkout
	Jobs           *job.Queue
	JobStatusCache *job.StatusCache
	EventWriter    history.EventWriter
	// bookkeeping
	syncSoon     chan struct{}
	initSyncSoon gosync.Once
}

// Invariant.
var _ remote.Platform = &Daemon{}

func (d *Daemon) Version() (string, error) {
	return d.V, nil
}

func (d *Daemon) Ping() error {
	return d.Cluster.Ping()
}

func (d *Daemon) Export() ([]byte, error) {
	return d.Cluster.Export()
}

func (d *Daemon) ListServices(namespace string) ([]flux.ServiceStatus, error) {
	var res []flux.ServiceStatus
	services, err := d.Cluster.AllServices(namespace)
	if err != nil {
		return nil, errors.Wrap(err, "getting services from cluster")
	}

	d.Checkout.RLock()
	defer d.Checkout.RUnlock()
	automatedServices, err := d.Manifests.ServicesWithPolicy(d.Checkout.ManifestDir(), policy.Automated)
	if err != nil {
		return nil, errors.Wrap(err, "checking service policies")
	}
	lockedServices, err := d.Manifests.ServicesWithPolicy(d.Checkout.ManifestDir(), policy.Locked)
	if err != nil {
		return nil, errors.Wrap(err, "checking service policies")
	}

	for _, service := range services {
		res = append(res, flux.ServiceStatus{
			ID:         service.ID,
			Containers: containers2containers(service.ContainersOrNil()),
			Status:     service.Status,
			Automated:  automatedServices.Contains(service.ID),
			Locked:     lockedServices.Contains(service.ID),
		})
	}

	return res, nil
}

// List the images available for set of services
func (d *Daemon) ListImages(spec update.ServiceSpec) ([]flux.ImageStatus, error) {
	var services []cluster.Service
	var err error
	if spec == update.ServiceSpecAll {
		services, err = d.Cluster.AllServices("")
	} else {
		id, err := spec.AsID()
		if err != nil {
			return nil, errors.Wrap(err, "treating service spec as ID")
		}
		services, err = d.Cluster.SomeServices([]flux.ServiceID{id})
	}

	images, err := release.CollectAvailableImages(d.Registry, services)
	if err != nil {
		return nil, errors.Wrap(err, "getting images for services")
	}

	var res []flux.ImageStatus
	for _, service := range services {
		containers := containersWithAvailable(service, images)
		res = append(res, flux.ImageStatus{
			ID:         service.ID,
			Containers: containers,
		})
	}

	return res, nil
}

type DaemonJobFunc func(jobID job.ID, working *git.Checkout, logger log.Logger) (*history.CommitEventMetadata, error)

func (d *Daemon) queueJob(do DaemonJobFunc) job.ID {
	id := job.ID(guid.New())
	d.Jobs.Enqueue(&job.Job{
		ID: id,
		Do: func(logger log.Logger) error {
			started := time.Now().UTC()
			d.JobStatusCache.SetStatus(id, job.Status{StatusString: job.StatusRunning})
			// make a working clone so we don't mess with files we
			// will be reading from elsewhere
			working, err := d.Checkout.WorkingClone()
			if err != nil {
				d.JobStatusCache.SetStatus(id, job.Status{StatusString: job.StatusFailed, Err: err.Error()})
				return err
			}
			defer working.Clean()
			metadata, err := do(id, working, logger)
			if err != nil {
				d.JobStatusCache.SetStatus(id, job.Status{StatusString: job.StatusFailed, Err: err.Error()})
				return err
			}
			d.JobStatusCache.SetStatus(id, job.Status{StatusString: job.StatusSucceeded, Result: *metadata})
			var serviceIDs []flux.ServiceID
			for id, result := range metadata.Result {
				if result.Status == update.ReleaseStatusSuccess {
					serviceIDs = append(serviceIDs, id)
				}
			}
			logger.Log("revision", metadata.Revision)
			return d.LogEvent(history.Event{
				ServiceIDs: serviceIDs,
				Type:       history.EventCommit,
				StartedAt:  started,
				EndedAt:    started,
				LogLevel:   history.LogLevelInfo,
				Metadata:   metadata,
			})
		},
	})
	d.JobStatusCache.SetStatus(id, job.Status{StatusString: job.StatusQueued})
	return id
}

// Apply the desired changes to the config files
func (d *Daemon) UpdateManifests(spec update.Spec) (job.ID, error) {
	switch s := spec.Spec.(type) {
	case update.ReleaseSpec:
		return d.queueJob(func(jobID job.ID, working *git.Checkout, logger log.Logger) (*history.CommitEventMetadata, error) {
			rc := release.NewReleaseContext(d.Cluster, d.Manifests, d.Registry, working)
			revision, result, err := release.Release(rc, s, logger)
			if err != nil {
				return nil, err
			}
			d.askForSync()
			return &history.CommitEventMetadata{
				Revision: revision,
				Spec:     &spec,
				Result:   result,
			}, nil
		}), nil
	case policy.Updates:
		return d.queueJob(func(jobID job.ID, working *git.Checkout, logger log.Logger) (*history.CommitEventMetadata, error) {
			// For each update
			var serviceIDs []flux.ServiceID
			metadata := &history.CommitEventMetadata{
				Spec:   &spec,
				Result: update.Result{},
			}
			for serviceID, u := range s {
				// find the service manifest
				err := cluster.UpdateManifest(d.Manifests, working.ManifestDir(), string(serviceID), func(def []byte) ([]byte, error) {
					newDef, err := d.Manifests.UpdatePolicies(def, u)
					if err != nil {
						metadata.Result[serviceID] = update.ServiceResult{
							Status: update.ReleaseStatusFailed,
							Error:  err.Error(),
						}
						return nil, err
					}
					if string(newDef) == string(def) {
						metadata.Result[serviceID] = update.ServiceResult{
							Status: update.ReleaseStatusSkipped,
						}
					} else {
						serviceIDs = append(serviceIDs, serviceID)
						metadata.Result[serviceID] = update.ServiceResult{
							Status: update.ReleaseStatusSuccess,
						}
					}
					return newDef, nil
				})
				switch err {
				case cluster.ErrNoResourceFilesFoundForService, cluster.ErrMultipleResourceFilesFoundForService:
					metadata.Result[serviceID] = update.ServiceResult{
						Status: update.ReleaseStatusFailed,
						Error:  err.Error(),
					}
				case nil:
					// continue
				default:
					return nil, err
				}
			}
			if len(serviceIDs) == 0 {
				return metadata, nil
			}

			if err := working.CommitAndPush(policyCommitMessage(s, time.Now().UTC()), &git.Note{JobID: jobID, Spec: spec}); err != nil {
				return nil, err
			}

			var err error
			metadata.Revision, err = working.HeadRevision()
			if err != nil {
				return nil, err
			}
			return metadata, nil
		}), nil
	default:
		var id job.ID
		return id, fmt.Errorf(`unknown update type "%s"`, spec.Type)
	}
}

// Tell the daemon to synchronise the cluster with the manifests in
// the git repo. This has an error return value because upstream there
// may be comms difficulties or other sources of problems; here, we
// always succeed because it's just bookkeeping.
func (d *Daemon) SyncNotify() error {
	d.askForSync()
	return nil
}

// Ask the daemon how far it's got committing things; in particular, is the job
// queued? running? committed? If it is done, the commit ref is returned.
func (d *Daemon) JobStatus(jobID job.ID) (job.Status, error) {
	// Is the job queued, running, or recently finished?
	status, ok := d.JobStatusCache.Status(jobID)
	if ok {
		return status, nil
	}

	// is there a commit for this job?
	// Look through the commits for a note referencing this job. What a hack.
	// But, it means that even if fluxd restarts, we will remember jobs which
	// have pushed a commit.
	if err := d.Checkout.Pull(); err != nil {
		return job.Status{}, errors.Wrap(err, "updating repo for status")
	}
	refs, err := d.Checkout.RevisionsBefore("HEAD")
	if err != nil {
		return job.Status{}, errors.Wrap(err, "checking revisions for status")
	}
	for _, ref := range refs {
		note, _ := d.Checkout.GetNote(ref)
		if note != nil && note.JobID == jobID {
			return job.Status{
				StatusString: job.StatusSucceeded,
				Result: history.CommitEventMetadata{
					Revision: ref,
					Spec:     &note.Spec,
					Result:   note.Result,
				},
			}, nil
		}
	}

	return job.Status{}, ErrUnknownJob
}

// Ask the daemon how far it's got applying things; in particular, is it
// past the supplied release? Return the list of commits between where
// we have applied and the ref given, inclusive. E.g., if you send HEAD,
// you'll get all the commits yet to be applied. If you send a hash
// and it's applied _past_ it, you'll get an empty list.
func (d *Daemon) SyncStatus(commitRef string) ([]string, error) {
	return d.Checkout.RevisionsBetween(d.Checkout.SyncTag, commitRef)
}

// Non-remote.Platform methods

func (d *Daemon) LogEvent(ev history.Event) error {
	if d.EventWriter == nil {
		return nil
	}
	return d.EventWriter.LogEvent(ev)
}

// vvv helpers vvv

func containers2containers(cs []cluster.Container) []flux.Container {
	res := make([]flux.Container, len(cs))
	for i, c := range cs {
		id, _ := flux.ParseImageID(c.Image)
		res[i] = flux.Container{
			Name: c.Name,
			Current: flux.Image{
				ID: id,
			},
		}
	}
	return res
}

func containersWithAvailable(service cluster.Service, images release.ImageMap) (res []flux.Container) {
	for _, c := range service.ContainersOrNil() {
		id, _ := flux.ParseImageID(c.Image)
		repo := id.Repository()
		available := images[repo]
		res = append(res, flux.Container{
			Name: c.Name,
			Current: flux.Image{
				ID: id,
			},
			Available: available,
		})
	}
	return res
}

// policyEvents builds a map of events (by type), for all the events in this set of
// updates. There will be one event per type, containing all service ids
// affected by that event. e.g. all automated services will share an event.
func policyEvents(us policy.Updates, now time.Time) map[string]history.Event {
	eventsByType := map[string]history.Event{}
	for serviceID, update := range us {
		for _, eventType := range policyEventTypes(update) {
			e, ok := eventsByType[eventType]
			if !ok {
				e = history.Event{
					ServiceIDs: []flux.ServiceID{},
					Type:       eventType,
					StartedAt:  now,
					EndedAt:    now,
					LogLevel:   history.LogLevelInfo,
				}
			}
			e.ServiceIDs = append(e.ServiceIDs, serviceID)
			eventsByType[eventType] = e
		}
	}
	return eventsByType
}

func policyCommitMessage(us policy.Updates, now time.Time) string {
	events := policyEvents(us, now)
	commitMsg := &bytes.Buffer{}
	prefix := ""
	if len(events) > 1 {
		fmt.Fprintf(commitMsg, "Updated service policies:\n\n")
		prefix = "- "
	}
	for _, event := range events {
		fmt.Fprintf(commitMsg, "%s%v\n", prefix, event)
	}
	return commitMsg.String()
}

// policyEventTypes is a deduped list of all event types this update contains
func policyEventTypes(u policy.Update) []string {
	types := map[string]struct{}{}
	for _, p := range u.Add {
		switch p {
		case policy.Automated:
			types[history.EventAutomate] = struct{}{}
		case policy.Locked:
			types[history.EventLock] = struct{}{}
		}
	}

	for _, p := range u.Remove {
		switch p {
		case policy.Automated:
			types[history.EventDeautomate] = struct{}{}
		case policy.Locked:
			types[history.EventUnlock] = struct{}{}
		}
	}
	var result []string
	for t := range types {
		result = append(result, t)
	}
	sort.Strings(result)
	return result
}
<commit_msg>Sanity check: `update.Spec`s should always have a `Type`<commit_after>package daemon

import (
	"bytes"
	"fmt"
	"sort"
	gosync "sync"
	"time"

	"github.com/go-kit/kit/log"
	"github.com/pkg/errors"

	"github.com/weaveworks/flux"
	"github.com/weaveworks/flux/cluster"
	"github.com/weaveworks/flux/git"
	"github.com/weaveworks/flux/guid"
	"github.com/weaveworks/flux/history"
	"github.com/weaveworks/flux/job"
	"github.com/weaveworks/flux/policy"
	"github.com/weaveworks/flux/registry"
	"github.com/weaveworks/flux/release"
	"github.com/weaveworks/flux/remote"
	"github.com/weaveworks/flux/update"
)

var ErrUnknownJob = fmt.Errorf("unkown job")

// Combine these things to form Devasta^Wan implementation of
// Platform.
type Daemon struct {
	V              string
	Cluster        cluster.Cluster
	Manifests      cluster.Manifests
	Registry       registry.Registry
	Repo           git.Repo
	Checkout       *git.Checkout
	Jobs           *job.Queue
	JobStatusCache *job.StatusCache
	EventWriter    history.EventWriter
	// bookkeeping
	syncSoon     chan struct{}
	initSyncSoon gosync.Once
}

// Invariant.
var _ remote.Platform = &Daemon{}

func (d *Daemon) Version() (string, error) {
	return d.V, nil
}

func (d *Daemon) Ping() error {
	return d.Cluster.Ping()
}

func (d *Daemon) Export() ([]byte, error) {
	return d.Cluster.Export()
}

func (d *Daemon) ListServices(namespace string) ([]flux.ServiceStatus, error) {
	var res []flux.ServiceStatus
	services, err := d.Cluster.AllServices(namespace)
	if err != nil {
		return nil, errors.Wrap(err, "getting services from cluster")
	}

	d.Checkout.RLock()
	defer d.Checkout.RUnlock()
	automatedServices, err := d.Manifests.ServicesWithPolicy(d.Checkout.ManifestDir(), policy.Automated)
	if err != nil {
		return nil, errors.Wrap(err, "checking service policies")
	}
	lockedServices, err := d.Manifests.ServicesWithPolicy(d.Checkout.ManifestDir(), policy.Locked)
	if err != nil {
		return nil, errors.Wrap(err, "checking service policies")
	}

	for _, service := range services {
		res = append(res, flux.ServiceStatus{
			ID:         service.ID,
			Containers: containers2containers(service.ContainersOrNil()),
			Status:     service.Status,
			Automated:  automatedServices.Contains(service.ID),
			Locked:     lockedServices.Contains(service.ID),
		})
	}

	return res, nil
}

// List the images available for set of services
func (d *Daemon) ListImages(spec update.ServiceSpec) ([]flux.ImageStatus, error) {
	var services []cluster.Service
	var err error
	if spec == update.ServiceSpecAll {
		services, err = d.Cluster.AllServices("")
	} else {
		id, err := spec.AsID()
		if err != nil {
			return nil, errors.Wrap(err, "treating service spec as ID")
		}
		services, err = d.Cluster.SomeServices([]flux.ServiceID{id})
	}

	images, err := release.CollectAvailableImages(d.Registry, services)
	if err != nil {
		return nil, errors.Wrap(err, "getting images for services")
	}

	var res []flux.ImageStatus
	for _, service := range services {
		containers := containersWithAvailable(service, images)
		res = append(res, flux.ImageStatus{
			ID:         service.ID,
			Containers: containers,
		})
	}

	return res, nil
}

type DaemonJobFunc func(jobID job.ID, working *git.Checkout, logger log.Logger) (*history.CommitEventMetadata, error)

func (d *Daemon) queueJob(do DaemonJobFunc) job.ID {
	id := job.ID(guid.New())
	d.Jobs.Enqueue(&job.Job{
		ID: id,
		Do: func(logger log.Logger) error {
			started := time.Now().UTC()
			d.JobStatusCache.SetStatus(id, job.Status{StatusString: job.StatusRunning})
			// make a working clone so we don't mess with files we
			// will be reading from elsewhere
			working, err := d.Checkout.WorkingClone()
			if err != nil {
				d.JobStatusCache.SetStatus(id, job.Status{StatusString: job.StatusFailed, Err: err.Error()})
				return err
			}
			defer working.Clean()
			metadata, err := do(id, working, logger)
			if err != nil {
				d.JobStatusCache.SetStatus(id, job.Status{StatusString: job.StatusFailed, Err: err.Error()})
				return err
			}
			d.JobStatusCache.SetStatus(id, job.Status{StatusString: job.StatusSucceeded, Result: *metadata})
			var serviceIDs []flux.ServiceID
			for id, result := range metadata.Result {
				if result.Status == update.ReleaseStatusSuccess {
					serviceIDs = append(serviceIDs, id)
				}
			}
			logger.Log("revision", metadata.Revision)
			return d.LogEvent(history.Event{
				ServiceIDs: serviceIDs,
				Type:       history.EventCommit,
				StartedAt:  started,
				EndedAt:    started,
				LogLevel:   history.LogLevelInfo,
				Metadata:   metadata,
			})
		},
	})
	d.JobStatusCache.SetStatus(id, job.Status{StatusString: job.StatusQueued})
	return id
}

// Apply the desired changes to the config files
func (d *Daemon) UpdateManifests(spec update.Spec) (job.ID, error) {
	var id job.ID
	if spec.Type == "" {
		return id, errors.New("no type in update spec")
	}
	switch s := spec.Spec.(type) {
	case update.ReleaseSpec:
		return d.queueJob(func(jobID job.ID, working *git.Checkout, logger log.Logger) (*history.CommitEventMetadata, error) {
			rc := release.NewReleaseContext(d.Cluster, d.Manifests, d.Registry, working)
			revision, result, err := release.Release(rc, s, logger)
			if err != nil {
				return nil, err
			}
			d.askForSync()
			return &history.CommitEventMetadata{
				Revision: revision,
				Spec:     &spec,
				Result:   result,
			}, nil
		}), nil
	case policy.Updates:
		return d.queueJob(func(jobID job.ID, working *git.Checkout, logger log.Logger) (*history.CommitEventMetadata, error) {
			// For each update
			var serviceIDs []flux.ServiceID
			metadata := &history.CommitEventMetadata{
				Spec:   &spec,
				Result: update.Result{},
			}
			for serviceID, u := range s {
				// find the service manifest
				err := cluster.UpdateManifest(d.Manifests, working.ManifestDir(), string(serviceID), func(def []byte) ([]byte, error) {
					newDef, err := d.Manifests.UpdatePolicies(def, u)
					if err != nil {
						metadata.Result[serviceID] = update.ServiceResult{
							Status: update.ReleaseStatusFailed,
							Error:  err.Error(),
						}
						return nil, err
					}
					if string(newDef) == string(def) {
						metadata.Result[serviceID] = update.ServiceResult{
							Status: update.ReleaseStatusSkipped,
						}
					} else {
						serviceIDs = append(serviceIDs, serviceID)
						metadata.Result[serviceID] = update.ServiceResult{
							Status: update.ReleaseStatusSuccess,
						}
					}
					return newDef, nil
				})
				switch err {
				case cluster.ErrNoResourceFilesFoundForService, cluster.ErrMultipleResourceFilesFoundForService:
					metadata.Result[serviceID] = update.ServiceResult{
						Status: update.ReleaseStatusFailed,
						Error:  err.Error(),
					}
				case nil:
					// continue
				default:
					return nil, err
				}
			}
			if len(serviceIDs) == 0 {
				return metadata, nil
			}

			if err := working.CommitAndPush(policyCommitMessage(s, time.Now().UTC()), &git.Note{JobID: jobID, Spec: spec}); err != nil {
				return nil, err
			}

			var err error
			metadata.Revision, err = working.HeadRevision()
			if err != nil {
				return nil, err
			}
			return metadata, nil
		}), nil
	default:
		return id, fmt.Errorf(`unknown update type "%s"`, spec.Type)
	}
}

// Tell the daemon to synchronise the cluster with the manifests in
// the git repo. This has an error return value because upstream there
// may be comms difficulties or other sources of problems; here, we
// always succeed because it's just bookkeeping.
func (d *Daemon) SyncNotify() error {
	d.askForSync()
	return nil
}

// Ask the daemon how far it's got committing things; in particular, is the job
// queued? running? committed? If it is done, the commit ref is returned.
func (d *Daemon) JobStatus(jobID job.ID) (job.Status, error) {
	// Is the job queued, running, or recently finished?
	status, ok := d.JobStatusCache.Status(jobID)
	if ok {
		return status, nil
	}

	// is there a commit for this job?
	// Look through the commits for a note referencing this job. What a hack.
	// But, it means that even if fluxd restarts, we will remember jobs which
	// have pushed a commit.
	if err := d.Checkout.Pull(); err != nil {
		return job.Status{}, errors.Wrap(err, "updating repo for status")
	}
	refs, err := d.Checkout.RevisionsBefore("HEAD")
	if err != nil {
		return job.Status{}, errors.Wrap(err, "checking revisions for status")
	}
	for _, ref := range refs {
		note, _ := d.Checkout.GetNote(ref)
		if note != nil && note.JobID == jobID {
			return job.Status{
				StatusString: job.StatusSucceeded,
				Result: history.CommitEventMetadata{
					Revision: ref,
					Spec:     &note.Spec,
					Result:   note.Result,
				},
			}, nil
		}
	}

	return job.Status{}, ErrUnknownJob
}

// Ask the daemon how far it's got applying things; in particular, is it
// past the supplied release? Return the list of commits between where
// we have applied and the ref given, inclusive. E.g., if you send HEAD,
// you'll get all the commits yet to be applied. If you send a hash
// and it's applied _past_ it, you'll get an empty list.
func (d *Daemon) SyncStatus(commitRef string) ([]string, error) {
	return d.Checkout.RevisionsBetween(d.Checkout.SyncTag, commitRef)
}

// Non-remote.Platform methods

func (d *Daemon) LogEvent(ev history.Event) error {
	if d.EventWriter == nil {
		return nil
	}
	return d.EventWriter.LogEvent(ev)
}

// vvv helpers vvv

func containers2containers(cs []cluster.Container) []flux.Container {
	res := make([]flux.Container, len(cs))
	for i, c := range cs {
		id, _ := flux.ParseImageID(c.Image)
		res[i] = flux.Container{
			Name: c.Name,
			Current: flux.Image{
				ID: id,
			},
		}
	}
	return res
}

func containersWithAvailable(service cluster.Service, images release.ImageMap) (res []flux.Container) {
	for _, c := range service.ContainersOrNil() {
		id, _ := flux.ParseImageID(c.Image)
		repo := id.Repository()
		available := images[repo]
		res = append(res, flux.Container{
			Name: c.Name,
			Current: flux.Image{
				ID: id,
			},
			Available: available,
		})
	}
	return res
}

// policyEvents builds a map of events (by type), for all the events in this set of
// updates. There will be one event per type, containing all service ids
// affected by that event. e.g. all automated services will share an event.
func policyEvents(us policy.Updates, now time.Time) map[string]history.Event {
	eventsByType := map[string]history.Event{}
	for serviceID, update := range us {
		for _, eventType := range policyEventTypes(update) {
			e, ok := eventsByType[eventType]
			if !ok {
				e = history.Event{
					ServiceIDs: []flux.ServiceID{},
					Type:       eventType,
					StartedAt:  now,
					EndedAt:    now,
					LogLevel:   history.LogLevelInfo,
				}
			}
			e.ServiceIDs = append(e.ServiceIDs, serviceID)
			eventsByType[eventType] = e
		}
	}
	return eventsByType
}

func policyCommitMessage(us policy.Updates, now time.Time) string {
	events := policyEvents(us, now)
	commitMsg := &bytes.Buffer{}
	prefix := ""
	if len(events) > 1 {
		fmt.Fprintf(commitMsg, "Updated service policies:\n\n")
		prefix = "- "
	}
	for _, event := range events {
		fmt.Fprintf(commitMsg, "%s%v\n", prefix, event)
	}
	return commitMsg.String()
}

// policyEventTypes is a deduped list of all event types this update contains
func policyEventTypes(u policy.Update) []string {
	types := map[string]struct{}{}
	for _, p := range u.Add {
		switch p {
		case policy.Automated:
			types[history.EventAutomate] = struct{}{}
		case policy.Locked:
			types[history.EventLock] = struct{}{}
		}
	}

	for _, p := range u.Remove {
		switch p {
		case policy.Automated:
			types[history.EventDeautomate] = struct{}{}
		case policy.Locked:
			types[history.EventUnlock] = struct{}{}
		}
	}
	var result []string
	for t := range types {
		result = append(result, t)
	}
	sort.Strings(result)
	return result
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"log/syslog"
	"strings"

	log "code.google.com/p/log4go"
)

type sysLogWriter chan *log.LogRecord

func (w sysLogWriter) LogWrite(rec *log.LogRecord) {
	w <- rec
}

func (w sysLogWriter) Close() {
	close(w)
}

func GetSysLogFacility(name string) (syslog.Priority, bool) {
	switch strings.ToLower(name) {
	case "syslog":
		return syslog.LOG_SYSLOG, true
	case "local0":
		return syslog.LOG_LOCAL0, true
	case "local1":
		return syslog.LOG_LOCAL1, true
	case "local2":
		return syslog.LOG_LOCAL2, true
	case "local3":
		return syslog.LOG_LOCAL3, true
	case "local4":
		return syslog.LOG_LOCAL4, true
	case "local5":
		return syslog.LOG_LOCAL5, true
	case "local6":
		return syslog.LOG_LOCAL6, true
	case "local7":
		return syslog.LOG_LOCAL7, true
	default:
		return syslog.LOG_SYSLOG, false
	}
}

func getWriter(writer *syslog.Writer, level string) func(string) error {
	switch level {
	case "DEBG", "TRAC", "FINE", "FNST":
		return writer.Debug
	case "INFO":
		return writer.Info
	case "WARN":
		return writer.Warning
	case "EROR":
		return writer.Err
	default:
		return writer.Crit
	}
}

func connectSyslogDaemon(priority syslog.Priority) (writer *syslog.Writer, err error) {
	logTypes := []string{"unixgram", "unix"}
	logPaths := []string{"/dev/log", "/var/run/syslog"}
	var raddr string
	for _, network := range logTypes {
		for _, path := range logPaths {
			raddr = path
			writer, err = syslog.Dial(network, raddr, priority, "influxdb")
			if err != nil {
				continue
			} else {
				return
			}
		}
	}
	if err != nil {
		err = errors.New("cannot connect to Syslog Daemon")
	}
	return
}

func NewSysLogWriter(priority syslog.Priority) (w sysLogWriter, err error) {
	writer, err := connectSyslogDaemon(priority)
	if err != nil {
		return
	}
	w = sysLogWriter(make(chan *log.LogRecord, log.LogBufferLength))
	go func() {
		defer func() {
			if w != nil {
				w.Close()
			}
		}()
		for rec := range w {
			m := log.FormatLogRecord("(%S) %M", rec)
			getWriter(writer, rec.Level.String())(m)
		}
	}()
	return
}
<commit_msg>add some docs<commit_after>package main

import (
	"errors"
	"log/syslog"
	"strings"

	log "code.google.com/p/log4go"
)

type sysLogWriter chan *log.LogRecord

func (w sysLogWriter) LogWrite(rec *log.LogRecord) {
	w <- rec
}

func (w sysLogWriter) Close() {
	close(w)
}

func GetSysLogFacility(name string) (syslog.Priority, bool) {
	switch strings.ToLower(name) {
	case "syslog":
		return syslog.LOG_SYSLOG, true
	case "local0":
		return syslog.LOG_LOCAL0, true
	case "local1":
		return syslog.LOG_LOCAL1, true
	case "local2":
		return syslog.LOG_LOCAL2, true
	case "local3":
		return syslog.LOG_LOCAL3, true
	case "local4":
		return syslog.LOG_LOCAL4, true
	case "local5":
		return syslog.LOG_LOCAL5, true
	case "local6":
		return syslog.LOG_LOCAL6, true
	case "local7":
		return syslog.LOG_LOCAL7, true
	default:
		return syslog.LOG_SYSLOG, false
	}
}

func getWriter(writer *syslog.Writer, level string) func(string) error {
	switch level {
	case "DEBG", "TRAC", "FINE", "FNST":
		return writer.Debug
	case "INFO":
		return writer.Info
	case "WARN":
		return writer.Warning
	case "EROR":
		return writer.Err
	default:
		return writer.Crit
	}
}

func connectSyslogDaemon(priority syslog.Priority) (writer *syslog.Writer, err error) {
	logTypes := []string{"unixgram", "unix"}
	logPaths := []string{
		"/dev/log",        // unix socket for syslog on linux
		"/var/run/syslog", // unix socket for syslog on osx
	}
	var raddr string
	for _, network := range logTypes {
		for _, path := range logPaths {
			raddr = path
			writer, err = syslog.Dial(network, raddr, priority, "influxdb")
			if err != nil {
				continue
			} else {
				return
			}
		}
	}
	if err != nil {
		err = errors.New("cannot connect to Syslog Daemon")
	}
	return
}

func NewSysLogWriter(priority syslog.Priority) (w sysLogWriter, err error) {
	writer, err := connectSyslogDaemon(priority)
	if err != nil {
		return
	}
	w = sysLogWriter(make(chan *log.LogRecord, log.LogBufferLength))
	go func() {
		defer func() {
			if w != nil {
				w.Close()
			}
		}()
		for rec := range w {
			m := log.FormatLogRecord("(%S) %M", rec)
			getWriter(writer, rec.Level.String())(m)
		}
	}()
	return
}
<|endoftext|>


<commit_before>package myaws

import (
	"fmt"
	"github.com/aws/aws-sdk-go/service/ec2"
	"strings"
)

type EC2RILsOptions struct {
	All    bool
	Quiet  bool
	Fields []string
}

func (client *Client) EC2RILs(options EC2RILsOptions) error {
	instances, err := client.FindEC2ReservedInstances(options.All)
	if err != nil {
		return err
	}

	for _, instance := range instances {
		fmt.Fprintln(client.stdout, formatEC2RIInstance(client, options, instance))
	}

	return nil
}

func formatEC2RIInstance(client *Client, options EC2RILsOptions, instance *ec2.ReservedInstances) string {
	formatFuncs := map[string]func(client *Client, options EC2RILsOptions, instance *ec2.ReservedInstances) string{
		"ReservedInstancesId": formatEC2ReservedInstanceID,
		"AvailabilityZone":    formatEC2RIAvailabilityZone,
		"InstanceType":        formatEC2RIInstanceType,
		"InstanceCount":       formatEC2RIInstanceCount,
		"State":               formatEC2RIState,
		"Scope":               formatEC2RIScope,
		"Start":               formatEC2RIStart,
		"End":                 formatEC2RIEnd,
		"Duration":            formatEC2RIDuration,
	}

	var outputFields []string
	if options.Quiet {
		outputFields = []string{"InstanceId"}
	} else {
		outputFields = options.Fields
	}

	output := []string{}

	for _, field := range outputFields {
		value := formatFuncs[field](client, options, instance)
		output = append(output, value)
	}

	return strings.Join(output[:], "\t")
}

func formatEC2ReservedInstanceID(client *Client, options EC2RILsOptions, instance *ec2.ReservedInstances) string {
	return *instance.ReservedInstancesId
}

func formatEC2RIAvailabilityZone(client *Client, options EC2RILsOptions, instance *ec2.ReservedInstances) string {
	if instance.AvailabilityZone != nil {
		return *instance.AvailabilityZone
	}
	return "N/A"
}

func formatEC2RIInstanceType(client *Client, options EC2RILsOptions, instance *ec2.ReservedInstances) string {
	return *instance.InstanceType
}

func formatEC2RIInstanceCount(client *Client, options EC2RILsOptions, instance *ec2.ReservedInstances) string {
	return fmt.Sprintf("%3d", *instance.InstanceCount)
}

func formatEC2RIState(client *Client, options EC2RILsOptions, instance *ec2.ReservedInstances) string {
	return *instance.State
}

func formatEC2RIScope(client *Client, options EC2RILsOptions, instance *ec2.ReservedInstances) string {
	return *instance.Scope
}

func formatEC2RIStart(client *Client, options EC2RILsOptions, instance *ec2.ReservedInstances) string {
	return instance.Start.Format("2006-01-02")
}

func formatEC2RIEnd(client *Client, options EC2RILsOptions, instance *ec2.ReservedInstances) string {
	return instance.End.Format("2006-01-02")
}

func formatEC2RIDuration(client *Client, options EC2RILsOptions, instance *ec2.ReservedInstances) string {
	return fmt.Sprintf("%2dyear", *instance.Duration/(3600*24*365))
}
<commit_msg>apply goimports<commit_after>package myaws

import (
	"fmt"
	"strings"

	"github.com/aws/aws-sdk-go/service/ec2"
)

type EC2RILsOptions struct {
	All    bool
	Quiet  bool
	Fields []string
}

func (client *Client) EC2RILs(options EC2RILsOptions) error {
	instances, err := client.FindEC2ReservedInstances(options.All)
	if err != nil {
		return err
	}

	for _, instance := range instances {
		fmt.Fprintln(client.stdout, formatEC2RIInstance(client, options, instance))
	}

	return nil
}

func formatEC2RIInstance(client *Client, options EC2RILsOptions, instance *ec2.ReservedInstances) string {
	formatFuncs := map[string]func(client *Client, options EC2RILsOptions, instance *ec2.ReservedInstances) string{
		"ReservedInstancesId": formatEC2ReservedInstanceID,
		"AvailabilityZone":    formatEC2RIAvailabilityZone,
		"InstanceType":        formatEC2RIInstanceType,
		"InstanceCount":       formatEC2RIInstanceCount,
		"State":               formatEC2RIState,
		"Scope":               formatEC2RIScope,
		"Start":               formatEC2RIStart,
		"End":                 formatEC2RIEnd,
		"Duration":            formatEC2RIDuration,
	}

	var outputFields []string
	if options.Quiet {
		outputFields = []string{"InstanceId"}
	} else {
		outputFields = options.Fields
	}

	output := []string{}

	for _, field := range outputFields {
		value := formatFuncs[field](client, options, instance)
		output = append(output, value)
	}

	return strings.Join(output[:], "\t")
}

func formatEC2ReservedInstanceID(client *Client, options EC2RILsOptions, instance *ec2.ReservedInstances) string {
	return *instance.ReservedInstancesId
}

func formatEC2RIAvailabilityZone(client *Client, options EC2RILsOptions, instance *ec2.ReservedInstances) string {
	if instance.AvailabilityZone != nil {
		return *instance.AvailabilityZone
	}
	return "N/A"
}

func formatEC2RIInstanceType(client *Client, options EC2RILsOptions, instance *ec2.ReservedInstances) string {
	return *instance.InstanceType
}

func formatEC2RIInstanceCount(client *Client, options EC2RILsOptions, instance *ec2.ReservedInstances) string {
	return fmt.Sprintf("%3d", *instance.InstanceCount)
}

func formatEC2RIState(client *Client, options EC2RILsOptions, instance *ec2.ReservedInstances) string {
	return *instance.State
}

func formatEC2RIScope(client *Client, options EC2RILsOptions, instance *ec2.ReservedInstances) string {
	return *instance.Scope
}

func formatEC2RIStart(client *Client, options EC2RILsOptions, instance *ec2.ReservedInstances) string {
	return instance.Start.Format("2006-01-02")
}

func formatEC2RIEnd(client *Client, options EC2RILsOptions, instance *ec2.ReservedInstances) string {
	return instance.End.Format("2006-01-02")
}

func formatEC2RIDuration(client *Client, options EC2RILsOptions, instance *ec2.ReservedInstances) string {
	return fmt.Sprintf("%2dyear", *instance.Duration/(3600*24*365))
}
<|endoftext|>


<commit_before>package daemon

import (
	"strings"
	"time"

	"github.com/go-kit/kit/log"
	"github.com/pkg/errors"

	"github.com/weaveworks/flux"
	"github.com/weaveworks/flux/history"
	"github.com/weaveworks/flux/resource"
	"github.com/weaveworks/flux/sync"
)

const (
	gitPollInterval    = 5 * time.Minute
	imagesPollInterval = 5 * time.Minute
)

// Loop for potentially long-running stuff. This includes running
// jobs, and looking for new commits.

func (d *Daemon) Loop(stop chan struct{}, logger log.Logger) {
	pollGit := time.NewTimer(gitPollInterval)
	resetGitPoll := func() {
		if pollGit != nil {
			pollGit.Stop()
			pollGit = time.NewTimer(gitPollInterval)
		}
	}

	pollImages := time.Tick(imagesPollInterval)
	// Ask for a sync straight away
	d.askForSync()
	for {
		select {
		case <-stop:
			logger.Log("stopping", "true")
			return
		case <-d.syncSoon:
			d.pullAndSync(logger)
			resetGitPoll()
		case <-pollGit.C:
			// Time to poll for new commits (unless we're already
			// about to do that)
			d.askForSync()
		case <-pollImages:
			// Time to poll for new images
			d.PollImages()
		case job := <-d.Jobs.Ready():
			logger.Log("job", job.ID)
			// It's assumed that (successful) jobs will push commits
			// to the upstream repo, and therefore we probably want to
			// pull from there and sync the cluster.
			if err := job.Do(); err != nil {
				logger.Log("job", job.ID, "err", err)
				continue
			}
			logger.Log("job", job.ID, "success", "true")
			d.askForSync()
		}
	}
}

// Ask for a sync, or if there's one waiting, let that happen.
func (d *Daemon) askForSync() {
	d.initSyncSoon.Do(func() {
		d.syncSoon = make(chan struct{}, 1)
	})
	select {
	case d.syncSoon <- struct{}{}:
	default:
	}
}

func (d *Daemon) pullAndSync(logger log.Logger) {
	started := time.Now().UTC()

	// Pull for new commits
	if err := d.Checkout.Pull(); err != nil {
		logger.Log("err", err)
		return
	}

	// checkout a working clone so we can mess around with tags later
	working, err := d.Checkout.WorkingClone()
	if err != nil {
		logger.Log("err", err)
		return
	}
	defer working.Clean()

	// update notes and emit events for applied commits
	revisions, err := working.RevisionsBetween(working.SyncTag+"~1", "HEAD")
	if isUnknownRevision(err) {
		// No sync tag, grab all revisions
		revisions, err = working.RevisionsBefore("HEAD")
	}
	if err != nil {
		logger.Log("err", err)
	}

	// TODO logging, metrics?
	// Get a map of all resources defined in the repo
	allResources, err := d.Manifests.LoadManifests(working.ManifestDir())
	if err != nil {
		logger.Log("err", errors.Wrap(err, "loading resources from repo"))
		return
	}

	// TODO supply deletes argument from somewhere (command-line?)
	if err := sync.Sync(d.Manifests, allResources, d.Cluster, false); err != nil {
		logger.Log("err", err)
	}

	// Figure out which service IDs changed in this release
	changedResources := map[string]resource.Resource{}
	changedFiles, err := working.ChangedFiles(working.SyncTag)
	switch {
	case err == nil:
		// We had some changed files, we're syncing a diff
		changedResources, err = d.Manifests.LoadManifests(changedFiles...)
		if err != nil {
			logger.Log("err", errors.Wrap(err, "loading resources from repo"))
			return
		}
	case isUnknownRevision(err):
		// no synctag, We are syncing everything from scratch
		changedResources = allResources
	default:
		logger.Log("err", err)
	}
	serviceIDs := flux.ServiceIDSet{}
	for _, r := range changedResources {
		serviceIDs.Add(r.ServiceIDs(allResources))
	}

	// Emit an event
	if len(revisions) > 0 {
		if err := d.LogEvent(history.Event{
			ServiceIDs: serviceIDs.ToSlice(),
			Type:       history.EventSync,
			StartedAt:  started,
			EndedAt:    started,
			LogLevel:   history.LogLevelInfo,
			Metadata:   &history.SyncEventMetadata{Revisions: revisions},
		}); err != nil {
			logger.Log("err", err)
		}
	}

	// Move the tag and push it so we know how far we've gotten.
	if err := working.MoveTagAndPush("HEAD", "Sync pointer"); err != nil {
		logger.Log("err", err)
	}
}

func isUnknownRevision(err error) bool {
	return err != nil &&
		(strings.Contains(err.Error(), "unknown revision or path not in the working tree.") ||
			strings.Contains(err.Error(), "bad revision"))
}
<commit_msg>Update sync tag in pullAndSync<commit_after>package daemon

import (
	"strings"
	"time"

	"github.com/go-kit/kit/log"
	"github.com/pkg/errors"

	"github.com/weaveworks/flux"
	"github.com/weaveworks/flux/git"
	"github.com/weaveworks/flux/history"
	"github.com/weaveworks/flux/resource"
	"github.com/weaveworks/flux/sync"
)

const (
	gitPollInterval    = 5 * time.Minute
	imagesPollInterval = 5 * time.Minute
)

// Loop for potentially long-running stuff. This includes running
// jobs, and looking for new commits.

func (d *Daemon) Loop(stop chan struct{}, logger log.Logger) {
	pollGit := time.NewTimer(gitPollInterval)
	resetGitPoll := func() {
		if pollGit != nil {
			pollGit.Stop()
			pollGit = time.NewTimer(gitPollInterval)
		}
	}

	pollImages := time.Tick(imagesPollInterval)
	// Ask for a sync straight away
	d.askForSync()
	for {
		select {
		case <-stop:
			logger.Log("stopping", "true")
			return
		case <-d.syncSoon:
			d.pullAndSync(logger)
			resetGitPoll()
		case <-pollGit.C:
			// Time to poll for new commits (unless we're already
			// about to do that)
			d.askForSync()
		case <-pollImages:
			// Time to poll for new images
			d.PollImages()
		case job := <-d.Jobs.Ready():
			logger.Log("job", job.ID)
			// It's assumed that (successful) jobs will push commits
			// to the upstream repo, and therefore we probably want to
			// pull from there and sync the cluster.
			if err := job.Do(); err != nil {
				logger.Log("job", job.ID, "err", err)
				continue
			}
			logger.Log("job", job.ID, "success", "true")
			d.askForSync()
		}
	}
}

// Ask for a sync, or if there's one waiting, let that happen.
func (d *Daemon) askForSync() {
	d.initSyncSoon.Do(func() {
		d.syncSoon = make(chan struct{}, 1)
	})
	select {
	case d.syncSoon <- struct{}{}:
	default:
	}
}

func (d *Daemon) pullAndSync(logger log.Logger) {
	started := time.Now().UTC()

	// Pull for new commits
	if err := d.Checkout.Pull(); err != nil {
		logger.Log("err", err)
		return
	}

	// checkout a working clone so we can mess around with tags later
	working, err := d.Checkout.WorkingClone()
	if err != nil {
		logger.Log("err", err)
		return
	}
	defer working.Clean()

	// update notes and emit events for applied commits
	revisions, err := working.RevisionsBetween(working.SyncTag+"~1", "HEAD")
	if isUnknownRevision(err) {
		// No sync tag, grab all revisions
		revisions, err = working.RevisionsBefore("HEAD")
	}
	if err != nil {
		logger.Log("err", err)
	}

	// TODO logging, metrics?
	// Get a map of all resources defined in the repo
	allResources, err := d.Manifests.LoadManifests(working.ManifestDir())
	if err != nil {
		logger.Log("err", errors.Wrap(err, "loading resources from repo"))
		return
	}

	// TODO supply deletes argument from somewhere (command-line?)
	if err := sync.Sync(d.Manifests, allResources, d.Cluster, false); err != nil {
		logger.Log("err", err)
	}

	// Figure out which service IDs changed in this release
	changedResources := map[string]resource.Resource{}
	changedFiles, err := working.ChangedFiles(working.SyncTag)
	switch {
	case err == nil:
		// We had some changed files, we're syncing a diff
		changedResources, err = d.Manifests.LoadManifests(changedFiles...)
		if err != nil {
			logger.Log("err", errors.Wrap(err, "loading resources from repo"))
			return
		}
	case isUnknownRevision(err):
		// no synctag, We are syncing everything from scratch
		changedResources = allResources
	default:
		logger.Log("err", err)
	}
	serviceIDs := flux.ServiceIDSet{}
	for _, r := range changedResources {
		serviceIDs.Add(r.ServiceIDs(allResources))
	}

	// Emit an event
	if len(revisions) > 0 {
		if err := d.LogEvent(history.Event{
			ServiceIDs: serviceIDs.ToSlice(),
			Type:       history.EventSync,
			StartedAt:  started,
			EndedAt:    started,
			LogLevel:   history.LogLevelInfo,
			Metadata:   &history.SyncEventMetadata{Revisions: revisions},
		}); err != nil {
			logger.Log("err", err)
		}
	}

	// Move the tag and push it so we know how far we've gotten.
	if err := working.MoveTagAndPush("HEAD", "Sync pointer"); err != nil {
		logger.Log("err", err)
	}

	// Pull the tag if it has changed
	if err := d.updateTagRev(working, logger); err != nil {
		logger.Log("err", errors.Wrap(err, "updating tag"))
	}
}

func (d *Daemon) updateTagRev(working *git.Checkout, logger log.Logger) error {
	oldTagRev, err := d.Checkout.TagRevision(d.Checkout.SyncTag)
	if err != nil {
		return err
	}
	newTagRev, err := working.TagRevision(working.SyncTag)
	if err != nil {
		return err
	}

	if oldTagRev != newTagRev {
		logger.Log("tag", d.Checkout.SyncTag, "old", oldTagRev, "new", newTagRev)

		if err := d.Checkout.Pull(); err != nil {
			return err
		}
	}

	return nil
}

func isUnknownRevision(err error) bool {
	return err != nil &&
		(strings.Contains(err.Error(), "unknown revision or path not in the working tree.") ||
			strings.Contains(err.Error(), "bad revision"))
}
<|endoftext|>


<commit_before>// Copyright 2013 gandalf authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package api

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/globocom/gandalf/repository"
	"github.com/globocom/gandalf/user"
	"io"
	"io/ioutil"
	"net/http"
	"reflect"
)

func accessParameters(body io.ReadCloser) (repositories, users []string, err error) {
	var params map[string][]string
	if err := parseBody(body, &params); err != nil {
		return []string{}, []string{}, err
	}
	users, ok := params["users"]
	if !ok {
		return []string{}, []string{}, errors.New("It is need a user list")
	}
	repositories, ok = params["repositories"]
	if !ok {
		return []string{}, []string{}, errors.New("It is need a repository list")
	}
	return repositories, users, nil
}

func GrantAccess(w http.ResponseWriter, r *http.Request) {
	// TODO: update README
	repositories, users, err := accessParameters(r.Body)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	if err := repository.GrantAccess(repositories, users); err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}
	fmt.Fprintf(w, "Successfully granted access to users \"%s\" into repository \"%s\"", users, repositories)
}

func RevokeAccess(w http.ResponseWriter, r *http.Request) {
	repositories, users, err := accessParameters(r.Body)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	if err := repository.RevokeAccess(repositories, users); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	fmt.Fprintf(w, "Successfully revoked access to users \"%s\" into repositories \"%s\"", users, repositories)
}

func AddKey(w http.ResponseWriter, r *http.Request) {
	keys := map[string]string{}
	if err := parseBody(r.Body, &keys); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	if len(keys) == 0 {
		http.Error(w, "A key is needed", http.StatusBadRequest)
		return
	}
	uName := r.URL.Query().Get(":name")
	if err := user.AddKey(uName, keys); err != nil {
		switch err {
		case user.ErrInvalidKey:
			http.Error(w, err.Error(), http.StatusBadRequest)
		case user.ErrDuplicateKey:
			http.Error(w, "Key already exists.", http.StatusConflict)
		case user.ErrUserNotFound:
			http.Error(w, err.Error(), http.StatusNotFound)
		default:
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
		return
	}
	fmt.Fprint(w, "Key(s) successfully created")
}

func RemoveKey(w http.ResponseWriter, r *http.Request) {
	uName := r.URL.Query().Get(":name")
	kName := r.URL.Query().Get(":keyname")
	if err := user.RemoveKey(uName, kName); err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}
	fmt.Fprintf(w, "Key \"%s\" successfully removed", kName)
}

func ListKeys(w http.ResponseWriter, r *http.Request) {
	uName := r.URL.Query().Get(":name")
	keys, err := user.ListKeys(uName)
	if err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}
	out, err := json.Marshal(&keys)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Write(out)
}

type jsonUser struct {
	Name string
	Keys map[string]string
}

func NewUser(w http.ResponseWriter, r *http.Request) {
	var usr jsonUser
	if err := parseBody(r.Body, &usr); err != nil {
		http.Error(w, "Got error while parsing body: "+err.Error(), http.StatusBadRequest)
		return
	}
	u, err := user.New(usr.Name, usr.Keys)
	if err != nil {
		http.Error(w, "Got error while creating user: "+err.Error(), http.StatusBadRequest)
		return
	}
	fmt.Fprintf(w, "User \"%s\" successfully created\n", u.Name)
}

func RemoveUser(w http.ResponseWriter, r *http.Request) {
	name := r.URL.Query().Get(":name")
	if err := user.Remove(name); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	fmt.Fprintf(w, "User \"%s\" successfully removed\n", name)
}

func NewRepository(w http.ResponseWriter, r *http.Request) {
	var repo repository.Repository
	if err := parseBody(r.Body, &repo); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	rep, err := repository.New(repo.Name, repo.Users, repo.IsPublic)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	fmt.Fprintf(w, "Repository \"%s\" successfully created\n", rep.Name)
}

func GetRepository(w http.ResponseWriter, r *http.Request) {
	repo, err := repository.Get(r.URL.Query().Get(":name"))
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	out, err := json.Marshal(&repo)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Write(out)
}

func RemoveRepository(w http.ResponseWriter, r *http.Request) {
	name := r.URL.Query().Get(":name")
	if err := repository.Remove(name); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	fmt.Fprintf(w, "Repository \"%s\" successfully removed\n", name)
}

func RenameRepository(w http.ResponseWriter, r *http.Request) {
	var p struct{ Name string }
	defer r.Body.Close()
	err := parseBody(r.Body, &p)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
	}
	name := r.URL.Query().Get(":name")
	err = repository.Rename(name, p.Name)
	if err != nil && err.Error() == "not found" {
		http.Error(w, err.Error(), http.StatusNotFound)
	} else if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func parseBody(body io.ReadCloser, result interface{}) error {
	if reflect.ValueOf(result).Kind() == reflect.Struct {
		return errors.New("parseBody function cannot deal with struct. Use pointer")
	}
	b, err := ioutil.ReadAll(body)
	if err != nil {
		return err
	}
	if err := json.Unmarshal(b, &result); err != nil {
		return errors.New(fmt.Sprintf("Could not parse json: %s", err.Error()))
	}
	return nil
}

func HealthCheck(w http.ResponseWriter, r *http.Request) {
	w.Write([]byte("WORKING"))
}
<commit_msg>api: ping the database before assuming that everything is ok<commit_after>// Copyright 2014 gandalf authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package api

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/globocom/gandalf/db"
	"github.com/globocom/gandalf/repository"
	"github.com/globocom/gandalf/user"
	"io"
	"io/ioutil"
	"net/http"
	"reflect"
)

func accessParameters(body io.ReadCloser) (repositories, users []string, err error) {
	var params map[string][]string
	if err := parseBody(body, &params); err != nil {
		return []string{}, []string{}, err
	}
	users, ok := params["users"]
	if !ok {
		return []string{}, []string{}, errors.New("It is need a user list")
	}
	repositories, ok = params["repositories"]
	if !ok {
		return []string{}, []string{}, errors.New("It is need a repository list")
	}
	return repositories, users, nil
}

func GrantAccess(w http.ResponseWriter, r *http.Request) {
	// TODO: update README
	repositories, users, err := accessParameters(r.Body)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	if err := repository.GrantAccess(repositories, users); err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}
	fmt.Fprintf(w, "Successfully granted access to users \"%s\" into repository \"%s\"", users, repositories)
}

func RevokeAccess(w http.ResponseWriter, r *http.Request) {
	repositories, users, err := accessParameters(r.Body)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	if err := repository.RevokeAccess(repositories, users); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	fmt.Fprintf(w, "Successfully revoked access to users \"%s\" into repositories \"%s\"", users, repositories)
}

func AddKey(w http.ResponseWriter, r *http.Request) {
	keys := map[string]string{}
	if err := parseBody(r.Body, &keys); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	if len(keys) == 0 {
		http.Error(w, "A key is needed", http.StatusBadRequest)
		return
	}
	uName := r.URL.Query().Get(":name")
	if err := user.AddKey(uName, keys); err != nil {
		switch err {
		case user.ErrInvalidKey:
			http.Error(w, err.Error(), http.StatusBadRequest)
		case user.ErrDuplicateKey:
			http.Error(w, "Key already exists.", http.StatusConflict)
		case user.ErrUserNotFound:
			http.Error(w, err.Error(), http.StatusNotFound)
		default:
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
		return
	}
	fmt.Fprint(w, "Key(s) successfully created")
}

func RemoveKey(w http.ResponseWriter, r *http.Request) {
	uName := r.URL.Query().Get(":name")
	kName := r.URL.Query().Get(":keyname")
	if err := user.RemoveKey(uName, kName); err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}
	fmt.Fprintf(w, "Key \"%s\" successfully removed", kName)
}

func ListKeys(w http.ResponseWriter, r *http.Request) {
	uName := r.URL.Query().Get(":name")
	keys, err := user.ListKeys(uName)
	if err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}
	out, err := json.Marshal(&keys)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Write(out)
}

type jsonUser struct {
	Name string
	Keys map[string]string
}

func NewUser(w http.ResponseWriter, r *http.Request) {
	var usr jsonUser
	if err := parseBody(r.Body, &usr); err != nil {
		http.Error(w, "Got error while parsing body: "+err.Error(), http.StatusBadRequest)
		return
	}
	u, err := user.New(usr.Name, usr.Keys)
	if err != nil {
		http.Error(w, "Got error while creating user: "+err.Error(), http.StatusBadRequest)
		return
	}
	fmt.Fprintf(w, "User \"%s\" successfully created\n", u.Name)
}

func RemoveUser(w http.ResponseWriter, r *http.Request) {
	name := r.URL.Query().Get(":name")
	if err := user.Remove(name); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	fmt.Fprintf(w, "User \"%s\" successfully removed\n", name)
}

func NewRepository(w http.ResponseWriter, r *http.Request) {
	var repo repository.Repository
	if err := parseBody(r.Body, &repo); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	rep, err := repository.New(repo.Name, repo.Users, repo.IsPublic)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	fmt.Fprintf(w, "Repository \"%s\" successfully created\n", rep.Name)
}

func GetRepository(w http.ResponseWriter, r *http.Request) {
	repo, err := repository.Get(r.URL.Query().Get(":name"))
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	out, err := json.Marshal(&repo)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Write(out)
}

func RemoveRepository(w http.ResponseWriter, r *http.Request) {
	name := r.URL.Query().Get(":name")
	if err := repository.Remove(name); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	fmt.Fprintf(w, "Repository \"%s\" successfully removed\n", name)
}

func RenameRepository(w http.ResponseWriter, r *http.Request) {
	var p struct{ Name string }
	defer r.Body.Close()
	err := parseBody(r.Body, &p)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
	}
	name := r.URL.Query().Get(":name")
	err = repository.Rename(name, p.Name)
	if err != nil && err.Error() == "not found" {
		http.Error(w, err.Error(), http.StatusNotFound)
	} else if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func parseBody(body io.ReadCloser, result interface{}) error {
	if reflect.ValueOf(result).Kind() == reflect.Struct {
		return errors.New("parseBody function cannot deal with struct. Use pointer")
	}
	b, err := ioutil.ReadAll(body)
	if err != nil {
		return err
	}
	if err := json.Unmarshal(b, &result); err != nil {
		return errors.New(fmt.Sprintf("Could not parse json: %s", err.Error()))
	}
	return nil
}

func HealthCheck(w http.ResponseWriter, r *http.Request) {
	if err := db.Session.DB.Session.Ping(); err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintf(w, "Failed to ping the database: %s\n", err)
		return
	}
	w.Write([]byte("WORKING"))
}
<|endoftext|>


<commit_before>package api

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"time"

	"code.cloudfoundry.org/bytefmt"

	"golang.org/x/net/websocket"

	"net/url"

	sigar "github.com/cloudfoundry/gosigar"
	"github.com/hyperhq/runv/hypervisor/pod"
	"github.com/labstack/echo"
)

func (a *Api) CreateBox(c echo.Context) error {
	body, err := ioutil.ReadAll(c.Request().Body)
	if err != nil {
		return err
	}

	var req CreateBoxRequest
	if err = json.Unmarshal(body, &req); err != nil {
		return err
	}

	// validate captcha if secret is configured
	if a.Config.RCSecret != "" {
		data := url.Values{}
		data.Set("secret", a.Config.RCSecret)
		data.Set("response", req.Captcha)
		data.Set("remoteip", c.RealIP())

		res, err := http.PostForm("https://www.google.com/recaptcha/api/siteverify", data)
		if err != nil {
			return err
		}

		defer res.Body.Close()

		var verify CaptchaVerifyResponse
		if err := json.NewDecoder(res.Body).Decode(&verify); err != nil {
			return err
		}
		if !verify.Success {
			return c.String(http.StatusBadRequest, "Captcha verification failed")
		}
	} else {
		a.Log.Warn("Creating box without captcha verfication")
	}

	// verify image is whitelisted
	var image Image
	for _, i := range *a.Images {
		if req.Image == i.Image {
			for _, version := range i.Versions {
				if req.Version == version {
					image = i
				}
			}
		}
	}
	if image.Image == "" {
		return c.String(http.StatusBadRequest, "Image not allowed")
	}

	// make sure we are not running out of memory
	mem := sigar.Mem{}
	mem.Get()
	if mem.ActualFree < bytefmt.GIGABYTE {
		return c.String(http.StatusTooManyRequests, "Server capacity reached , please try again later")
	}

	container := pod.UserContainer{
		Image:   fmt.Sprintf("%s:%s", image.Image, req.Version),
		Command: []string{"sh"},
		Envs: []pod.UserEnvironmentVar{
			pod.UserEnvironmentVar{
				Env:   "PORT",
				Value: "2000",
			},
		},
	}

	if a.Config.PublicAddr != "" {
		container.Ports = []pod.UserContainerPort{
			pod.UserContainerPort{
				// random port between 10000 and 60000
				HostPort:      10000 + rand.Intn(50000),
				ContainerPort: 2000,
			},
		}
	}

	pod := pod.UserPod{
		Name:       "termbox",
		Hostname:   image.Name,
		Containers: []pod.UserContainer{container},
		Resource:   pod.UserResource{Vcpu: a.Config.BoxCpus, Memory: a.Config.BoxMemory},
	}

	podID, statusCode, err := a.Hyper.CreatePod(pod)
	if err != nil {
		if statusCode == http.StatusNotFound {
			if err := a.HyperClient.PullImages(&pod); err != nil {
				return err
			}
			podID, statusCode, err = a.Hyper.CreatePod(pod)
			if err != nil {
				return err
			}
		} else {
			return err
		}
	}

	if a.Config.PublicAddr != "" {
		a.Log.Infof("IP Address %v created %v with public port %v", c.RealIP(), podID, container.Ports[0].HostPort)
	}

	return c.JSON(http.StatusOK, CreateBoxResponse{PodID: podID})
}

func (a *Api) ExecBox(c echo.Context) error {

	podID := c.Param("id")
	podInfo, err := a.Hyper.GetPodInfo(podID)
	if err != nil {
		return c.String(http.StatusNotFound, "Box does not exist")
	}

	websocket.Handler(func(ws *websocket.Conn) {
		defer ws.Close()

		if podInfo.Status.Phase != "Running" {
			_, err := a.Hyper.StartPod(podID, "", false, false, nil, nil, nil)
			if err != nil {
				a.Log.Warn(err)
				return
			}
		}
		containerID, _ := a.Hyper.GetContainerByPod(podID)

		command, err := json.Marshal([]string{"sh", "-c", "tmux attach -d 2> /dev/null || tmux"})
		if err != nil {
			a.Log.Error(err)
			return
		}

		execID, err := a.Hyper.CreateExec(containerID, command, true)
		if err != nil {
			a.Log.Error(err)
			return
		}

		dec := json.NewDecoder(ws)

		r, w := io.Pipe()
		defer r.Close()
		defer w.Close()

		go func() {
			if err := a.Hyper.StartExec(containerID, execID, true, r, ws, ws); err != nil {
				a.Log.Error(err)
			}
		}()

		for dec.More() {
			var message ExecBoxMessage
			if err := dec.Decode(&message); err != nil {
				a.Log.Error(err)
				break
			}

			if message.Width != 0 && message.Height != 0 {
				if err := a.Hyper.WinResize(containerID, execID, message.Height, message.Width); err != nil {
					a.Log.Warn(err)
				}
				continue
			}
			if message.Data != "" {
				_, err = io.WriteString(w, message.Data)
				if err != nil {
					a.Log.Error(err)
					break
				}
			}
		}

	}).ServeHTTP(c.Response(), c.Request())
	return nil

}

func (a *Api) GetBox(c echo.Context) error {

	podID := c.Param("id")
	podInfo, err := a.Hyper.GetPodInfo(podID)
	if err != nil {
		return c.String(http.StatusNotFound, "Box does not exist")
	}

	remaining := time.Duration(a.Config.BoxDuration)*time.Hour - time.Since(time.Unix(podInfo.CreatedAt, 0))

	container := podInfo.Spec.Containers[0]

	res := GetBoxResponse{
		Id:            podID,
		TimeRemaining: int(remaining.Seconds()),
		Image:         container.Image,
		Port:          int(container.Ports[0].HostPort),
	}

	return c.JSON(http.StatusOK, res)
}
<commit_msg>include port only if it exists<commit_after>package api

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"time"

	"code.cloudfoundry.org/bytefmt"

	"golang.org/x/net/websocket"

	"net/url"

	sigar "github.com/cloudfoundry/gosigar"
	"github.com/hyperhq/runv/hypervisor/pod"
	"github.com/labstack/echo"
)

func (a *Api) CreateBox(c echo.Context) error {
	body, err := ioutil.ReadAll(c.Request().Body)
	if err != nil {
		return err
	}

	var req CreateBoxRequest
	if err = json.Unmarshal(body, &req); err != nil {
		return err
	}

	// validate captcha if secret is configured
	if a.Config.RCSecret != "" {
		data := url.Values{}
		data.Set("secret", a.Config.RCSecret)
		data.Set("response", req.Captcha)
		data.Set("remoteip", c.RealIP())

		res, err := http.PostForm("https://www.google.com/recaptcha/api/siteverify", data)
		if err != nil {
			return err
		}

		defer res.Body.Close()

		var verify CaptchaVerifyResponse
		if err := json.NewDecoder(res.Body).Decode(&verify); err != nil {
			return err
		}
		if !verify.Success {
			return c.String(http.StatusBadRequest, "Captcha verification failed")
		}
	} else {
		a.Log.Warn("Creating box without captcha verfication")
	}

	// verify image is whitelisted
	var image Image
	for _, i := range *a.Images {
		if req.Image == i.Image {
			for _, version := range i.Versions {
				if req.Version == version {
					image = i
				}
			}
		}
	}
	if image.Image == "" {
		return c.String(http.StatusBadRequest, "Image not allowed")
	}

	// make sure we are not running out of memory
	mem := sigar.Mem{}
	mem.Get()
	if mem.ActualFree < bytefmt.GIGABYTE {
		return c.String(http.StatusTooManyRequests, "Server capacity reached , please try again later")
	}

	container := pod.UserContainer{
		Image:   fmt.Sprintf("%s:%s", image.Image, req.Version),
		Command: []string{"sh"},
		Envs: []pod.UserEnvironmentVar{
			pod.UserEnvironmentVar{
				Env:   "PORT",
				Value: "2000",
			},
		},
	}

	if a.Config.PublicAddr != "" {
		container.Ports = []pod.UserContainerPort{
			pod.UserContainerPort{
				// random port between 10000 and 60000
				HostPort:      10000 + rand.Intn(50000),
				ContainerPort: 2000,
			},
		}
	}

	pod := pod.UserPod{
		Name:       "termbox",
		Hostname:   image.Name,
		Containers: []pod.UserContainer{container},
		Resource:   pod.UserResource{Vcpu: a.Config.BoxCpus, Memory: a.Config.BoxMemory},
	}

	podID, statusCode, err := a.Hyper.CreatePod(pod)
	if err != nil {
		if statusCode == http.StatusNotFound {
			if err := a.HyperClient.PullImages(&pod); err != nil {
				return err
			}
			podID, statusCode, err = a.Hyper.CreatePod(pod)
			if err != nil {
				return err
			}
		} else {
			return err
		}
	}

	if a.Config.PublicAddr != "" {
		a.Log.Infof("IP Address %v created %v with public port %v", c.RealIP(), podID, container.Ports[0].HostPort)
	}

	return c.JSON(http.StatusOK, CreateBoxResponse{PodID: podID})
}

func (a *Api) ExecBox(c echo.Context) error {

	podID := c.Param("id")
	podInfo, err := a.Hyper.GetPodInfo(podID)
	if err != nil {
		return c.String(http.StatusNotFound, "Box does not exist")
	}

	websocket.Handler(func(ws *websocket.Conn) {
		defer ws.Close()

		if podInfo.Status.Phase != "Running" {
			_, err := a.Hyper.StartPod(podID, "", false, false, nil, nil, nil)
			if err != nil {
				a.Log.Warn(err)
				return
			}
		}
		containerID, _ := a.Hyper.GetContainerByPod(podID)

		command, err := json.Marshal([]string{"sh", "-c", "tmux attach -d 2> /dev/null || tmux"})
		if err != nil {
			a.Log.Error(err)
			return
		}

		execID, err := a.Hyper.CreateExec(containerID, command, true)
		if err != nil {
			a.Log.Error(err)
			return
		}

		dec := json.NewDecoder(ws)

		r, w := io.Pipe()
		defer r.Close()
		defer w.Close()

		go func() {
			if err := a.Hyper.StartExec(containerID, execID, true, r, ws, ws); err != nil {
				a.Log.Error(err)
			}
		}()

		for dec.More() {
			var message ExecBoxMessage
			if err := dec.Decode(&message); err != nil {
				a.Log.Error(err)
				break
			}

			if message.Width != 0 && message.Height != 0 {
				if err := a.Hyper.WinResize(containerID, execID, message.Height, message.Width); err != nil {
					a.Log.Warn(err)
				}
				continue
			}
			if message.Data != "" {
				_, err = io.WriteString(w, message.Data)
				if err != nil {
					a.Log.Error(err)
					break
				}
			}
		}

	}).ServeHTTP(c.Response(), c.Request())
	return nil

}

func (a *Api) GetBox(c echo.Context) error {

	podID := c.Param("id")
	podInfo, err := a.Hyper.GetPodInfo(podID)
	if err != nil {
		return c.String(http.StatusNotFound, "Box does not exist")
	}

	remaining := time.Duration(a.Config.BoxDuration)*time.Hour - time.Since(time.Unix(podInfo.CreatedAt, 0))

	container := podInfo.Spec.Containers[0]

	res := GetBoxResponse{
		Id:            podID,
		TimeRemaining: int(remaining.Seconds()),
		Image:         container.Image,
	}

	if len(container.Ports) != 0 {
		res.Port = int(container.Ports[0].HostPort)
	}

	return c.JSON(http.StatusOK, res)
}
<|endoftext|>


<commit_before>package httphandle

import (
	"go-server-template/api"
	"go-server-template/config"
	"go-server-template/filter"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"reflect"
)

var apiInterface interface{}

func SetApiInterface(interf interface{}) {
	apiInterface = interf
}

func PerformApiCall(handlerName string, rw http.ResponseWriter, req *http.Request, route *config.Route) {
	// Prepare data vector for an api/endpoint call
	inputs := make([]reflect.Value, 1)

	// Create the variables containing request data
	vars := createApiVars(req, rw, route)
	if vars == nil {
		return
	}

	// Populate the data vector for the api call
	inputs[0] = reflect.ValueOf(vars)

	// Perform the call on the corresponding endpoint and function
	// This is done by using reflection techniques
	var respObjects []reflect.Value
	apiMethod := reflect.ValueOf(apiInterface).MethodByName(route.Handlers[req.Method])

	// Check for zero value
	if apiMethod != *new(reflect.Value) {
		respObjects = apiMethod.Call(inputs)
	} else {
		GiveApiStatus(http.StatusInternalServerError, rw, req, route.Pattern)

		log.Println("The endpoint method is either inexistent or incorrectly mapped. Please check the server configuration files!")

		return
	}

	if respObjects == nil {
		GiveApiStatus(http.StatusInternalServerError, rw, req, route.Pattern)
		return
	}

	// Extract the response from the endpoint into a concrete type
	resp := respObjects[0].Interface().(api.ApiResponse)

	// Give the response to the api client
	if resp.StatusCode == 0 {
		resp.StatusCode = http.StatusInternalServerError
		GiveApiMessage(resp.StatusCode, http.StatusText(resp.StatusCode), rw, req, route.Pattern)
	} else if len(resp.ErrorMessage) > 0 {
		GiveApiMessage(resp.StatusCode, resp.ErrorMessage, rw, req, route.Pattern)
	} else {
		if len(resp.ContentType) == 0 {
			resp.ContentType = ContentJSON
		}

		GiveApiResponse(resp.StatusCode, resp.Message, rw, req, route.Pattern, resp.ContentType, resp.File)
	}
}

func createApiVars(req *http.Request, rw http.ResponseWriter, route *config.Route) *api.ApiVar {
	err, statusCode := filter.CheckMethodAndParseContent(req)
	if err != nil {
		GiveApiMessage(statusCode, err.Error(), rw, req, route.Pattern)
		return nil
	}

	body, err := convertBodyToReadableFormat(req.Body)
	if err != nil {
		GiveApiMessage(http.StatusBadRequest, err.Error(), rw, req, route.Pattern)
		return nil
	}

	vars := &api.ApiVar{
		RequestHeader:        req.Header,
		RequestForm:          req.Form,
		RequestContentLength: req.ContentLength,
		RequestBody:          body,
	}

	return vars
}

func convertBodyToReadableFormat(data io.ReadCloser) ([]byte, error) {
	body, err := ioutil.ReadAll(data)

	return body, err
}
<commit_msg>Fixed point where the server would timeout and return a http status 0 instead of an internal server error<commit_after>package httphandle

import (
	"go-server-template/api"
	"go-server-template/config"
	"go-server-template/filter"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"reflect"
)

var apiInterface interface{}

func SetApiInterface(interf interface{}) {
	apiInterface = interf
}

func PerformApiCall(handlerName string, rw http.ResponseWriter, req *http.Request, route *config.Route) {
	// Prepare data vector for an api/endpoint call
	inputs := make([]reflect.Value, 1)

	// Create the variables containing request data
	vars := createApiVars(req, rw, route)
	if vars == nil {
		GiveApiStatus(http.StatusInternalServerError, rw, req, route.Pattern)
		return
	}

	// Populate the data vector for the api call
	inputs[0] = reflect.ValueOf(vars)

	// Perform the call on the corresponding endpoint and function
	// This is done by using reflection techniques
	var respObjects []reflect.Value
	apiMethod := reflect.ValueOf(apiInterface).MethodByName(route.Handlers[req.Method])

	// Check for zero value
	if apiMethod != *new(reflect.Value) {
		respObjects = apiMethod.Call(inputs)
	} else {
		GiveApiStatus(http.StatusInternalServerError, rw, req, route.Pattern)

		log.Println("The endpoint method is either inexistent or incorrectly mapped. Please check the server configuration files!")

		return
	}

	if respObjects == nil {
		GiveApiStatus(http.StatusInternalServerError, rw, req, route.Pattern)
		return
	}

	// Extract the response from the endpoint into a concrete type
	resp := respObjects[0].Interface().(api.ApiResponse)

	// Give the response to the api client
	if resp.StatusCode == 0 {
		resp.StatusCode = http.StatusInternalServerError
		GiveApiMessage(resp.StatusCode, http.StatusText(resp.StatusCode), rw, req, route.Pattern)
	} else if len(resp.ErrorMessage) > 0 {
		GiveApiMessage(resp.StatusCode, resp.ErrorMessage, rw, req, route.Pattern)
	} else {
		if len(resp.ContentType) == 0 {
			resp.ContentType = ContentJSON
		}

		GiveApiResponse(resp.StatusCode, resp.Message, rw, req, route.Pattern, resp.ContentType, resp.File)
	}
}

func createApiVars(req *http.Request, rw http.ResponseWriter, route *config.Route) *api.ApiVar {
	err, statusCode := filter.CheckMethodAndParseContent(req)
	if err != nil {
		GiveApiMessage(statusCode, err.Error(), rw, req, route.Pattern)
		return nil
	}

	body, err := convertBodyToReadableFormat(req.Body)
	if err != nil {
		GiveApiMessage(http.StatusBadRequest, err.Error(), rw, req, route.Pattern)
		return nil
	}

	vars := &api.ApiVar{
		RequestHeader:        req.Header,
		RequestForm:          req.Form,
		RequestContentLength: req.ContentLength,
		RequestBody:          body,
	}

	return vars
}

func convertBodyToReadableFormat(data io.ReadCloser) ([]byte, error) {
	body, err := ioutil.ReadAll(data)

	return body, err
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package testing

import (
	"bytes"
	"text/template"

	gc "launchpad.net/gocheck"
)

var expectedAccumulateSyslogConfTemplate = `
$ModLoad imfile

$InputFilePersistStateInterval 50
$InputFilePollInterval 5
$InputFileName /var/log/juju/{{machine}}.log
$InputFileTag juju{{namespace}}-{{machine}}:
$InputFileStateFile {{machine}}{{namespace}}
$InputRunFileMonitor

$ModLoad imudp
$UDPServerRun {{port}}

# Messages received from remote rsyslog machines have messages prefixed with a space,
# so add one in for local messages too if needed.
$template JujuLogFormat{{namespace}},"%syslogtag:{{offset}}:$%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"

$FileCreateMode 0644
:syslogtag, startswith, "juju{{namespace}}-" /var/log/juju{{namespace}}/all-machines.log;JujuLogFormat{{namespace}}
& ~
$FileCreateMode 0640
`

func ExpectedAccumulateSyslogConf(c *gc.C, machineTag, namespace string, port int) string {
	if namespace != "" {
		namespace = "-" + namespace
	}
	t := template.New("")
	t.Funcs(template.FuncMap{
		"machine":   func() string { return machineTag },
		"namespace": func() string { return namespace },
		"port":      func() int { return port },
		"offset":    func() int { return 6 + len(namespace) },
	})
	t = template.Must(t.Parse(expectedAccumulateSyslogConfTemplate))
	var conf bytes.Buffer
	err := t.Execute(&conf, nil)
	c.Assert(err, gc.IsNil)
	return conf.String()
}

var expectedForwardSyslogConfTemplate = `
$ModLoad imfile

$InputFilePersistStateInterval 50
$InputFilePollInterval 5
$InputFileName /var/log/juju/{{machine}}.log
$InputFileTag juju{{namespace}}-{{machine}}:
$InputFileStateFile {{machine}}{{namespace}}
$InputRunFileMonitor

$template LongTagForwardFormat,"<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg%"

:syslogtag, startswith, "juju{{namespace}}-" @server:{{port}};LongTagForwardFormat
& ~
`

func ExpectedForwardSyslogConf(c *gc.C, machineTag, namespace string, port int) string {
	if namespace != "" {
		namespace = "-" + namespace
	}
	t := template.New("")
	t.Funcs(template.FuncMap{
		"machine":   func() string { return machineTag },
		"namespace": func() string { return namespace },
		"port":      func() int { return port },
	})
	t = template.Must(t.Parse(expectedForwardSyslogConfTemplate))
	var conf bytes.Buffer
	err := t.Execute(&conf, nil)
	c.Assert(err, gc.IsNil)
	return conf.String()
}
<commit_msg>Add doc<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package testing

import (
	"bytes"
	"text/template"

	gc "launchpad.net/gocheck"
)

var expectedAccumulateSyslogConfTemplate = `
$ModLoad imfile

$InputFilePersistStateInterval 50
$InputFilePollInterval 5
$InputFileName /var/log/juju/{{machine}}.log
$InputFileTag juju{{namespace}}-{{machine}}:
$InputFileStateFile {{machine}}{{namespace}}
$InputRunFileMonitor

$ModLoad imudp
$UDPServerRun {{port}}

# Messages received from remote rsyslog machines have messages prefixed with a space,
# so add one in for local messages too if needed.
$template JujuLogFormat{{namespace}},"%syslogtag:{{offset}}:$%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"

$FileCreateMode 0644
:syslogtag, startswith, "juju{{namespace}}-" /var/log/juju{{namespace}}/all-machines.log;JujuLogFormat{{namespace}}
& ~
$FileCreateMode 0640
`

// ExpectedAccumulateSyslogConf returns the expected content for a rsyslog file on a state server.
func ExpectedAccumulateSyslogConf(c *gc.C, machineTag, namespace string, port int) string {
	if namespace != "" {
		namespace = "-" + namespace
	}
	t := template.New("")
	t.Funcs(template.FuncMap{
		"machine":   func() string { return machineTag },
		"namespace": func() string { return namespace },
		"port":      func() int { return port },
		"offset":    func() int { return 6 + len(namespace) },
	})
	t = template.Must(t.Parse(expectedAccumulateSyslogConfTemplate))
	var conf bytes.Buffer
	err := t.Execute(&conf, nil)
	c.Assert(err, gc.IsNil)
	return conf.String()
}

var expectedForwardSyslogConfTemplate = `
$ModLoad imfile

$InputFilePersistStateInterval 50
$InputFilePollInterval 5
$InputFileName /var/log/juju/{{machine}}.log
$InputFileTag juju{{namespace}}-{{machine}}:
$InputFileStateFile {{machine}}{{namespace}}
$InputRunFileMonitor

$template LongTagForwardFormat,"<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg%"

:syslogtag, startswith, "juju{{namespace}}-" @server:{{port}};LongTagForwardFormat
& ~
`

// ExpectedForwardSyslogConf returns the expected content for a rsyslog file on a host machine.
func ExpectedForwardSyslogConf(c *gc.C, machineTag, namespace string, port int) string {
	if namespace != "" {
		namespace = "-" + namespace
	}
	t := template.New("")
	t.Funcs(template.FuncMap{
		"machine":   func() string { return machineTag },
		"namespace": func() string { return namespace },
		"port":      func() int { return port },
	})
	t = template.Must(t.Parse(expectedForwardSyslogConfTemplate))
	var conf bytes.Buffer
	err := t.Execute(&conf, nil)
	c.Assert(err, gc.IsNil)
	return conf.String()
}
<|endoftext|>


<commit_before>package apns

import (
	// "crypto/tls"
	"encoding/hex"
	"encoding/json"
	"net"
	"testing"
)

var (
	conn *PushConnection
	apn  Notification
)

func BenchmarkSendBuffered(b *testing.B) {
	b.StopTimer()
	token, _ := hex.DecodeString("47ee04b9e673f7ddc86cd126d2504b3661336a60c17e06cec382881b1bd839f8")
	jsonPayload := make(map[string]interface{})
	jsonPayload["aps"] = map[string]string{"alert": "Hello World 1"}
	payload, _ := json.Marshal(jsonPayload)
	apn = NewEnhancedNotification()
	apn.SetToken(token)
	apn.SetPayload(payload)

	// cert, err := tls.LoadX509KeyPair("test_data/cert.pem", "test_data/key.pem")
	// if err != nil {
	// 	b.Error("Failed to load certificate/key pair.")
	// }

	c, err := net.Dial("tcp", "localhost:8080")
	if err != nil {
		b.Error("Connection error.")
	}
	conn := &PushConnection{c}

	b.StartTimer()
	for i := 0; i < b.N; i++ {
		SendBuffered(conn, apn)
	}
	b.StopTimer()
	c.Close()
}

func BenchmarkSendBufferedWithDelay(b *testing.B) {
	b.StopTimer()
	token, _ := hex.DecodeString("47ee04b9e673f7ddc86cd126d2504b3661336a60c17e06cec382881b1bd839f8")
	jsonPayload := make(map[string]interface{})
	jsonPayload["aps"] = map[string]string{"alert": "Hello World 1"}
	payload, _ := json.Marshal(jsonPayload)
	apn = NewEnhancedNotification()
	apn.SetToken(token)
	apn.SetPayload(payload)

	// cert, err := tls.LoadX509KeyPair("test_data/cert.pem", "test_data/key.pem")
	// if err != nil {
	//  b.Error("Failed to load certificate/key pair.")
	// }

	raddr, _ := net.ResolveTCPAddr("tcp", "localhost:8080")
	c, err := net.DialTCP("tcp", nil, raddr)
	if err != nil {
		b.Error("Connection error.")
	}
	c.SetNoDelay(false)
	conn := &PushConnection{c}

	b.StartTimer()
	for i := 0; i < b.N; i++ {
		SendBuffered(conn, apn)
	}
	b.StopTimer()
	c.Close()
}

func BenchmarkSend(b *testing.B) {
	b.StopTimer()
	token, _ := hex.DecodeString("47ee04b9e673f7ddc86cd126d2504b3661336a60c17e06cec382881b1bd839f8")
	jsonPayload := make(map[string]interface{})
	jsonPayload["aps"] = map[string]string{"alert": "Hello World 2"}
	payload, _ := json.Marshal(jsonPayload)
	apn = NewEnhancedNotification()
	apn.SetToken(token)
	apn.SetPayload(payload)

	// cert, err := tls.LoadX509KeyPair("test_data/cert.pem", "test_data/key.pem")
	// if err != nil {
	//  b.Error("Failed to load certificate/key pair.")
	// }

	c, err := net.Dial("tcp", "localhost:8080")
	if err != nil {
		b.Error("Connection error.")
	}
	conn := &PushConnection{c}

	b.StartTimer()
	for i := 0; i < b.N; i++ {
		Send(conn, apn)
	}
	b.StopTimer()
	c.Close()
}

func BenchmarkSendWithDelay(b *testing.B) {
	b.StopTimer()
	token, _ := hex.DecodeString("47ee04b9e673f7ddc86cd126d2504b3661336a60c17e06cec382881b1bd839f8")
	jsonPayload := make(map[string]interface{})
	jsonPayload["aps"] = map[string]string{"alert": "Hello World 2"}
	payload, _ := json.Marshal(jsonPayload)
	apn = NewEnhancedNotification()
	apn.SetToken(token)
	apn.SetPayload(payload)

	// cert, err := tls.LoadX509KeyPair("test_data/cert.pem", "test_data/key.pem")
	// if err != nil {
	// 	b.Error("Failed to load certificate/key pair.")
	// }

	raddr, _ := net.ResolveTCPAddr("tcp", "localhost:8080")
	c, err := net.DialTCP("tcp", nil, raddr)
	if err != nil {
		b.Error("Connection error.")
	}
	c.SetNoDelay(false)
	conn := &PushConnection{c}

	b.StartTimer()
	for i := 0; i < b.N; i++ {
		Send(conn, apn)
	}
	b.StopTimer()
	c.Close()
}
<commit_msg>Remove broken test.<commit_after><|endoftext|>