<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package instancepoller

import (
	"fmt"
	"time"

	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/errors"
	"launchpad.net/juju-core/instance"
	"launchpad.net/juju-core/testing/testbase"
)

type aggregateSuite struct {
	testbase.LoggingSuite
}

var _ = gc.Suite(&aggregateSuite{})

type testInstance struct {
	instance.Instance
	addresses []instance.Address
	status    string
	err       error
}

var _ instance.Instance = (*testInstance)(nil)

func (t *testInstance) Addresses() ([]instance.Address, error) {
	if t.err != nil {
		return nil, t.err
	}
	return t.addresses, nil
}

func (t *testInstance) Status() string {
	return t.status
}

type testInstanceGetter struct {
	ids     []instance.Id
	results []*testInstance
	err     error
}

func (i *testInstanceGetter) Instances(ids []instance.Id) (result []instance.Instance, err error) {
	i.ids = ids
	err = i.err
	for _, inst := range i.results {
		if inst == nil {
			result = append(result, nil)
		} else {
			result = append(result, inst)
		}
	}
	return
}

func newTestInstance(status string, addresses []string) *testInstance {
	thisInstance := testInstance{status: status}
	for _, address := range addresses {
		thisInstance.addresses = append(thisInstance.addresses, instance.NewAddress(address))
	}
	return &thisInstance
}

func (s *aggregateSuite) TestSingleRequest(c *gc.C) {
	testGetter := new(testInstanceGetter)
	instance1 := newTestInstance("foobar", []string{"127.0.0.1", "192.168.1.1"})
	testGetter.results = []*testInstance{instance1}
	aggregator := newAggregator(testGetter)

	info, err := aggregator.instanceInfo("foo")
	c.Assert(err, gc.IsNil)
	c.Assert(info, gc.DeepEquals, instanceInfo{
		status:    "foobar",
		addresses: instance1.addresses,
	})
	c.Assert(testGetter.ids, gc.DeepEquals, []instance.Id{"foo"})
}

func (s *aggregateSuite) TestRequestBatching(c *gc.C) {
	s.PatchValue(&gatherTime, 30*time.Millisecond)
	testGetter := new(testInstanceGetter)

	instance1 := newTestInstance("foobar", []string{"127.0.0.1", "192.168.1.1"})
	testGetter.results = []*testInstance{instance1}
	aggregator := newAggregator(testGetter)

	replyChan := make(chan instanceInfoReply)
	req := instanceInfoReq{
		reply:  replyChan,
		instId: instance.Id("foo"),
	}
	aggregator.reqc <- req
	reply := <-replyChan
	c.Assert(reply.err, gc.IsNil)

	instance2 := newTestInstance("not foobar", []string{"192.168.1.2"})
	instance3 := newTestInstance("ok-ish", []string{"192.168.1.3"})

	replyChan2 := make(chan instanceInfoReply)
	replyChan3 := make(chan instanceInfoReply)

	aggregator.reqc <- instanceInfoReq{reply: replyChan2, instId: instance.Id("foo2")}
	aggregator.reqc <- instanceInfoReq{reply: replyChan3, instId: instance.Id("foo3")}

	testGetter.results = []*testInstance{instance2, instance3}
	reply2 := <-replyChan2
	reply3 := <-replyChan3
	c.Assert(reply2.err, gc.IsNil)
	c.Assert(reply3.err, gc.IsNil)
	c.Assert(reply2.info.status, gc.Equals, "not foobar")
	c.Assert(reply3.info.status, gc.Equals, "ok-ish")

	c.Assert(testGetter.ids, gc.DeepEquals, []instance.Id{instance.Id("foo2"), instance.Id("foo3")})
}

func (s *aggregateSuite) TestError(c *gc.C) {
	testGetter := new(testInstanceGetter)
	ourError := fmt.Errorf("Some error")
	testGetter.err = ourError

	aggregator := newAggregator(testGetter)

	_, err := aggregator.instanceInfo("foo")
	c.Assert(err, gc.Equals, ourError)
}

func (s *aggregateSuite) TestPartialErrResponse(c *gc.C) {
	testGetter := new(testInstanceGetter)
	testGetter.err = environs.ErrPartialInstances
	testGetter.results = []*testInstance{nil}

	aggregator := newAggregator(testGetter)
	_, err := aggregator.instanceInfo("foo")

	c.Assert(err, gc.DeepEquals, errors.NotFoundf("instance foo"))
}

func (s *aggregateSuite) TestAddressesError(c *gc.C) {
	testGetter := new(testInstanceGetter)
	instance1 := newTestInstance("foobar", []string{"127.0.0.1", "192.168.1.1"})
	ourError := fmt.Errorf("gotcha")
	instance1.err = ourError
	testGetter.results = []*testInstance{instance1}

	aggregator := newAggregator(testGetter)
	_, err := aggregator.instanceInfo("foo")
	c.Assert(err, gc.Equals, ourError)
}

func (s *aggregateSuite) TestKillAndWait(c *gc.C) {
	testGetter := new(testInstanceGetter)
	aggregator := newAggregator(testGetter)
	aggregator.Kill()
	err := aggregator.Wait()
	c.Assert(err, gc.IsNil)
}
<commit_msg>Don't use channel directly in batch test<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package instancepoller

import (
	"fmt"
	"sync"
	"time"

	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/errors"
	"launchpad.net/juju-core/instance"
	"launchpad.net/juju-core/testing/testbase"
)

type aggregateSuite struct {
	testbase.LoggingSuite
}

var _ = gc.Suite(&aggregateSuite{})

type testInstance struct {
	instance.Instance
	addresses []instance.Address
	status    string
	err       error
}

var _ instance.Instance = (*testInstance)(nil)

func (t *testInstance) Addresses() ([]instance.Address, error) {
	if t.err != nil {
		return nil, t.err
	}
	return t.addresses, nil
}

func (t *testInstance) Status() string {
	return t.status
}

type testInstanceGetter struct {
	ids     []instance.Id
	results []*testInstance
	err     error
}

func (i *testInstanceGetter) Instances(ids []instance.Id) (result []instance.Instance, err error) {
	i.ids = ids
	err = i.err
	for _, inst := range i.results {
		if inst == nil {
			result = append(result, nil)
		} else {
			result = append(result, inst)
		}
	}
	return
}

func newTestInstance(status string, addresses []string) *testInstance {
	thisInstance := testInstance{status: status}
	for _, address := range addresses {
		thisInstance.addresses = append(thisInstance.addresses, instance.NewAddress(address))
	}
	return &thisInstance
}

func (s *aggregateSuite) TestSingleRequest(c *gc.C) {
	testGetter := new(testInstanceGetter)
	instance1 := newTestInstance("foobar", []string{"127.0.0.1", "192.168.1.1"})
	testGetter.results = []*testInstance{instance1}
	aggregator := newAggregator(testGetter)

	info, err := aggregator.instanceInfo("foo")
	c.Assert(err, gc.IsNil)
	c.Assert(info, gc.DeepEquals, instanceInfo{
		status:    "foobar",
		addresses: instance1.addresses,
	})
	c.Assert(testGetter.ids, gc.DeepEquals, []instance.Id{"foo"})
}

func (s *aggregateSuite) TestRequestBatching(c *gc.C) {
	s.PatchValue(&gatherTime, 30*time.Millisecond)
	testGetter := new(testInstanceGetter)

	instance1 := newTestInstance("foobar", []string{"127.0.0.1", "192.168.1.1"})
	testGetter.results = []*testInstance{instance1}
	aggregator := newAggregator(testGetter)

	replyChan := make(chan instanceInfoReply)
	req := instanceInfoReq{
		reply:  replyChan,
		instId: instance.Id("foo"),
	}
	aggregator.reqc <- req
	reply := <-replyChan
	c.Assert(reply.err, gc.IsNil)

	instance2 := newTestInstance("not foobar", []string{"192.168.1.2"})
	instance3 := newTestInstance("ok-ish", []string{"192.168.1.3"})
	testGetter.results = []*testInstance{instance2, instance3}

	var wg sync.WaitGroup
	checkInfo := func(id instance.Id, expectStatus string) {
		info, err := aggregator.instanceInfo(id)
		c.Check(err, gc.IsNil)
		c.Check(info.status, gc.Equals, expectStatus)
		wg.Done()
	}

	wg.Add(2)
	go checkInfo("foo2", "not foobar")
	go checkInfo("foo3", "ok-ish")
	wg.Wait()

	c.Assert(len(testGetter.ids), gc.DeepEquals, 2)
}

func (s *aggregateSuite) TestError(c *gc.C) {
	testGetter := new(testInstanceGetter)
	ourError := fmt.Errorf("Some error")
	testGetter.err = ourError

	aggregator := newAggregator(testGetter)

	_, err := aggregator.instanceInfo("foo")
	c.Assert(err, gc.Equals, ourError)
}

func (s *aggregateSuite) TestPartialErrResponse(c *gc.C) {
	testGetter := new(testInstanceGetter)
	testGetter.err = environs.ErrPartialInstances
	testGetter.results = []*testInstance{nil}

	aggregator := newAggregator(testGetter)
	_, err := aggregator.instanceInfo("foo")

	c.Assert(err, gc.DeepEquals, errors.NotFoundf("instance foo"))
}

func (s *aggregateSuite) TestAddressesError(c *gc.C) {
	testGetter := new(testInstanceGetter)
	instance1 := newTestInstance("foobar", []string{"127.0.0.1", "192.168.1.1"})
	ourError := fmt.Errorf("gotcha")
	instance1.err = ourError
	testGetter.results = []*testInstance{instance1}

	aggregator := newAggregator(testGetter)
	_, err := aggregator.instanceInfo("foo")
	c.Assert(err, gc.Equals, ourError)
}

func (s *aggregateSuite) TestKillAndWait(c *gc.C) {
	testGetter := new(testInstanceGetter)
	aggregator := newAggregator(testGetter)
	aggregator.Kill()
	err := aggregator.Wait()
	c.Assert(err, gc.IsNil)
}
<|endoftext|>


<commit_before>// Package reference is a fork of the upstream docker/docker/reference package.
// The package is forked because we need consistency especially when storing and
// checking signatures (RH patches break this consistency because they modify
// docker/docker/reference as part of a patch carried in projectatomic/docker).
// The version of this package is v1.12.1 from upstream, update as necessary.
package reference
<commit_msg>Finally, get rid of the last remains of c/i/docker/reference.<commit_after><|endoftext|>


<commit_before>package driver

import (
	"errors"
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/hashicorp/nomad/client/allocdir"
	"github.com/hashicorp/nomad/client/config"
	"github.com/hashicorp/nomad/client/driver/env"
	"github.com/hashicorp/nomad/client/fingerprint"
	"github.com/hashicorp/nomad/nomad/structs"

	dstructs "github.com/hashicorp/nomad/client/driver/structs"
	cstructs "github.com/hashicorp/nomad/client/structs"
)

var (
	// BuiltinDrivers contains the built in registered drivers
	// which are available for allocation handling
	BuiltinDrivers = map[string]Factory{
		"docker":   NewDockerDriver,
		"exec":     NewExecDriver,
		"raw_exec": NewRawExecDriver,
		"java":     NewJavaDriver,
		"qemu":     NewQemuDriver,
		"rkt":      NewRktDriver,
	}

	// DriverStatsNotImplemented is the error to be returned if a driver doesn't
	// implement stats.
	DriverStatsNotImplemented = errors.New("stats not implemented for driver")
)

// NewDriver is used to instantiate and return a new driver
// given the name and a logger
func NewDriver(name string, ctx *DriverContext) (Driver, error) {
	// Lookup the factory function
	factory, ok := BuiltinDrivers[name]
	if !ok {
		return nil, fmt.Errorf("unknown driver '%s'", name)
	}

	// Instantiate the driver
	f := factory(ctx)
	return f, nil
}

// Factory is used to instantiate a new Driver
type Factory func(*DriverContext) Driver

// CreatedResources is a map of resources (eg downloaded images) created by a driver
// that must be cleaned up.
type CreatedResources struct {
	Resources map[string][]string
}

func NewCreatedResources() *CreatedResources {
	return &CreatedResources{Resources: make(map[string][]string)}
}

// Add a new resource if it doesn't already exist.
func (r *CreatedResources) Add(k, v string) {
	if r.Resources == nil {
		r.Resources = map[string][]string{k: []string{v}}
		return
	}
	existing, ok := r.Resources[k]
	if !ok {
		// Key doesn't exist, create it
		r.Resources[k] = []string{v}
		return
	}
	for _, item := range existing {
		if item == v {
			// resource exists, return
			return
		}
	}

	// Resource type exists but value did not, append it
	r.Resources[k] = append(existing, v)
	return
}

// Remove a resource. Return true if removed, otherwise false.
//
// Removes the entire key if the needle is the last value in the list.
func (r *CreatedResources) Remove(k, needle string) bool {
	haystack := r.Resources[k]
	for i, item := range haystack {
		if item == needle {
			r.Resources[k] = append(haystack[:i], haystack[i+1:]...)
			if len(r.Resources[k]) == 0 {
				delete(r.Resources, k)
			}
			return true
		}
	}
	return false
}

// Copy returns a new deep copy of CreatedResrouces.
func (r *CreatedResources) Copy() *CreatedResources {
	newr := CreatedResources{
		Resources: make(map[string][]string, len(r.Resources)),
	}
	for k, v := range r.Resources {
		newv := make([]string, len(v))
		copy(newv, v)
		newr.Resources[k] = newv
	}
	return &newr
}

// Merge another CreatedResources into this one. If the other CreatedResources
// is nil this method is a noop.
func (r *CreatedResources) Merge(o *CreatedResources) {
	if o == nil {
		return
	}

	for k, v := range o.Resources {
		// New key
		if len(r.Resources[k]) == 0 {
			r.Resources[k] = v
			continue
		}

		// Existing key
	OUTER:
		for _, item := range v {
			for _, existing := range r.Resources[k] {
				if item == existing {
					// Found it, move on
					continue OUTER
				}
			}

			// New item, append it
			r.Resources[k] = append(r.Resources[k], item)
		}
	}
}

// Driver is used for execution of tasks. This allows Nomad
// to support many pluggable implementations of task drivers.
// Examples could include LXC, Docker, Qemu, etc.
type Driver interface {
	// Drivers must support the fingerprint interface for detection
	fingerprint.Fingerprint

	// Prestart prepares the task environment and performs expensive
	// intialization steps like downloading images.
	//
	// CreatedResources may be non-nil even when an error occurs.
	Prestart(*ExecContext, *structs.Task) (*CreatedResources, error)

	// Start is used to being task execution
	Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error)

	// Open is used to re-open a handle to a task
	Open(ctx *ExecContext, handleID string) (DriverHandle, error)

	// Cleanup is called to remove resources which were created for a task
	// and no longer needed.
	//
	// If Cleanup returns a recoverable error it may be retried. On retry
	// it will be passed the same CreatedResources, so all successfully
	// cleaned up resources should be removed.
	Cleanup(*ExecContext, *CreatedResources) error

	// Drivers must validate their configuration
	Validate(map[string]interface{}) error

	// Abilities returns the abilities of the driver
	Abilities() DriverAbilities

	// FSIsolation returns the method of filesystem isolation used
	FSIsolation() cstructs.FSIsolation
}

// DriverAbilities marks the abilities the driver has.
type DriverAbilities struct {
	// SendSignals marks the driver as being able to send signals
	SendSignals bool
}

// LogEventFn is a callback which allows Drivers to emit task events.
type LogEventFn func(message string, args ...interface{})

// DriverContext is a means to inject dependencies such as loggers, configs, and
// node attributes into a Driver without having to change the Driver interface
// each time we do it. Used in conjection with Factory, above.
type DriverContext struct {
	taskName string
	config   *config.Config
	logger   *log.Logger
	node     *structs.Node
	taskEnv  *env.TaskEnvironment

	emitEvent LogEventFn
}

// NewEmptyDriverContext returns a DriverContext with all fields set to their
// zero value.
func NewEmptyDriverContext() *DriverContext {
	return &DriverContext{}
}

// NewDriverContext initializes a new DriverContext with the specified fields.
// This enables other packages to create DriverContexts but keeps the fields
// private to the driver. If we want to change this later we can gorename all of
// the fields in DriverContext.
func NewDriverContext(taskName string, config *config.Config, node *structs.Node,
	logger *log.Logger, taskEnv *env.TaskEnvironment, eventEmitter LogEventFn) *DriverContext {
	return &DriverContext{
		taskName:  taskName,
		config:    config,
		node:      node,
		logger:    logger,
		taskEnv:   taskEnv,
		emitEvent: eventEmitter,
	}
}

// DriverHandle is an opaque handle into a driver used for task
// manipulation
type DriverHandle interface {
	// Returns an opaque handle that can be used to re-open the handle
	ID() string

	// WaitCh is used to return a channel used wait for task completion
	WaitCh() chan *dstructs.WaitResult

	// Update is used to update the task if possible and update task related
	// configurations.
	Update(task *structs.Task) error

	// Kill is used to stop the task
	Kill() error

	// Stats returns aggregated stats of the driver
	Stats() (*cstructs.TaskResourceUsage, error)

	// Signal is used to send a signal to the task
	Signal(s os.Signal) error
}

// ExecContext is a task's execution context
type ExecContext struct {
	// TaskDir contains information about the task directory structure.
	TaskDir *allocdir.TaskDir

	// Alloc ID
	AllocID string
}

// NewExecContext is used to create a new execution context
func NewExecContext(td *allocdir.TaskDir, allocID string) *ExecContext {
	return &ExecContext{
		TaskDir: td,
		AllocID: allocID,
	}
}

// GetTaskEnv converts the alloc dir, the node, task and alloc into a
// TaskEnvironment.
func GetTaskEnv(taskDir *allocdir.TaskDir, node *structs.Node,
	task *structs.Task, alloc *structs.Allocation, conf *config.Config,
	vaultToken string) (*env.TaskEnvironment, error) {

	env := env.NewTaskEnvironment(node).
		SetTaskMeta(alloc.Job.CombinedTaskMeta(alloc.TaskGroup, task.Name)).
		SetJobName(alloc.Job.Name).
		SetEnvvars(task.Env).
		SetTaskName(task.Name)

	// Vary paths by filesystem isolation used
	drv, err := NewDriver(task.Driver, NewEmptyDriverContext())
	if err != nil {
		return nil, err
	}
	switch drv.FSIsolation() {
	case cstructs.FSIsolationNone:
		// Use host paths
		env.SetAllocDir(taskDir.SharedAllocDir)
		env.SetTaskLocalDir(taskDir.LocalDir)
		env.SetSecretsDir(taskDir.SecretsDir)
	default:
		// filesystem isolation; use container paths
		env.SetAllocDir(allocdir.SharedAllocContainerPath)
		env.SetTaskLocalDir(allocdir.TaskLocalContainerPath)
		env.SetSecretsDir(allocdir.TaskSecretsContainerPath)
	}

	if task.Resources != nil {
		env.SetMemLimit(task.Resources.MemoryMB).
			SetCpuLimit(task.Resources.CPU).
			SetNetworks(task.Resources.Networks)
	}

	if alloc != nil {
		env.SetAlloc(alloc)
	}

	if task.Vault != nil {
		env.SetVaultToken(vaultToken, task.Vault.Env)
	}

	// Set the host environment variables for non-image based drivers
	if drv.FSIsolation() != cstructs.FSIsolationImage {
		filter := strings.Split(conf.ReadDefault("env.blacklist", config.DefaultEnvBlacklist), ",")
		env.AppendHostEnvvars(filter)
	}

	return env.Build(), nil
}

func mapMergeStrInt(maps ...map[string]int) map[string]int {
	out := map[string]int{}
	for _, in := range maps {
		for key, val := range in {
			out[key] = val
		}
	}
	return out
}

func mapMergeStrStr(maps ...map[string]string) map[string]string {
	out := map[string]string{}
	for _, in := range maps {
		for key, val := range in {
			out[key] = val
		}
	}
	return out
}
<commit_msg>Add nil guard<commit_after>package driver

import (
	"errors"
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/hashicorp/nomad/client/allocdir"
	"github.com/hashicorp/nomad/client/config"
	"github.com/hashicorp/nomad/client/driver/env"
	"github.com/hashicorp/nomad/client/fingerprint"
	"github.com/hashicorp/nomad/nomad/structs"

	dstructs "github.com/hashicorp/nomad/client/driver/structs"
	cstructs "github.com/hashicorp/nomad/client/structs"
)

var (
	// BuiltinDrivers contains the built in registered drivers
	// which are available for allocation handling
	BuiltinDrivers = map[string]Factory{
		"docker":   NewDockerDriver,
		"exec":     NewExecDriver,
		"raw_exec": NewRawExecDriver,
		"java":     NewJavaDriver,
		"qemu":     NewQemuDriver,
		"rkt":      NewRktDriver,
	}

	// DriverStatsNotImplemented is the error to be returned if a driver doesn't
	// implement stats.
	DriverStatsNotImplemented = errors.New("stats not implemented for driver")
)

// NewDriver is used to instantiate and return a new driver
// given the name and a logger
func NewDriver(name string, ctx *DriverContext) (Driver, error) {
	// Lookup the factory function
	factory, ok := BuiltinDrivers[name]
	if !ok {
		return nil, fmt.Errorf("unknown driver '%s'", name)
	}

	// Instantiate the driver
	f := factory(ctx)
	return f, nil
}

// Factory is used to instantiate a new Driver
type Factory func(*DriverContext) Driver

// CreatedResources is a map of resources (eg downloaded images) created by a driver
// that must be cleaned up.
type CreatedResources struct {
	Resources map[string][]string
}

func NewCreatedResources() *CreatedResources {
	return &CreatedResources{Resources: make(map[string][]string)}
}

// Add a new resource if it doesn't already exist.
func (r *CreatedResources) Add(k, v string) {
	if r.Resources == nil {
		r.Resources = map[string][]string{k: []string{v}}
		return
	}
	existing, ok := r.Resources[k]
	if !ok {
		// Key doesn't exist, create it
		r.Resources[k] = []string{v}
		return
	}
	for _, item := range existing {
		if item == v {
			// resource exists, return
			return
		}
	}

	// Resource type exists but value did not, append it
	r.Resources[k] = append(existing, v)
	return
}

// Remove a resource. Return true if removed, otherwise false.
//
// Removes the entire key if the needle is the last value in the list.
func (r *CreatedResources) Remove(k, needle string) bool {
	haystack := r.Resources[k]
	for i, item := range haystack {
		if item == needle {
			r.Resources[k] = append(haystack[:i], haystack[i+1:]...)
			if len(r.Resources[k]) == 0 {
				delete(r.Resources, k)
			}
			return true
		}
	}
	return false
}

// Copy returns a new deep copy of CreatedResrouces.
func (r *CreatedResources) Copy() *CreatedResources {
	if r == nil {
		return nil
	}

	newr := CreatedResources{
		Resources: make(map[string][]string, len(r.Resources)),
	}
	for k, v := range r.Resources {
		newv := make([]string, len(v))
		copy(newv, v)
		newr.Resources[k] = newv
	}
	return &newr
}

// Merge another CreatedResources into this one. If the other CreatedResources
// is nil this method is a noop.
func (r *CreatedResources) Merge(o *CreatedResources) {
	if o == nil {
		return
	}

	for k, v := range o.Resources {
		// New key
		if len(r.Resources[k]) == 0 {
			r.Resources[k] = v
			continue
		}

		// Existing key
	OUTER:
		for _, item := range v {
			for _, existing := range r.Resources[k] {
				if item == existing {
					// Found it, move on
					continue OUTER
				}
			}

			// New item, append it
			r.Resources[k] = append(r.Resources[k], item)
		}
	}
}

// Driver is used for execution of tasks. This allows Nomad
// to support many pluggable implementations of task drivers.
// Examples could include LXC, Docker, Qemu, etc.
type Driver interface {
	// Drivers must support the fingerprint interface for detection
	fingerprint.Fingerprint

	// Prestart prepares the task environment and performs expensive
	// intialization steps like downloading images.
	//
	// CreatedResources may be non-nil even when an error occurs.
	Prestart(*ExecContext, *structs.Task) (*CreatedResources, error)

	// Start is used to being task execution
	Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error)

	// Open is used to re-open a handle to a task
	Open(ctx *ExecContext, handleID string) (DriverHandle, error)

	// Cleanup is called to remove resources which were created for a task
	// and no longer needed.
	//
	// If Cleanup returns a recoverable error it may be retried. On retry
	// it will be passed the same CreatedResources, so all successfully
	// cleaned up resources should be removed.
	Cleanup(*ExecContext, *CreatedResources) error

	// Drivers must validate their configuration
	Validate(map[string]interface{}) error

	// Abilities returns the abilities of the driver
	Abilities() DriverAbilities

	// FSIsolation returns the method of filesystem isolation used
	FSIsolation() cstructs.FSIsolation
}

// DriverAbilities marks the abilities the driver has.
type DriverAbilities struct {
	// SendSignals marks the driver as being able to send signals
	SendSignals bool
}

// LogEventFn is a callback which allows Drivers to emit task events.
type LogEventFn func(message string, args ...interface{})

// DriverContext is a means to inject dependencies such as loggers, configs, and
// node attributes into a Driver without having to change the Driver interface
// each time we do it. Used in conjection with Factory, above.
type DriverContext struct {
	taskName string
	config   *config.Config
	logger   *log.Logger
	node     *structs.Node
	taskEnv  *env.TaskEnvironment

	emitEvent LogEventFn
}

// NewEmptyDriverContext returns a DriverContext with all fields set to their
// zero value.
func NewEmptyDriverContext() *DriverContext {
	return &DriverContext{}
}

// NewDriverContext initializes a new DriverContext with the specified fields.
// This enables other packages to create DriverContexts but keeps the fields
// private to the driver. If we want to change this later we can gorename all of
// the fields in DriverContext.
func NewDriverContext(taskName string, config *config.Config, node *structs.Node,
	logger *log.Logger, taskEnv *env.TaskEnvironment, eventEmitter LogEventFn) *DriverContext {
	return &DriverContext{
		taskName:  taskName,
		config:    config,
		node:      node,
		logger:    logger,
		taskEnv:   taskEnv,
		emitEvent: eventEmitter,
	}
}

// DriverHandle is an opaque handle into a driver used for task
// manipulation
type DriverHandle interface {
	// Returns an opaque handle that can be used to re-open the handle
	ID() string

	// WaitCh is used to return a channel used wait for task completion
	WaitCh() chan *dstructs.WaitResult

	// Update is used to update the task if possible and update task related
	// configurations.
	Update(task *structs.Task) error

	// Kill is used to stop the task
	Kill() error

	// Stats returns aggregated stats of the driver
	Stats() (*cstructs.TaskResourceUsage, error)

	// Signal is used to send a signal to the task
	Signal(s os.Signal) error
}

// ExecContext is a task's execution context
type ExecContext struct {
	// TaskDir contains information about the task directory structure.
	TaskDir *allocdir.TaskDir

	// Alloc ID
	AllocID string
}

// NewExecContext is used to create a new execution context
func NewExecContext(td *allocdir.TaskDir, allocID string) *ExecContext {
	return &ExecContext{
		TaskDir: td,
		AllocID: allocID,
	}
}

// GetTaskEnv converts the alloc dir, the node, task and alloc into a
// TaskEnvironment.
func GetTaskEnv(taskDir *allocdir.TaskDir, node *structs.Node,
	task *structs.Task, alloc *structs.Allocation, conf *config.Config,
	vaultToken string) (*env.TaskEnvironment, error) {

	env := env.NewTaskEnvironment(node).
		SetTaskMeta(alloc.Job.CombinedTaskMeta(alloc.TaskGroup, task.Name)).
		SetJobName(alloc.Job.Name).
		SetEnvvars(task.Env).
		SetTaskName(task.Name)

	// Vary paths by filesystem isolation used
	drv, err := NewDriver(task.Driver, NewEmptyDriverContext())
	if err != nil {
		return nil, err
	}
	switch drv.FSIsolation() {
	case cstructs.FSIsolationNone:
		// Use host paths
		env.SetAllocDir(taskDir.SharedAllocDir)
		env.SetTaskLocalDir(taskDir.LocalDir)
		env.SetSecretsDir(taskDir.SecretsDir)
	default:
		// filesystem isolation; use container paths
		env.SetAllocDir(allocdir.SharedAllocContainerPath)
		env.SetTaskLocalDir(allocdir.TaskLocalContainerPath)
		env.SetSecretsDir(allocdir.TaskSecretsContainerPath)
	}

	if task.Resources != nil {
		env.SetMemLimit(task.Resources.MemoryMB).
			SetCpuLimit(task.Resources.CPU).
			SetNetworks(task.Resources.Networks)
	}

	if alloc != nil {
		env.SetAlloc(alloc)
	}

	if task.Vault != nil {
		env.SetVaultToken(vaultToken, task.Vault.Env)
	}

	// Set the host environment variables for non-image based drivers
	if drv.FSIsolation() != cstructs.FSIsolationImage {
		filter := strings.Split(conf.ReadDefault("env.blacklist", config.DefaultEnvBlacklist), ",")
		env.AppendHostEnvvars(filter)
	}

	return env.Build(), nil
}

func mapMergeStrInt(maps ...map[string]int) map[string]int {
	out := map[string]int{}
	for _, in := range maps {
		for key, val := range in {
			out[key] = val
		}
	}
	return out
}

func mapMergeStrStr(maps ...map[string]string) map[string]string {
	out := map[string]string{}
	for _, in := range maps {
		for key, val := range in {
			out[key] = val
		}
	}
	return out
}
<|endoftext|>


<commit_before>/*
   Copyright 2014 GoPivotal (UK) Limited.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package rootfs_test

import (
	"code.google.com/p/gomock/gomock"
	"fmt"
	"github.com/cf-guardian/guardian/gerror"
	"github.com/cf-guardian/guardian/kernel/fileutils"
	"github.com/cf-guardian/guardian/kernel/fileutils/mock_fileutils"
	"github.com/cf-guardian/guardian/kernel/rootfs"
	"github.com/cf-guardian/guardian/kernel/syscall/mock_syscall"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"testing"
)

func TestNonExistentReadWriteBaseDir(t *testing.T) {
	mockCtrl, mockFileUtils, mockSyscallFS := setupMocks(t)
	defer mockCtrl.Finish()

	mockFileUtils.EXPECT().Filemode("/nosuch").Return(os.FileMode(0), gerror.New(fileutils.ErrFileNotFound, "test error"))

	rfs, gerr := rootfs.NewRootFS(mockSyscallFS, mockFileUtils, "/nosuch")
	if rfs != nil || !gerr.EqualTag(rootfs.ErrRwBaseDirMissing) {
		t.Errorf("Incorrect return values (%s, %s)", rfs, gerr)
		return
	}
}

func TestNonDirReadWriteBaseDir(t *testing.T) {
	mockCtrl, mockFileUtils, mockSyscallFS := setupMocks(t)
	defer mockCtrl.Finish()

	tempDir := createTempDir()
	filePath := createFile(tempDir, "testFile")
	mockFileUtils.EXPECT().Filemode(filePath).Return(os.FileMode(0700), nil)

	rfs, gerr := rootfs.NewRootFS(mockSyscallFS, mockFileUtils, filePath)
	if rfs != nil || !gerr.EqualTag(rootfs.ErrRwBaseDirIsFile) {
		t.Errorf("Incorrect return values (%s, %s)", rfs, gerr)
		return
	}
}

func TestReadOnlyReadWriteBaseDir(t *testing.T) {
	mockCtrl, mockFileUtils, mockSyscallFS := setupMocks(t)
	defer mockCtrl.Finish()

	tempDir := createTempDir()
	dirPath := createDirWithMode(tempDir, "test-rootfs", os.FileMode(0400))
	mockFileUtils.EXPECT().Filemode(dirPath).Return(os.ModeDir|os.FileMode(0100), nil)

	rfs, gerr := rootfs.NewRootFS(mockSyscallFS, mockFileUtils, dirPath)
	if rfs != nil || !gerr.EqualTag(rootfs.ErrRwBaseDirNotRw) {
		t.Errorf("Incorrect return values (%s, %s)", rfs, gerr)
		return
	}
}

func TestGenerate(t *testing.T) {
	mockCtrl, mockFileUtils, mockSyscallFS := setupMocks(t)
	defer mockCtrl.Finish()

	tempDir := createTempDir()
	mockFileUtils.EXPECT().Filemode(tempDir).Return(os.ModeDir|os.FileMode(0700), nil)
	rfs, gerr := rootfs.NewRootFS(mockSyscallFS, mockFileUtils, tempDir)
	if gerr != nil {
		t.Errorf("%s", gerr)
		return
	}
	prototypeDir := filepath.Join(tempDir, "test-prototype")

	mockSyscallFS.EXPECT().BindMountReadOnly(prototypeDir, &stringPrefixMatcher{filepath.Join(tempDir, "mnt")})

	dirs := []string{`proc`, `dev`, `etc`, `home`, `sbin`, `var`, `tmp`}
	for _, dir := range dirs {
		mockSyscallFS.EXPECT().BindMountReadWrite(&stringRegexMatcher{filepath.Join(tempDir, "tmp-rootfs-.*", dir)}, &stringRegexMatcher{filepath.Join(tempDir, "mnt-.*", dir)})
	}

	root, gerr := rfs.Generate(prototypeDir)
	if gerr != nil {
		t.Errorf("%s", gerr)
		return
	}

	_ = root
}

type stringPrefixMatcher struct {
	prefix string
}

func (m *stringPrefixMatcher) Matches(x interface{}) bool {
	if x, ok := x.(string); ok {
		return strings.HasPrefix(x, m.prefix)
	} else {
		return false
	}
}

func (m *stringPrefixMatcher) String() string {
	return fmt.Sprintf("is a string with prefix %s", m.prefix)
}

type stringRegexMatcher struct {
	regex string
}

func (m *stringRegexMatcher) Matches(x interface{}) bool {
	if x, ok := x.(string); ok {
		if matched, err := regexp.MatchString(m.regex, x); err == nil {
			return matched
		} else {
			return false
		}
	} else {
		return false
	}
}

func (m *stringRegexMatcher) String() string {
	return fmt.Sprintf("is a string which matches regular expression %s", m.regex)
}

func createTempDir() string {
	tempDir, err := ioutil.TempDir("/tmp", "guardian-test-")
	check(err)
	return tempDir
}

func setupMocks(t *testing.T) (*gomock.Controller, *mock_fileutils.MockFileutils, *mock_syscall.MockSyscall_FS) {
	mockCtrl := gomock.NewController(t)
	mockFileUtils := mock_fileutils.NewMockFileutils(mockCtrl)
	mockSyscallFS := mock_syscall.NewMockSyscall_FS(mockCtrl)
	return mockCtrl, mockFileUtils, mockSyscallFS
}

// TODO: Remove duplication with fileutils_test.
func createFile(td string, fileName string) string {
	return createFileWithMode(td, fileName, os.FileMode(0666))
}

func createFileWithMode(td string, fileName string, mode os.FileMode) string {
	fp := filepath.Join(td, fileName)
	f, err := os.OpenFile(fp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, mode)
	check(err)
	_, err = f.WriteString("test contents")
	check(err)
	check(f.Close())
	return fp
}

func createDir(td string, dirName string) string {
	return createDirWithMode(td, dirName, os.FileMode(0777))
}

func createDirWithMode(td string, dirName string, mode os.FileMode) string {
	fp := filepath.Join(td, dirName)
	err := os.Mkdir(fp, mode)
	check(err)
	return fp
}

func check(err error) {
	if err != nil {
		panic(err)
	}
}
<commit_msg>Check return value of Generate<commit_after>/*
   Copyright 2014 GoPivotal (UK) Limited.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package rootfs_test

import (
	"code.google.com/p/gomock/gomock"
	"fmt"
	"github.com/cf-guardian/guardian/gerror"
	"github.com/cf-guardian/guardian/kernel/fileutils"
	"github.com/cf-guardian/guardian/kernel/fileutils/mock_fileutils"
	"github.com/cf-guardian/guardian/kernel/rootfs"
	"github.com/cf-guardian/guardian/kernel/syscall/mock_syscall"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"testing"
)

func TestNonExistentReadWriteBaseDir(t *testing.T) {
	mockCtrl, mockFileUtils, mockSyscallFS := setupMocks(t)
	defer mockCtrl.Finish()

	mockFileUtils.EXPECT().Filemode("/nosuch").Return(os.FileMode(0), gerror.New(fileutils.ErrFileNotFound, "test error"))

	rfs, gerr := rootfs.NewRootFS(mockSyscallFS, mockFileUtils, "/nosuch")
	if rfs != nil || !gerr.EqualTag(rootfs.ErrRwBaseDirMissing) {
		t.Errorf("Incorrect return values (%s, %s)", rfs, gerr)
		return
	}
}

func TestNonDirReadWriteBaseDir(t *testing.T) {
	mockCtrl, mockFileUtils, mockSyscallFS := setupMocks(t)
	defer mockCtrl.Finish()

	tempDir := createTempDir()
	filePath := createFile(tempDir, "testFile")
	mockFileUtils.EXPECT().Filemode(filePath).Return(os.FileMode(0700), nil)

	rfs, gerr := rootfs.NewRootFS(mockSyscallFS, mockFileUtils, filePath)
	if rfs != nil || !gerr.EqualTag(rootfs.ErrRwBaseDirIsFile) {
		t.Errorf("Incorrect return values (%s, %s)", rfs, gerr)
		return
	}
}

func TestReadOnlyReadWriteBaseDir(t *testing.T) {
	mockCtrl, mockFileUtils, mockSyscallFS := setupMocks(t)
	defer mockCtrl.Finish()

	tempDir := createTempDir()
	dirPath := createDirWithMode(tempDir, "test-rootfs", os.FileMode(0400))
	mockFileUtils.EXPECT().Filemode(dirPath).Return(os.ModeDir|os.FileMode(0100), nil)

	rfs, gerr := rootfs.NewRootFS(mockSyscallFS, mockFileUtils, dirPath)
	if rfs != nil || !gerr.EqualTag(rootfs.ErrRwBaseDirNotRw) {
		t.Errorf("Incorrect return values (%s, %s)", rfs, gerr)
		return
	}
}

func TestGenerate(t *testing.T) {
	mockCtrl, mockFileUtils, mockSyscallFS := setupMocks(t)
	defer mockCtrl.Finish()

	tempDir := createTempDir()
	mockFileUtils.EXPECT().Filemode(tempDir).Return(os.ModeDir|os.FileMode(0700), nil)
	rfs, gerr := rootfs.NewRootFS(mockSyscallFS, mockFileUtils, tempDir)
	if gerr != nil {
		t.Errorf("%s", gerr)
		return
	}
	prototypeDir := filepath.Join(tempDir, "test-prototype")

	mockSyscallFS.EXPECT().BindMountReadOnly(prototypeDir, &stringPrefixMatcher{filepath.Join(tempDir, "mnt")})

	dirs := []string{`proc`, `dev`, `etc`, `home`, `sbin`, `var`, `tmp`}
	for _, dir := range dirs {
		mockSyscallFS.EXPECT().BindMountReadWrite(&stringRegexMatcher{filepath.Join(tempDir, "tmp-rootfs-.*", dir)}, &stringRegexMatcher{filepath.Join(tempDir, "mnt-.*", dir)})
	}

	root, gerr := rfs.Generate(prototypeDir)
	if gerr != nil {
		t.Errorf("%s", gerr)
		return
	}

	rootPrefix := filepath.Join(tempDir, "mnt-")
	if !strings.HasPrefix(root, rootPrefix) {
		t.Errorf("root was %s, but expected it to have prefix %s", root, rootPrefix)
		return
	}
}

type stringPrefixMatcher struct {
	prefix string
}

func (m *stringPrefixMatcher) Matches(x interface{}) bool {
	if x, ok := x.(string); ok {
		return strings.HasPrefix(x, m.prefix)
	} else {
		return false
	}
}

func (m *stringPrefixMatcher) String() string {
	return fmt.Sprintf("is a string with prefix %s", m.prefix)
}

type stringRegexMatcher struct {
	regex string
}

func (m *stringRegexMatcher) Matches(x interface{}) bool {
	if x, ok := x.(string); ok {
		if matched, err := regexp.MatchString(m.regex, x); err == nil {
			return matched
		} else {
			return false
		}
	} else {
		return false
	}
}

func (m *stringRegexMatcher) String() string {
	return fmt.Sprintf("is a string which matches regular expression %s", m.regex)
}

func createTempDir() string {
	tempDir, err := ioutil.TempDir("/tmp", "guardian-test-")
	check(err)
	return tempDir
}

func setupMocks(t *testing.T) (*gomock.Controller, *mock_fileutils.MockFileutils, *mock_syscall.MockSyscall_FS) {
	mockCtrl := gomock.NewController(t)
	mockFileUtils := mock_fileutils.NewMockFileutils(mockCtrl)
	mockSyscallFS := mock_syscall.NewMockSyscall_FS(mockCtrl)
	return mockCtrl, mockFileUtils, mockSyscallFS
}

// TODO: Remove duplication with fileutils_test.
func createFile(td string, fileName string) string {
	return createFileWithMode(td, fileName, os.FileMode(0666))
}

func createFileWithMode(td string, fileName string, mode os.FileMode) string {
	fp := filepath.Join(td, fileName)
	f, err := os.OpenFile(fp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, mode)
	check(err)
	_, err = f.WriteString("test contents")
	check(err)
	check(f.Close())
	return fp
}

func createDir(td string, dirName string) string {
	return createDirWithMode(td, dirName, os.FileMode(0777))
}

func createDirWithMode(td string, dirName string, mode os.FileMode) string {
	fp := filepath.Join(td, dirName)
	err := os.Mkdir(fp, mode)
	check(err)
	return fp
}

func check(err error) {
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package fix

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/mitchellh/packer/packer"
	"os"
	"strings"
)

type Command byte

func (Command) Help() string {
	return strings.TrimSpace(helpString)
}

func (c Command) Run(env packer.Environment, args []string) int {
	cmdFlags := flag.NewFlagSet("fix", flag.ContinueOnError)
	cmdFlags.Usage = func() { env.Ui().Say(c.Help()) }
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	args = cmdFlags.Args()
	if len(args) != 1 {
		cmdFlags.Usage()
		return 1
	}

	// Read the file for decoding
	tplF, err := os.Open(args[0])
	if err != nil {
		env.Ui().Error(fmt.Sprintf("Error opening template: %s", err))
		return 1
	}
	defer tplF.Close()

	// Decode the JSON into a generic map structure
	var templateData map[string]interface{}
	decoder := json.NewDecoder(tplF)
	if err := decoder.Decode(&templateData); err != nil {
		env.Ui().Error(fmt.Sprintf("Error parsing template: %s", err))
		return 1
	}

	// Close the file since we're done with that
	tplF.Close()

	// Run the template through the various fixers
	fixers := []Fixer{Fixers["iso-md5"]}
	input := templateData
	for _, fixer := range fixers {
		var err error
		input, err = fixer.Fix(input)
		if err != nil {
			env.Ui().Error(fmt.Sprintf("Error fixing: %s", err))
			return 1
		}
	}

	var output bytes.Buffer
	encoder := json.NewEncoder(&output)
	if err := encoder.Encode(input); err != nil {
		env.Ui().Error(fmt.Sprintf("Error encoding: %s", err))
		return 1
	}

	var indented bytes.Buffer
	if err := json.Indent(&indented, output.Bytes(), "", "  "); err != nil {
		env.Ui().Error(fmt.Sprintf("Error encoding: %s", err))
		return 1
	}

	result := indented.String()
	result = strings.Replace(result, `\u003c`, "<", -1)
	result = strings.Replace(result, `\u003e`, ">", -1)
	env.Ui().Say(result)
	return 0
}

func (c Command) Synopsis() string {
	return "fixes templates from old versions of packer"
}
<commit_msg>command/fix: use strings instead of Fixer for list<commit_after>package fix

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/mitchellh/packer/packer"
	"os"
	"strings"
)

type Command byte

func (Command) Help() string {
	return strings.TrimSpace(helpString)
}

func (c Command) Run(env packer.Environment, args []string) int {
	cmdFlags := flag.NewFlagSet("fix", flag.ContinueOnError)
	cmdFlags.Usage = func() { env.Ui().Say(c.Help()) }
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	args = cmdFlags.Args()
	if len(args) != 1 {
		cmdFlags.Usage()
		return 1
	}

	// Read the file for decoding
	tplF, err := os.Open(args[0])
	if err != nil {
		env.Ui().Error(fmt.Sprintf("Error opening template: %s", err))
		return 1
	}
	defer tplF.Close()

	// Decode the JSON into a generic map structure
	var templateData map[string]interface{}
	decoder := json.NewDecoder(tplF)
	if err := decoder.Decode(&templateData); err != nil {
		env.Ui().Error(fmt.Sprintf("Error parsing template: %s", err))
		return 1
	}

	// Close the file since we're done with that
	tplF.Close()

	// Run the template through the various fixers
	fixers := []string{
		"iso-md5",
	}

	input := templateData
	for _, name := range fixers {
		var err error
		fixer, ok := Fixers[name]
		if !ok {
			panic("fixer not found: " + name)
		}

		input, err = fixer.Fix(input)
		if err != nil {
			env.Ui().Error(fmt.Sprintf("Error fixing: %s", err))
			return 1
		}
	}

	var output bytes.Buffer
	encoder := json.NewEncoder(&output)
	if err := encoder.Encode(input); err != nil {
		env.Ui().Error(fmt.Sprintf("Error encoding: %s", err))
		return 1
	}

	var indented bytes.Buffer
	if err := json.Indent(&indented, output.Bytes(), "", "  "); err != nil {
		env.Ui().Error(fmt.Sprintf("Error encoding: %s", err))
		return 1
	}

	result := indented.String()
	result = strings.Replace(result, `\u003c`, "<", -1)
	result = strings.Replace(result, `\u003e`, ">", -1)
	env.Ui().Say(result)
	return 0
}

func (c Command) Synopsis() string {
	return "fixes templates from old versions of packer"
}
<|endoftext|>


<commit_before>package worker

import (
	"bufio"
	"encoding/json"
	"fmt"
	"github.com/discoproject/goworker/jobutil"
	"io"
	"log"
	"os"
)

const (
	DEBUG = true
)

func Check(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func debug(prefix string, msg interface{}) {
	if DEBUG {
		file, err := os.OpenFile("/tmp/debug", os.O_RDWR|os.O_APPEND|os.O_CREATE, 0644)
		Check(err)
		defer file.Close()
		fmt.Fprintf(file, "%s: %v\n", prefix, msg)
	}
}

func send(key string, payload interface{}) {
	enc, err := json.Marshal(payload)
	if err != nil {
		panic("could not encode")
	}
	str := fmt.Sprintf("%s %d %s\n", key, len(enc), enc)
	fmt.Printf(str)
	debug("send", str)
}

func recv() (string, int, []byte) {
	var size int
	var status string
	fmt.Scanf("%s %d", &status, &size)
	reader := bufio.NewReader(os.Stdin)
	input := make([]byte, size)
	reader.Read(input)
	debug("recv", fmt.Sprintf("%d ", size)+string(input))
	return status, size, input
}

func send_worker() {
	type WorkerMsg struct {
		Pid     int    `json:"pid"`
		Version string `json:"version"`
	}
	wm := WorkerMsg{os.Getpid(), "1.1"}
	send("WORKER", wm)

	_, _, response := recv()
	if string(response) != "\"ok\"" {
		panic(response)
	}
}

func request_task() *Task {
	task := new(Task)
	send("TASK", "")
	_, _, line := recv()
	json.Unmarshal(line, &task)
	debug("info", task)
	return task
}

func request_input() *Input {
	send("INPUT", "")
	_, _, line := recv()
	var mj []interface{}
	json.Unmarshal(line, &mj)

	flag := mj[0].(string)
	if flag != "done" {
		panic(flag)
	}
	_inputs := mj[1].([]interface{})
	inputs := _inputs[0].([]interface{})

	id := inputs[0].(float64)
	status := inputs[1].(string)

	label := -1
	switch t := inputs[2].(type) {
	case string:
		label = -1
	case float64:
		label = int(t)
	}
	_replicas := inputs[3].([]interface{})

	replicas := _replicas[0].([]interface{})

	//FIXME avoid conversion to float when reading the item
	replica_id := replicas[0].(float64)
	replica_location := replicas[1].(string)

	debug("info", fmt.Sprintln(id, status, label, replica_id, replica_location))

	input := new(Input)
	input.id = int(id)
	input.status = status
	input.label = label
	input.replica_id = int(replica_id)
	input.replica_location = replica_location
	return input
}

func send_output(output *Output) {
	v := make([]interface{}, 3)
	v[0] = output.label
	v[1] = output.output_location //"http://example.com"
	v[2] = output.output_size

	send("OUTPUT", v)
	_, _, line := recv()
	debug("info", string(line))
}

func request_done() {
	send("DONE", "")
	_, _, line := recv()
	debug("info", string(line))
}

type Task struct {
	Host       string
	Master     string
	Jobname    string
	Taskid     int
	Stage      string
	Grouping   string
	Group      string
	Disco_port int
	Put_port   int
	Disco_data string
	Ddfs_data  string
	Jobfile    string
}

type Input struct {
	id               int
	status           string
	label            int
	replica_id       int
	replica_location string
}

type Output struct {
	label           int
	output_location string
	output_size     int64
}

type Worker struct {
	task   *Task
	input  *Input
	output *Output
}

type Process func(io.Reader, io.Writer)

func (w *Worker) runStage(output_name string, process Process) {
	output, err := os.Create(output_name)
	Check(err)
	readCloser := jobutil.AddressReader(w.input.replica_location,
		jobutil.Setting("DISCO_DATA"))
	process(readCloser, output)
	readCloser.Close()
	output.Close()
	w.output.output_location =
		"disco://" + jobutil.Setting("HOST") + "/disco/" + output_name[len(w.task.Disco_data)+1:]
	output, err = os.Open(output_name)
	Check(err)
	fileinfo, err := output.Stat()
	Check(err)
	w.output.output_size = fileinfo.Size()
}

func Run(Map Process, Reduce Process) {
	var w Worker
	send_worker()
	w.task = request_task()

	jobutil.SetKeyValue("HOST", w.task.Host)
	master, port := jobutil.HostAndPort(w.task.Master)
	jobutil.SetKeyValue("DISCO_MASTER", master)
	if port != fmt.Sprintf("%d", w.task.Disco_port) {
		panic("port mismatch: " + port)
	}
	jobutil.SetKeyValue("DISCO_PORT", port)
	jobutil.SetKeyValue("PUT_PORT", string(w.task.Put_port))
	jobutil.SetKeyValue("DISCO_DATA", w.task.Disco_data)
	jobutil.SetKeyValue("DDFS_DATA", w.task.Ddfs_data)

	w.input = request_input()

	pwd, err := os.Getwd()
	Check(err)

	w.output = new(Output)
	if w.task.Stage == "map" {
		w.runStage(pwd+"/map_out", Map)
	} else if w.task.Stage == "map_shuffle" {
		w.output.output_location = w.input.replica_location
	} else {
		w.runStage(pwd+"/reduce_out", Reduce)
	}

	send_output(w.output)
	request_done()
}
<commit_msg>worker: use random filenames for outputs of the stages.<commit_after>package worker

import (
	"bufio"
	"encoding/json"
	"fmt"
	"github.com/discoproject/goworker/jobutil"
	"io"
	"io/ioutil"
	"log"
	"os"
)

const (
	DEBUG = true
)

func Check(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func debug(prefix string, msg interface{}) {
	if DEBUG {
		file, err := os.OpenFile("/tmp/debug", os.O_RDWR|os.O_APPEND|os.O_CREATE, 0644)
		Check(err)
		defer file.Close()
		fmt.Fprintf(file, "%s: %v\n", prefix, msg)
	}
}

func send(key string, payload interface{}) {
	enc, err := json.Marshal(payload)
	if err != nil {
		panic("could not encode")
	}
	str := fmt.Sprintf("%s %d %s\n", key, len(enc), enc)
	fmt.Printf(str)
	debug("send", str)
}

func recv() (string, int, []byte) {
	var size int
	var status string
	fmt.Scanf("%s %d", &status, &size)
	reader := bufio.NewReader(os.Stdin)
	input := make([]byte, size)
	reader.Read(input)
	debug("recv", fmt.Sprintf("%d ", size)+string(input))
	return status, size, input
}

func send_worker() {
	type WorkerMsg struct {
		Pid     int    `json:"pid"`
		Version string `json:"version"`
	}
	wm := WorkerMsg{os.Getpid(), "1.1"}
	send("WORKER", wm)

	_, _, response := recv()
	if string(response) != "\"ok\"" {
		panic(response)
	}
}

func request_task() *Task {
	task := new(Task)
	send("TASK", "")
	_, _, line := recv()
	json.Unmarshal(line, &task)
	debug("info", task)
	return task
}

func request_input() *Input {
	send("INPUT", "")
	_, _, line := recv()
	var mj []interface{}
	json.Unmarshal(line, &mj)

	flag := mj[0].(string)
	if flag != "done" {
		panic(flag)
	}
	_inputs := mj[1].([]interface{})
	inputs := _inputs[0].([]interface{})

	id := inputs[0].(float64)
	status := inputs[1].(string)

	label := -1
	switch t := inputs[2].(type) {
	case string:
		label = -1
	case float64:
		label = int(t)
	}
	_replicas := inputs[3].([]interface{})

	replicas := _replicas[0].([]interface{})

	//FIXME avoid conversion to float when reading the item
	replica_id := replicas[0].(float64)
	replica_location := replicas[1].(string)

	debug("info", fmt.Sprintln(id, status, label, replica_id, replica_location))

	input := new(Input)
	input.id = int(id)
	input.status = status
	input.label = label
	input.replica_id = int(replica_id)
	input.replica_location = replica_location
	return input
}

func send_output(output *Output) {
	v := make([]interface{}, 3)
	v[0] = output.label
	v[1] = output.output_location //"http://example.com"
	v[2] = output.output_size

	send("OUTPUT", v)
	_, _, line := recv()
	debug("info", string(line))
}

func request_done() {
	send("DONE", "")
	_, _, line := recv()
	debug("info", string(line))
}

type Task struct {
	Host       string
	Master     string
	Jobname    string
	Taskid     int
	Stage      string
	Grouping   string
	Group      string
	Disco_port int
	Put_port   int
	Disco_data string
	Ddfs_data  string
	Jobfile    string
}

type Input struct {
	id               int
	status           string
	label            int
	replica_id       int
	replica_location string
}

type Output struct {
	label           int
	output_location string
	output_size     int64
}

type Worker struct {
	task   *Task
	input  *Input
	output *Output
}

type Process func(io.Reader, io.Writer)

func (w *Worker) runStage(pwd string, prefix string, process Process) {
	output, err := ioutil.TempFile(pwd, prefix)
	output_name := output.Name()
	Check(err)
	readCloser := jobutil.AddressReader(w.input.replica_location,
		jobutil.Setting("DISCO_DATA"))
	process(readCloser, output)
	readCloser.Close()
	output.Close()
	w.output.output_location =
		"disco://" + jobutil.Setting("HOST") + "/disco/" + output_name[len(w.task.Disco_data)+1:]
	output, err = os.Open(output_name)
	Check(err)
	fileinfo, err := output.Stat()
	Check(err)
	w.output.output_size = fileinfo.Size()
}

func Run(Map Process, Reduce Process) {
	var w Worker
	send_worker()
	w.task = request_task()

	jobutil.SetKeyValue("HOST", w.task.Host)
	master, port := jobutil.HostAndPort(w.task.Master)
	jobutil.SetKeyValue("DISCO_MASTER", master)
	if port != fmt.Sprintf("%d", w.task.Disco_port) {
		panic("port mismatch: " + port)
	}
	jobutil.SetKeyValue("DISCO_PORT", port)
	jobutil.SetKeyValue("PUT_PORT", string(w.task.Put_port))
	jobutil.SetKeyValue("DISCO_DATA", w.task.Disco_data)
	jobutil.SetKeyValue("DDFS_DATA", w.task.Ddfs_data)

	w.input = request_input()

	pwd, err := os.Getwd()
	Check(err)

	w.output = new(Output)
	if w.task.Stage == "map" {
		w.runStage(pwd, "map_out_", Map)
	} else if w.task.Stage == "map_shuffle" {
		w.output.output_location = w.input.replica_location
	} else {
		w.runStage(pwd, "reduce_out_", Reduce)
	}

	send_output(w.output)
	request_done()
}
<|endoftext|>


<commit_before>// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package worker implements the logic to extract useful informations from a
// container layer and store it in the database.
package worker

import (
	"github.com/coreos/pkg/capnslog"

	"github.com/coreos/clair/database"
	"github.com/coreos/clair/utils"
	cerrors "github.com/coreos/clair/utils/errors"
	"github.com/coreos/clair/worker/detectors"
)

const (
	// Version (integer) represents the worker version.
	// Increased each time the engine changes.
	Version = 2

	// maxFileSize is the maximum size of a single file we should extract.
	maxFileSize = 200 * 1024 * 1024 // 200 MiB
)

var (
	log = capnslog.NewPackageLogger("github.com/coreos/clair", "worker")

	// ErrUnsupported is the error that should be raised when an OS or package
	// manager is not supported.
	ErrUnsupported = cerrors.NewBadRequestError("worker: OS and/or package manager are not supported")

	// ErrParentUnknown is the error that should be raised when a parent layer
	// has yet to be processed for the current layer.
	ErrParentUnknown = cerrors.NewBadRequestError("worker: parent layer is unknown, it must be processed first")
)

// Process detects the Namespace of a layer, the features it adds/removes, and
// then stores everything in the database.
// TODO(Quentin-M): We could have a goroutine that looks for layers that have been analyzed with an
// older engine version and that processes them.
func Process(datastore database.Datastore, imageFormat, name, parentName, path string, headers map[string]string) error {
	// Verify parameters.
	if name == "" {
		return cerrors.NewBadRequestError("could not process a layer which does not have a name")
	}

	if path == "" {
		return cerrors.NewBadRequestError("could not process a layer which does not have a path")
	}

	if imageFormat == "" {
		return cerrors.NewBadRequestError("could not process a layer which does not have a format")
	}

	log.Debugf("layer %s: processing (Location: %s, Engine version: %d, Parent: %s, Format: %s)",
		name, utils.CleanURL(path), Version, parentName, imageFormat)

	// Check to see if the layer is already in the database.
	layer, err := datastore.FindLayer(name, false, false)
	if err != nil && err != cerrors.ErrNotFound {
		return err
	}

	if err == cerrors.ErrNotFound {
		// New layer case.
		layer = database.Layer{Name: name, EngineVersion: Version}

		// Retrieve the parent if it has one.
		// We need to get it with its Features in order to diff them.
		if parentName != "" {
			parent, err := datastore.FindLayer(parentName, true, false)
			if err != nil && err != cerrors.ErrNotFound {
				return err
			}
			if err == cerrors.ErrNotFound {
				log.Warningf("layer %s: the parent layer (%s) is unknown. it must be processed first", name,
					parentName)
				return ErrParentUnknown
			}
			layer.Parent = &parent
		}
	} else {
		// The layer is already in the database, check if we need to update it.
		if layer.EngineVersion >= Version {
			log.Debugf(`layer %s: layer content has already been processed in the past with engine %d.
        Current engine is %d. skipping analysis`, name, layer.EngineVersion, Version)
			return nil
		}

		log.Debugf(`layer %s: layer content has been analyzed in the past with engine %d. Current
      engine is %d. analyzing again`, name, layer.EngineVersion, Version)
	}

	// Analyze the content.
	layer.Namespace, layer.Features, err = detectContent(imageFormat, name, path, headers, layer.Parent)
	if err != nil {
		return err
	}

	return datastore.InsertLayer(layer)
}

// detectContent downloads a layer's archive and extracts its Namespace and Features.
func detectContent(imageFormat, name, path string, headers map[string]string, parent *database.Layer) (namespace *database.Namespace, featureVersions []database.FeatureVersion, err error) {
	data, err := detectors.DetectData(imageFormat, path, headers, append(detectors.GetRequiredFilesFeatures(), detectors.GetRequiredFilesNamespace()...), maxFileSize)
	if err != nil {
		log.Errorf("layer %s: failed to extract data from %s: %s", name, utils.CleanURL(path), err)
		return
	}

	// Detect namespace.
	namespace = detectNamespace(name, data, parent)

	// Detect features.
	featureVersions, err = detectFeatureVersions(name, data, namespace, parent)
	if err != nil {
		return
	}
	if len(featureVersions) > 0 {
		log.Debugf("layer %s: detected %d features", name, len(featureVersions))
	}

	return
}

func detectNamespace(name string, data map[string][]byte, parent *database.Layer) (namespace *database.Namespace) {
	// Use registered detectors to get the Namespace.
	namespace = detectors.DetectNamespace(data)
	if namespace != nil {
		log.Debugf("layer %s: detected namespace %q", name, namespace.Name)
		return
	}

	// Use the parent's Namespace.
	if parent != nil {
		namespace = parent.Namespace
		if namespace != nil {
			log.Debugf("layer %s: detected namespace %q (from parent)", name, namespace.Name)
			return
		}
	}

	return
}

func detectFeatureVersions(name string, data map[string][]byte, namespace *database.Namespace, parent *database.Layer) (features []database.FeatureVersion, err error) {
	// TODO(Quentin-M): We need to pass the parent image to DetectFeatures because it's possible that
	// some detectors would need it in order to produce the entire feature list (if they can only
	// detect a diff). Also, we should probably pass the detected namespace so detectors could
	// make their own decision.
	features, err = detectors.DetectFeatures(data)
	if err != nil {
		return
	}

	// If there are no FeatureVersions, use parent's FeatureVersions if possible.
	// TODO(Quentin-M): We eventually want to give the choice to each detectors to use none/some of
	// their parent's FeatureVersions. It would be useful for detectors that can't find their entire
	// result using one Layer.
	if len(features) == 0 && parent != nil {
		features = parent.Features
		return
	}

	// Build a map of the namespaces for each FeatureVersion in our parent layer.
	parentFeatureNamespaces := make(map[string]database.Namespace)
	if parent != nil {
		for _, parentFeature := range parent.Features {
			parentFeatureNamespaces[parentFeature.Feature.Name+":"+parentFeature.Version.String()] = parentFeature.Feature.Namespace
		}
	}

	// Ensure that each FeatureVersion has an associated Namespace.
	for i, feature := range features {
		if feature.Feature.Namespace.Name != "" {
			// There is a Namespace associated.
			continue
		}

		if parentFeatureNamespace, ok := parentFeatureNamespaces[feature.Feature.Name+":"+feature.Version.String()]; ok {
			// The FeatureVersion is present in the parent layer; associate with their Namespace.
			features[i].Feature.Namespace = parentFeatureNamespace
			continue
		}

		if namespace != nil {
			// The Namespace has been detected in this layer; associate it.
			features[i].Feature.Namespace = *namespace
			continue
		}

		log.Warningf("layer %s: Layer's namespace is unknown but non-namespaced features have been detected", name)
		err = ErrUnsupported
		return
	}

	return
}
<commit_msg>worker: bump engine version<commit_after>// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package worker implements the logic to extract useful informations from a
// container layer and store it in the database.
package worker

import (
	"github.com/coreos/pkg/capnslog"

	"github.com/coreos/clair/database"
	"github.com/coreos/clair/utils"
	cerrors "github.com/coreos/clair/utils/errors"
	"github.com/coreos/clair/worker/detectors"
)

const (
	// Version (integer) represents the worker version.
	// Increased each time the engine changes.
	Version = 3

	// maxFileSize is the maximum size of a single file we should extract.
	maxFileSize = 200 * 1024 * 1024 // 200 MiB
)

var (
	log = capnslog.NewPackageLogger("github.com/coreos/clair", "worker")

	// ErrUnsupported is the error that should be raised when an OS or package
	// manager is not supported.
	ErrUnsupported = cerrors.NewBadRequestError("worker: OS and/or package manager are not supported")

	// ErrParentUnknown is the error that should be raised when a parent layer
	// has yet to be processed for the current layer.
	ErrParentUnknown = cerrors.NewBadRequestError("worker: parent layer is unknown, it must be processed first")
)

// Process detects the Namespace of a layer, the features it adds/removes, and
// then stores everything in the database.
// TODO(Quentin-M): We could have a goroutine that looks for layers that have been analyzed with an
// older engine version and that processes them.
func Process(datastore database.Datastore, imageFormat, name, parentName, path string, headers map[string]string) error {
	// Verify parameters.
	if name == "" {
		return cerrors.NewBadRequestError("could not process a layer which does not have a name")
	}

	if path == "" {
		return cerrors.NewBadRequestError("could not process a layer which does not have a path")
	}

	if imageFormat == "" {
		return cerrors.NewBadRequestError("could not process a layer which does not have a format")
	}

	log.Debugf("layer %s: processing (Location: %s, Engine version: %d, Parent: %s, Format: %s)",
		name, utils.CleanURL(path), Version, parentName, imageFormat)

	// Check to see if the layer is already in the database.
	layer, err := datastore.FindLayer(name, false, false)
	if err != nil && err != cerrors.ErrNotFound {
		return err
	}

	if err == cerrors.ErrNotFound {
		// New layer case.
		layer = database.Layer{Name: name, EngineVersion: Version}

		// Retrieve the parent if it has one.
		// We need to get it with its Features in order to diff them.
		if parentName != "" {
			parent, err := datastore.FindLayer(parentName, true, false)
			if err != nil && err != cerrors.ErrNotFound {
				return err
			}
			if err == cerrors.ErrNotFound {
				log.Warningf("layer %s: the parent layer (%s) is unknown. it must be processed first", name,
					parentName)
				return ErrParentUnknown
			}
			layer.Parent = &parent
		}
	} else {
		// The layer is already in the database, check if we need to update it.
		if layer.EngineVersion >= Version {
			log.Debugf(`layer %s: layer content has already been processed in the past with engine %d.
        Current engine is %d. skipping analysis`, name, layer.EngineVersion, Version)
			return nil
		}

		log.Debugf(`layer %s: layer content has been analyzed in the past with engine %d. Current
      engine is %d. analyzing again`, name, layer.EngineVersion, Version)
	}

	// Analyze the content.
	layer.Namespace, layer.Features, err = detectContent(imageFormat, name, path, headers, layer.Parent)
	if err != nil {
		return err
	}

	return datastore.InsertLayer(layer)
}

// detectContent downloads a layer's archive and extracts its Namespace and Features.
func detectContent(imageFormat, name, path string, headers map[string]string, parent *database.Layer) (namespace *database.Namespace, featureVersions []database.FeatureVersion, err error) {
	data, err := detectors.DetectData(imageFormat, path, headers, append(detectors.GetRequiredFilesFeatures(), detectors.GetRequiredFilesNamespace()...), maxFileSize)
	if err != nil {
		log.Errorf("layer %s: failed to extract data from %s: %s", name, utils.CleanURL(path), err)
		return
	}

	// Detect namespace.
	namespace = detectNamespace(name, data, parent)

	// Detect features.
	featureVersions, err = detectFeatureVersions(name, data, namespace, parent)
	if err != nil {
		return
	}
	if len(featureVersions) > 0 {
		log.Debugf("layer %s: detected %d features", name, len(featureVersions))
	}

	return
}

func detectNamespace(name string, data map[string][]byte, parent *database.Layer) (namespace *database.Namespace) {
	// Use registered detectors to get the Namespace.
	namespace = detectors.DetectNamespace(data)
	if namespace != nil {
		log.Debugf("layer %s: detected namespace %q", name, namespace.Name)
		return
	}

	// Use the parent's Namespace.
	if parent != nil {
		namespace = parent.Namespace
		if namespace != nil {
			log.Debugf("layer %s: detected namespace %q (from parent)", name, namespace.Name)
			return
		}
	}

	return
}

func detectFeatureVersions(name string, data map[string][]byte, namespace *database.Namespace, parent *database.Layer) (features []database.FeatureVersion, err error) {
	// TODO(Quentin-M): We need to pass the parent image to DetectFeatures because it's possible that
	// some detectors would need it in order to produce the entire feature list (if they can only
	// detect a diff). Also, we should probably pass the detected namespace so detectors could
	// make their own decision.
	features, err = detectors.DetectFeatures(data)
	if err != nil {
		return
	}

	// If there are no FeatureVersions, use parent's FeatureVersions if possible.
	// TODO(Quentin-M): We eventually want to give the choice to each detectors to use none/some of
	// their parent's FeatureVersions. It would be useful for detectors that can't find their entire
	// result using one Layer.
	if len(features) == 0 && parent != nil {
		features = parent.Features
		return
	}

	// Build a map of the namespaces for each FeatureVersion in our parent layer.
	parentFeatureNamespaces := make(map[string]database.Namespace)
	if parent != nil {
		for _, parentFeature := range parent.Features {
			parentFeatureNamespaces[parentFeature.Feature.Name+":"+parentFeature.Version.String()] = parentFeature.Feature.Namespace
		}
	}

	// Ensure that each FeatureVersion has an associated Namespace.
	for i, feature := range features {
		if feature.Feature.Namespace.Name != "" {
			// There is a Namespace associated.
			continue
		}

		if parentFeatureNamespace, ok := parentFeatureNamespaces[feature.Feature.Name+":"+feature.Version.String()]; ok {
			// The FeatureVersion is present in the parent layer; associate with their Namespace.
			features[i].Feature.Namespace = parentFeatureNamespace
			continue
		}

		if namespace != nil {
			// The Namespace has been detected in this layer; associate it.
			features[i].Feature.Namespace = *namespace
			continue
		}

		log.Warningf("layer %s: Layer's namespace is unknown but non-namespaced features have been detected", name)
		err = ErrUnsupported
		return
	}

	return
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"github.com/centurylinkcloud/clc-go-cli/base"
	"github.com/centurylinkcloud/clc-go-cli/models"
	"github.com/centurylinkcloud/clc-go-cli/models/datacenter"
	"github.com/centurylinkcloud/clc-go-cli/models/group"
)

const (
	BaseURL = "https://api.ctl.io"
)

type GroupList struct {
	CommandBase
}

func NewGroupList(info CommandExcInfo) *GroupList {
	g := GroupList{}
	g.ExcInfo = info
	return &g
}

func (g *GroupList) Execute(cn base.Connection) error {
	var err error

	g.Output, err = GetGroups(cn)
	if err != nil {
		return err
	}
	return nil
}

func GetGroups(cn base.Connection) ([]group.Entity, error) {
	var err error
	var groups []group.Entity

	datacenters := []datacenter.GetRes{}
	dcURL := fmt.Sprintf("%s/v2/datacenters/{accountAlias}", BaseURL)
	err = cn.ExecuteRequest("GET", dcURL, nil, &datacenters)
	if err != nil {
		return nil, err
	}

	for _, ref := range datacenters {
		// Get detailed DC info.
		d := datacenter.GetRes{}
		dcURL = fmt.Sprintf("%s/%s?groupLinks=true", BaseURL, GetLink(ref.Links, "self"))
		err = cn.ExecuteRequest("GET", dcURL, nil, &d)
		if err != nil {
			return nil, err
		}
		// Get the root group of the given DC.
		g := group.Entity{}
		gURL := fmt.Sprintf("%s/%s", BaseURL, GetLink(d.Links, "group"))
		err = cn.ExecuteRequest("GET", gURL, nil, &g)
		if err != nil {
			return nil, err
		}
		groups = append(groups, g)
	}
	return groups, nil
}

func GetLink(links []models.LinkEntity, resource string) string {
	for _, link := range links {
		if link.Rel == resource {
			return link.Href
		}
	}
	panic(fmt.Sprintf("No %s link found", resource))
}
<commit_msg>Fix the URL joints.<commit_after>package commands

import (
	"fmt"
	"github.com/centurylinkcloud/clc-go-cli/base"
	"github.com/centurylinkcloud/clc-go-cli/models"
	"github.com/centurylinkcloud/clc-go-cli/models/datacenter"
	"github.com/centurylinkcloud/clc-go-cli/models/group"
)

const (
	BaseURL = "https://api.ctl.io"
)

type GroupList struct {
	CommandBase
}

func NewGroupList(info CommandExcInfo) *GroupList {
	g := GroupList{}
	g.ExcInfo = info
	return &g
}

func (g *GroupList) Execute(cn base.Connection) error {
	var err error

	g.Output, err = GetGroups(cn)
	if err != nil {
		return err
	}
	return nil
}

func GetGroups(cn base.Connection) ([]group.Entity, error) {
	var err error
	var groups []group.Entity

	datacenters := []datacenter.GetRes{}
	dcURL := fmt.Sprintf("%s/v2/datacenters/{accountAlias}", BaseURL)
	err = cn.ExecuteRequest("GET", dcURL, nil, &datacenters)
	if err != nil {
		return nil, err
	}

	for _, ref := range datacenters {
		// Get detailed DC info.
		d := datacenter.GetRes{}
		dcURL = fmt.Sprintf("%s%s?groupLinks=true", BaseURL, GetLink(ref.Links, "self"))
		err = cn.ExecuteRequest("GET", dcURL, nil, &d)
		if err != nil {
			return nil, err
		}
		// Get the root group of the given DC.
		g := group.Entity{}
		gURL := fmt.Sprintf("%s%s", BaseURL, GetLink(d.Links, "group"))
		err = cn.ExecuteRequest("GET", gURL, nil, &g)
		if err != nil {
			return nil, err
		}
		groups = append(groups, g)
	}
	return groups, nil
}

func GetLink(links []models.LinkEntity, resource string) string {
	for _, link := range links {
		if link.Rel == resource {
			return link.Href
		}
	}
	panic(fmt.Sprintf("No %s link found", resource))
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/json"
	"fmt"
	_ "github.com/bmizerany/pq"
	"io/ioutil"
	"log"
	"net/http"
	"time"
)

const OPEN311_API_URI = "http://311api.cityofchicago.org/open311/v2/requests.json?extensions=true&page_size=500"

type Open311Request struct {
	Lat, Long                                                                                float64
	Ward, Police_district                                                                                int
	Service_request_id, Status, Service_name, Service_code, Agency_responsible, Address, Channel, Media_url string
	Requested_datetime, Updated_datetime                                                                    string // FIXME: should these be proper time objects?
	Extended_attributes                                                                                map[string]interface{}
}

func main() {
	// open database
	db, err := sql.Open("postgres", "dbname=cwfy sslmode=disable")
	if err != nil {
		log.Fatal("Cannot open database connection", err)
	}
	defer db.Close()

	// load requests from open311
	requests := fetchRequests()

	for _, request := range requests {
		// for each request, either create or update the
		// corresponding record in the database.

		if request.Service_request_id == "" {
			log.Printf("Ignoring a request type %s because there is no SR number assigned", request.Service_name)
			continue
		}

		insert_stmt, err := db.Prepare("INSERT INTO service_requests(service_request_id," +
			"status, service_name, service_code, agency_responsible, " +
			"address, requested_datetime, updated_datetime, lat, long," +
			"ward, police_district, media_url, channel, duplicate, parent_service_request_id) " +
			"SELECT $1::varchar, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16 " +
			"WHERE NOT EXISTS (SELECT 1 FROM service_requests WHERE service_request_id = $1);")

		if err != nil {
			log.Fatal("error preparing database insert statement", err)
		}

		update_stmt, err := db.Prepare("UPDATE service_requests SET " +
			"status = $2, service_name = $3, service_code = $4, agency_responsible = $5, " +
			"address = $6, requested_datetime = $7, updated_datetime = $8, lat = $9, long = $10," +
			"ward = $11, police_district = $12, media_url = $13, channel = $14, duplicate = $15, " +
			"parent_service_request_id = $16, updated_at = NOW() WHERE service_request_id = $1;")

		if err != nil {
			log.Fatal("error preparing database update statement", err)
		}

		tx, err := db.Begin()

		if err != nil {
			log.Fatal("error beginning transaction", err)
		}

		_, err = tx.Stmt(update_stmt).Exec(request.Service_request_id,
			request.Status,
			request.Service_name,
			request.Service_code,
			request.Agency_responsible,
			request.Address,
			request.Requested_datetime,
			request.Updated_datetime,
			request.Lat,
			request.Long,
			request.Extended_attributes["ward"],
			request.Extended_attributes["police_district"],
			request.Media_url,
			request.Extended_attributes["channel"],
			request.Extended_attributes["duplicate"],
			request.Extended_attributes["parent_service_request_id"])

		if err != nil {
			log.Fatalf("could not update %s because %s", request.Service_request_id, err)
		}

		_, err = tx.Stmt(insert_stmt).Exec(request.Service_request_id,
			request.Status,
			request.Service_name,
			request.Service_code,
			request.Agency_responsible,
			request.Address,
			request.Requested_datetime,
			request.Updated_datetime,
			request.Lat,
			request.Long,
			request.Extended_attributes["ward"],
			request.Extended_attributes["police_district"],
			request.Media_url,
			request.Extended_attributes["channel"],
			request.Extended_attributes["duplicate"],
			request.Extended_attributes["parent_service_request_id"])

		if err != nil {
			log.Fatalf("could not save %s because %s", request.Service_request_id, err)
		} else {
			log.Printf("saved SR %s", request)
		}

		err = tx.Commit()
		if err != nil {
			log.Fatal("error closing transaction", err)
		}
	}
}

func (req Open311Request) String() string {
	// pretty print SR information
	return fmt.Sprintf("%s: %s at %s %f,%f, last update %s", req.Service_request_id, req.Service_name, req.Address, req.Lat, req.Long, req.Updated_datetime)
}

func fetchRequests() (requests []Open311Request) {
	db, err := sql.Open("postgres", "dbname=cwfy sslmode=disable")
	if err != nil {
		log.Fatal("Cannot open database connection", err)
	}
	defer db.Close()

	// find the most recent SR that we know about in the database
	rows, err := db.Query("SELECT MAX(updated_datetime) FROM service_requests;")
	if err != nil {
		log.Fatal("error finding most recent service request", err)
	}

	last_updated_at := time.Now()
	for rows.Next() {
		if err := rows.Scan(&last_updated_at); err != nil {
			log.Fatal(err)
		}

		log.Printf("most recent SR timestamp %s", last_updated_at)
	}

	// janky hack to transform the last updated timestamp into
	// a format that plays nicely with the Open311 API
	// FIXME: there HAS to be a better way to handle this.
	formatted_date_string := last_updated_at.Format(time.RFC3339)
	formatted_date_string_with_tz := formatted_date_string[0:len(formatted_date_string)-1] + "-0500" // trunc the trailing 'Z' and tack on timezone

	// construct the request URI using base params and the proper time
	open311_api_endpoint := OPEN311_API_URI + "&updated_after=" + formatted_date_string_with_tz

	log.Printf("fetching from %s", open311_api_endpoint)
	resp, err := http.Get(open311_api_endpoint)
	defer resp.Body.Close()

	if err != nil {
		log.Fatalln("error fetching from Open311 endpoint", err)
	}

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("error loading response body", err)
	}

	// parse JSON and load into an array of Open311Request objects
	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("error parsing JSON:", err)
	}

	log.Printf("received %d requests from Open311", len(requests))

	return requests
}
<commit_msg>poll on a 30 second interval<commit_after>package main

import (
	"database/sql"
	"encoding/json"
	"fmt"
	_ "github.com/bmizerany/pq"
	"io/ioutil"
	"log"
	"net/http"
	"time"
)

const OPEN311_API_URI = "http://311api.cityofchicago.org/open311/v2/requests.json?extensions=true&page_size=500"

type Open311Request struct {
	Lat, Long                                                                                float64
	Ward, Police_district                                                                                int
	Service_request_id, Status, Service_name, Service_code, Agency_responsible, Address, Channel, Media_url string
	Requested_datetime, Updated_datetime                                                                    string // FIXME: should these be proper time objects?
	Extended_attributes                                                                                map[string]interface{}
}

func main() {
	// open database
	db, err := sql.Open("postgres", "dbname=cwfy sslmode=disable")
	if err != nil {
		log.Fatal("Cannot open database connection", err)
	}
	defer db.Close()

	last_run_at := time.Now()

	for {
		switch {
		case time.Since(last_run_at) > (30 * time.Second):
			poll_open311(db)
			last_run_at = time.Now()
		default:
			log.Print("sleeping for 10 seconds")
			time.Sleep(10 * time.Second)
		}
	}
}

func (req Open311Request) String() string {
	// pretty print SR information
	return fmt.Sprintf("%s: %s at %s %f,%f, last update %s", req.Service_request_id, req.Service_name, req.Address, req.Lat, req.Long, req.Updated_datetime)
}

func fetchRequests() (requests []Open311Request) {
	db, err := sql.Open("postgres", "dbname=cwfy sslmode=disable")
	if err != nil {
		log.Fatal("Cannot open database connection", err)
	}
	defer db.Close()

	// find the most recent SR that we know about in the database
	rows, err := db.Query("SELECT MAX(updated_datetime) FROM service_requests;")
	if err != nil {
		log.Fatal("error finding most recent service request", err)
	}

	last_updated_at := time.Now()
	for rows.Next() {
		if err := rows.Scan(&last_updated_at); err != nil {
			log.Fatal(err)
		}

		log.Printf("most recent SR timestamp %s", last_updated_at)
	}

	// janky hack to transform the last updated timestamp into
	// a format that plays nicely with the Open311 API
	// FIXME: there HAS to be a better way to handle this.
	formatted_date_string := last_updated_at.Format(time.RFC3339)
	formatted_date_string_with_tz := formatted_date_string[0:len(formatted_date_string)-1] + "-0500" // trunc the trailing 'Z' and tack on timezone

	// construct the request URI using base params and the proper time
	open311_api_endpoint := OPEN311_API_URI + "&updated_after=" + formatted_date_string_with_tz

	log.Printf("fetching from %s", open311_api_endpoint)
	resp, err := http.Get(open311_api_endpoint)
	defer resp.Body.Close()

	if err != nil {
		log.Fatalln("error fetching from Open311 endpoint", err)
	}

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("error loading response body", err)
	}

	// parse JSON and load into an array of Open311Request objects
	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("error parsing JSON:", err)
	}

	log.Printf("received %d requests from Open311", len(requests))

	return requests
}

func poll_open311(db *sql.DB) {
	// load requests from open311
	requests := fetchRequests()

	for _, request := range requests {
		// for each request, either create or update the
		// corresponding record in the database.

		if request.Service_request_id == "" {
			log.Printf("Ignoring a request type %s because there is no SR number assigned", request.Service_name)
			continue
		}

		insert_stmt, err := db.Prepare("INSERT INTO service_requests(service_request_id," +
			"status, service_name, service_code, agency_responsible, " +
			"address, requested_datetime, updated_datetime, lat, long," +
			"ward, police_district, media_url, channel, duplicate, parent_service_request_id) " +
			"SELECT $1::varchar, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16 " +
			"WHERE NOT EXISTS (SELECT 1 FROM service_requests WHERE service_request_id = $1);")

		if err != nil {
			log.Fatal("error preparing database insert statement", err)
		}

		update_stmt, err := db.Prepare("UPDATE service_requests SET " +
			"status = $2, service_name = $3, service_code = $4, agency_responsible = $5, " +
			"address = $6, requested_datetime = $7, updated_datetime = $8, lat = $9, long = $10," +
			"ward = $11, police_district = $12, media_url = $13, channel = $14, duplicate = $15, " +
			"parent_service_request_id = $16, updated_at = NOW() WHERE service_request_id = $1;")

		if err != nil {
			log.Fatal("error preparing database update statement", err)
		}

		tx, err := db.Begin()

		if err != nil {
			log.Fatal("error beginning transaction", err)
		}

		_, err = tx.Stmt(update_stmt).Exec(request.Service_request_id,
			request.Status,
			request.Service_name,
			request.Service_code,
			request.Agency_responsible,
			request.Address,
			request.Requested_datetime,
			request.Updated_datetime,
			request.Lat,
			request.Long,
			request.Extended_attributes["ward"],
			request.Extended_attributes["police_district"],
			request.Media_url,
			request.Extended_attributes["channel"],
			request.Extended_attributes["duplicate"],
			request.Extended_attributes["parent_service_request_id"])

		if err != nil {
			log.Fatalf("could not update %s because %s", request.Service_request_id, err)
		}

		_, err = tx.Stmt(insert_stmt).Exec(request.Service_request_id,
			request.Status,
			request.Service_name,
			request.Service_code,
			request.Agency_responsible,
			request.Address,
			request.Requested_datetime,
			request.Updated_datetime,
			request.Lat,
			request.Long,
			request.Extended_attributes["ward"],
			request.Extended_attributes["police_district"],
			request.Media_url,
			request.Extended_attributes["channel"],
			request.Extended_attributes["duplicate"],
			request.Extended_attributes["parent_service_request_id"])

		if err != nil {
			log.Fatalf("could not save %s because %s", request.Service_request_id, err)
		} else {
			log.Printf("saved SR %s", request)
		}

		err = tx.Commit()
		if err != nil {
			log.Fatal("error closing transaction", err)
		}
	}
}
<|endoftext|>


<commit_before>package sparta

import (
	"bytes"
	"fmt"
	"strings"
	"text/template"

	gocf "github.com/mweagle/go-cloudformation"
	"github.com/sirupsen/logrus"
)

// Utility function to marshal an interface
func marshalInterface(item interface{}) interface{} {
	if item != nil {
		return item
	}
	return item
}

// Utility function to marshal an int
func marshalInt(intVal int64) *gocf.IntegerExpr {
	if intVal != 0 {
		return gocf.Integer(intVal)
	}
	return nil
}

// Utility function to marshal a string
func marshalString(stringVal string) *gocf.StringExpr {
	if stringVal != "" {
		return gocf.String(stringVal)
	}
	return nil
}

func marshalStringExpr(stringExpr gocf.Stringable) *gocf.StringExpr {
	if stringExpr != nil {
		return stringExpr.String()
	}
	return nil
}

// Utility function to marshal a string lsit
func marshalStringList(stringVals []string) *gocf.StringListExpr {
	if len(stringVals) != 0 {
		stringableList := make([]gocf.Stringable, len(stringVals))
		for eachIndex, eachStringVal := range stringVals {
			stringableList[eachIndex] = gocf.String(eachStringVal)
		}
		return gocf.StringList(stringableList...)
	}
	return nil
}

// Utility function to marshal a boolean
func marshalBool(boolValue bool) *gocf.BoolExpr {
	if !boolValue {
		return gocf.Bool(boolValue)
	}
	return nil
}

// resourceOutputs is responsible for returning the conditional
// set of CloudFormation outputs for a given resource type.
func resourceOutputs(resourceName string,
	resource gocf.ResourceProperties,
	logger *logrus.Logger) ([]string, error) {

	outputProps := []string{}
	switch typedResource := resource.(type) {
	case gocf.IAMRole:
		// NOP
	case *gocf.DynamoDBTable:
		if typedResource.StreamSpecification != nil {
			outputProps = append(outputProps, "StreamArn")
		}
	case gocf.DynamoDBTable:
		if typedResource.StreamSpecification != nil {
			outputProps = append(outputProps, "StreamArn")
		}
	case gocf.KinesisStream,
		*gocf.KinesisStream:
		outputProps = append(outputProps, "Arn")
	case gocf.Route53RecordSet,
		*gocf.Route53RecordSet:
		// NOP
	case gocf.S3Bucket,
		*gocf.S3Bucket:
		outputProps = append(outputProps, "DomainName", "WebsiteURL")
	case gocf.SNSTopic,
		*gocf.SNSTopic:
		outputProps = append(outputProps, "TopicName")
	case gocf.SQSQueue,
		*gocf.SQSQueue:
		outputProps = append(outputProps, "Arn", "QueueName")
	default:
		logger.WithFields(logrus.Fields{
			"ResourceType": fmt.Sprintf("%T", typedResource),
		}).Warn("Discovery information for dependency not yet implemented")
	}
	return outputProps, nil
}

func newCloudFormationResource(resourceType string, logger *logrus.Logger) (gocf.ResourceProperties, error) {
	resProps := gocf.NewResourceByType(resourceType)
	if nil == resProps {
		logger.WithFields(logrus.Fields{
			"Type": resourceType,
		}).Fatal("Failed to create CloudFormation CustomResource!")
		return nil, fmt.Errorf("unsupported CustomResourceType: %s", resourceType)
	}
	return resProps, nil
}

type discoveryDataTemplate struct {
	ResourceID         string
	ResourceType       string
	ResourceProperties string
}

var discoveryDataForResourceDependency = `
	{
		"ResourceID" : "<< .ResourceID >>",
		"ResourceRef" : "{"Ref":"<< .ResourceID >>"}",
		"ResourceType" : "<< .ResourceType >>",
		"Properties" : {
			<< .ResourceProperties >>
		}
	}
`

func discoveryResourceInfoForDependency(cfTemplate *gocf.Template,
	logicalResourceName string,
	logger *logrus.Logger) ([]byte, error) {

	item, ok := cfTemplate.Resources[logicalResourceName]
	if !ok {
		return nil, nil
	}
	resourceOutputs, resourceOutputsErr := resourceOutputs(logicalResourceName,
		item.Properties,
		logger)
	if resourceOutputsErr != nil {
		return nil, resourceOutputsErr
	}
	// Template data
	templateData := &discoveryDataTemplate{
		ResourceID:   logicalResourceName,
		ResourceType: item.Properties.CfnResourceType(),
	}
	quotedAttrs := make([]string, len(resourceOutputs))
	for eachIndex, eachOutput := range resourceOutputs {
		quotedAttrs[eachIndex] = fmt.Sprintf(`"%s" :"{ "Fn::GetAtt" : [ "%s", "%s" ] }"`,
			eachOutput,
			logicalResourceName,
			eachOutput)
	}
	templateData.ResourceProperties = strings.Join(quotedAttrs, ",")

	// Create the data that can be stuffed into Environment
	discoveryTemplate, discoveryTemplateErr := template.New("discoveryResourceData").
		Delims("<<", ">>").
		Parse(discoveryDataForResourceDependency)
	if nil != discoveryTemplateErr {
		return nil, discoveryTemplateErr
	}

	var templateResults bytes.Buffer
	evalResultErr := discoveryTemplate.Execute(&templateResults, templateData)
	return templateResults.Bytes(), evalResultErr
}
func safeAppendDependency(resource *gocf.Resource, dependencyName string) {
	if nil == resource.DependsOn {
		resource.DependsOn = []string{}
	}
	resource.DependsOn = append(resource.DependsOn, dependencyName)
}
func safeMetadataInsert(resource *gocf.Resource, key string, value interface{}) {
	if nil == resource.Metadata {
		resource.Metadata = make(map[string]interface{})
	}
	resource.Metadata[key] = value
}
<commit_msg>Be a bit more explicit<commit_after>package sparta

import (
	"bytes"
	"fmt"
	"strings"
	"text/template"

	gocf "github.com/mweagle/go-cloudformation"
	"github.com/sirupsen/logrus"
)

// Utility function to marshal an interface
func marshalInterface(item interface{}) interface{} {
	if item != nil {
		return item
	}
	return nil
}

// Utility function to marshal an int
func marshalInt(intVal int64) *gocf.IntegerExpr {
	if intVal != 0 {
		return gocf.Integer(intVal)
	}
	return nil
}

// Utility function to marshal a string
func marshalString(stringVal string) *gocf.StringExpr {
	if stringVal != "" {
		return gocf.String(stringVal)
	}
	return nil
}

func marshalStringExpr(stringExpr gocf.Stringable) *gocf.StringExpr {
	if stringExpr != nil {
		return stringExpr.String()
	}
	return nil
}

// Utility function to marshal a string lsit
func marshalStringList(stringVals []string) *gocf.StringListExpr {
	if len(stringVals) != 0 {
		stringableList := make([]gocf.Stringable, len(stringVals))
		for eachIndex, eachStringVal := range stringVals {
			stringableList[eachIndex] = gocf.String(eachStringVal)
		}
		return gocf.StringList(stringableList...)
	}
	return nil
}

// Utility function to marshal a boolean
func marshalBool(boolValue bool) *gocf.BoolExpr {
	if !boolValue {
		return gocf.Bool(boolValue)
	}
	return nil
}

// resourceOutputs is responsible for returning the conditional
// set of CloudFormation outputs for a given resource type.
func resourceOutputs(resourceName string,
	resource gocf.ResourceProperties,
	logger *logrus.Logger) ([]string, error) {

	outputProps := []string{}
	switch typedResource := resource.(type) {
	case gocf.IAMRole:
		// NOP
	case *gocf.DynamoDBTable:
		if typedResource.StreamSpecification != nil {
			outputProps = append(outputProps, "StreamArn")
		}
	case gocf.DynamoDBTable:
		if typedResource.StreamSpecification != nil {
			outputProps = append(outputProps, "StreamArn")
		}
	case gocf.KinesisStream,
		*gocf.KinesisStream:
		outputProps = append(outputProps, "Arn")
	case gocf.Route53RecordSet,
		*gocf.Route53RecordSet:
		// NOP
	case gocf.S3Bucket,
		*gocf.S3Bucket:
		outputProps = append(outputProps, "DomainName", "WebsiteURL")
	case gocf.SNSTopic,
		*gocf.SNSTopic:
		outputProps = append(outputProps, "TopicName")
	case gocf.SQSQueue,
		*gocf.SQSQueue:
		outputProps = append(outputProps, "Arn", "QueueName")
	default:
		logger.WithFields(logrus.Fields{
			"ResourceType": fmt.Sprintf("%T", typedResource),
		}).Warn("Discovery information for dependency not yet implemented")
	}
	return outputProps, nil
}

func newCloudFormationResource(resourceType string, logger *logrus.Logger) (gocf.ResourceProperties, error) {
	resProps := gocf.NewResourceByType(resourceType)
	if nil == resProps {
		logger.WithFields(logrus.Fields{
			"Type": resourceType,
		}).Fatal("Failed to create CloudFormation CustomResource!")
		return nil, fmt.Errorf("unsupported CustomResourceType: %s", resourceType)
	}
	return resProps, nil
}

type discoveryDataTemplate struct {
	ResourceID         string
	ResourceType       string
	ResourceProperties string
}

var discoveryDataForResourceDependency = `
	{
		"ResourceID" : "<< .ResourceID >>",
		"ResourceRef" : "{"Ref":"<< .ResourceID >>"}",
		"ResourceType" : "<< .ResourceType >>",
		"Properties" : {
			<< .ResourceProperties >>
		}
	}
`

func discoveryResourceInfoForDependency(cfTemplate *gocf.Template,
	logicalResourceName string,
	logger *logrus.Logger) ([]byte, error) {

	item, ok := cfTemplate.Resources[logicalResourceName]
	if !ok {
		return nil, nil
	}
	resourceOutputs, resourceOutputsErr := resourceOutputs(logicalResourceName,
		item.Properties,
		logger)
	if resourceOutputsErr != nil {
		return nil, resourceOutputsErr
	}
	// Template data
	templateData := &discoveryDataTemplate{
		ResourceID:   logicalResourceName,
		ResourceType: item.Properties.CfnResourceType(),
	}
	quotedAttrs := make([]string, len(resourceOutputs))
	for eachIndex, eachOutput := range resourceOutputs {
		quotedAttrs[eachIndex] = fmt.Sprintf(`"%s" :"{ "Fn::GetAtt" : [ "%s", "%s" ] }"`,
			eachOutput,
			logicalResourceName,
			eachOutput)
	}
	templateData.ResourceProperties = strings.Join(quotedAttrs, ",")

	// Create the data that can be stuffed into Environment
	discoveryTemplate, discoveryTemplateErr := template.New("discoveryResourceData").
		Delims("<<", ">>").
		Parse(discoveryDataForResourceDependency)
	if nil != discoveryTemplateErr {
		return nil, discoveryTemplateErr
	}

	var templateResults bytes.Buffer
	evalResultErr := discoveryTemplate.Execute(&templateResults, templateData)
	return templateResults.Bytes(), evalResultErr
}
func safeAppendDependency(resource *gocf.Resource, dependencyName string) {
	if nil == resource.DependsOn {
		resource.DependsOn = []string{}
	}
	resource.DependsOn = append(resource.DependsOn, dependencyName)
}
func safeMetadataInsert(resource *gocf.Resource, key string, value interface{}) {
	if nil == resource.Metadata {
		resource.Metadata = make(map[string]interface{})
	}
	resource.Metadata[key] = value
}
<|endoftext|>