<commit_before>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package restore

import (
	"fmt"
	"io/ioutil"
	"log"
	"testing"

	"golang.org/x/net/context"

	"github.com/jacobsa/comeback/internal/blob"
	"github.com/jacobsa/comeback/internal/dag"
	"github.com/jacobsa/comeback/internal/fs"
	"github.com/jacobsa/comeback/internal/util"
	"github.com/jacobsa/comeback/internal/wiring"
	"github.com/jacobsa/gcloud/gcs/gcsfake"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"github.com/jacobsa/timeutil"
)

func TestDependencyResolver(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func convertNodes(dagNodes []dag.Node) (nodes []*node) {
	for _, n := range dagNodes {
		nodes = append(nodes, n.(*node))
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type DependencyResolverTest struct {
	ctx       context.Context
	blobStore blob.Store
	dr        dag.DependencyResolver
}

var _ SetUpInterface = &DependencyResolverTest{}

func init() { RegisterTestSuite(&DependencyResolverTest{}) }

func (t *DependencyResolverTest) SetUp(ti *TestInfo) {
	var err error
	t.ctx = ti.Ctx

	// Create the blob store.
	bucket := gcsfake.NewFakeBucket(timeutil.RealClock(), "some_bucket")

	_, crypter, err := wiring.MakeRegistryAndCrypter(t.ctx, "password", bucket)
	AssertEq(nil, err)

	t.blobStore, err = wiring.MakeBlobStore(bucket, crypter, util.NewStringSet())
	AssertEq(nil, err)

	// Create the dependency resolver.
	t.dr = newDependencyResolver(t.blobStore, log.New(ioutil.Discard, "", 0))
}

func (t *DependencyResolverTest) call(n *node) (deps []*node, err error) {
	untyped, err := t.dr.FindDependencies(t.ctx, n)
	if err != nil {
		err = fmt.Errorf("FindDependencies: %v", err)
		return
	}

	for _, u := range untyped {
		deps = append(deps, u.(*node))
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *DependencyResolverTest) File() {
	node := &node{
		Info: fs.DirectoryEntry{
			Type: fs.TypeFile,
		},
	}

	// Call
	deps, err := t.call(node)

	AssertEq(nil, err)
	ExpectThat(deps, ElementsAre())
	ExpectThat(node.Children, ElementsAre())
}

func (t *DependencyResolverTest) Symlink() {
	AssertTrue(false, "TODO")
}

func (t *DependencyResolverTest) UnhandledType() {
	AssertTrue(false, "TODO")
}

func (t *DependencyResolverTest) BlobMissing() {
	AssertTrue(false, "TODO")
}

func (t *DependencyResolverTest) BlobCorrupted() {
	AssertTrue(false, "TODO")
}

func (t *DependencyResolverTest) NoChildren() {
	AssertTrue(false, "TODO")
}

func (t *DependencyResolverTest) SomeChildren() {
	AssertTrue(false, "TODO")
}
<commit_msg>DependencyResolverTest.BlobMissing<commit_after>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package restore

import (
	"fmt"
	"io/ioutil"
	"log"
	"testing"

	"golang.org/x/net/context"

	"github.com/jacobsa/comeback/internal/blob"
	"github.com/jacobsa/comeback/internal/dag"
	"github.com/jacobsa/comeback/internal/fs"
	"github.com/jacobsa/comeback/internal/util"
	"github.com/jacobsa/comeback/internal/wiring"
	"github.com/jacobsa/gcloud/gcs/gcsfake"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"github.com/jacobsa/timeutil"
)

func TestDependencyResolver(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func convertNodes(dagNodes []dag.Node) (nodes []*node) {
	for _, n := range dagNodes {
		nodes = append(nodes, n.(*node))
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type DependencyResolverTest struct {
	ctx       context.Context
	blobStore blob.Store
	dr        dag.DependencyResolver
}

var _ SetUpInterface = &DependencyResolverTest{}

func init() { RegisterTestSuite(&DependencyResolverTest{}) }

func (t *DependencyResolverTest) SetUp(ti *TestInfo) {
	var err error
	t.ctx = ti.Ctx

	// Create the blob store.
	bucket := gcsfake.NewFakeBucket(timeutil.RealClock(), "some_bucket")

	_, crypter, err := wiring.MakeRegistryAndCrypter(t.ctx, "password", bucket)
	AssertEq(nil, err)

	t.blobStore, err = wiring.MakeBlobStore(bucket, crypter, util.NewStringSet())
	AssertEq(nil, err)

	// Create the dependency resolver.
	t.dr = newDependencyResolver(t.blobStore, log.New(ioutil.Discard, "", 0))
}

func (t *DependencyResolverTest) call(n *node) (deps []*node, err error) {
	untyped, err := t.dr.FindDependencies(t.ctx, n)
	if err != nil {
		err = fmt.Errorf("FindDependencies: %v", err)
		return
	}

	for _, u := range untyped {
		deps = append(deps, u.(*node))
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *DependencyResolverTest) File() {
	node := &node{
		Info: fs.DirectoryEntry{
			Type: fs.TypeFile,
		},
	}

	// Call
	deps, err := t.call(node)

	AssertEq(nil, err)
	ExpectThat(deps, ElementsAre())
	ExpectThat(node.Children, ElementsAre())
}

func (t *DependencyResolverTest) Symlink() {
	node := &node{
		Info: fs.DirectoryEntry{
			Type: fs.TypeSymlink,
		},
	}

	// Call
	deps, err := t.call(node)

	AssertEq(nil, err)
	ExpectThat(deps, ElementsAre())
	ExpectThat(node.Children, ElementsAre())
}

func (t *DependencyResolverTest) BlobMissing() {
	s := blob.ComputeScore([]byte(""))
	node := &node{
		Info: fs.DirectoryEntry{
			Type:   fs.TypeDirectory,
			Scores: []blob.Score{s},
		},
	}

	// Call
	_, err := t.call(node)

	ExpectThat(err, Error(HasSubstr("TODO")))
	ExpectThat(err, Error(HasSubstr(s.Hex())))
}

func (t *DependencyResolverTest) BlobCorrupted() {
	AssertTrue(false, "TODO")
}

func (t *DependencyResolverTest) NoChildren() {
	AssertTrue(false, "TODO")
}

func (t *DependencyResolverTest) SomeChildren() {
	AssertTrue(false, "TODO")
}
<|endoftext|>


<commit_before>package addr

import (
	"fmt"
	"net"
)

var (
	privateBlocks []*net.IPNet
)

func init() {
	for _, b := range []string{"10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "100.64.0.0/10", "fd00::/8"} {
		if _, block, err := net.ParseCIDR(b); err == nil {
			privateBlocks = append(privateBlocks, block)
		}
	}
}

func isPrivateIP(ipAddr string) bool {
	ip := net.ParseIP(ipAddr)
	for _, priv := range privateBlocks {
		if priv.Contains(ip) {
			return true
		}
	}
	return false
}

// Extract returns a real ip
func Extract(addr string) (string, error) {
	// if addr specified then its returned
	if len(addr) > 0 && (addr != "0.0.0.0" && addr != "[::]" && addr != "::") {
		return addr, nil
	}

	ifaces, err := net.Interfaces()
	if err != nil {
		return "", fmt.Errorf("Failed to get interfaces! Err: %v", err)
	}

	var addrs []net.Addr
	for _, iface := range ifaces {
		ifaceAddrs, err := iface.Addrs()
		if err != nil {
			// ignore error, interface can dissapear from system
			continue
		}
		addrs = append(addrs, ifaceAddrs...)
	}

	var ipAddr []byte
	var publicIP []byte

	for _, rawAddr := range addrs {
		var ip net.IP
		switch addr := rawAddr.(type) {
		case *net.IPAddr:
			ip = addr.IP
		case *net.IPNet:
			ip = addr.IP
		default:
			continue
		}

		if !isPrivateIP(ip.String()) {
			publicIP = ip
			continue
		}

		ipAddr = ip
		break
	}

	// return private ip
	if ipAddr != nil {
		return net.IP(ipAddr).String(), nil
	}

	// return public or virtual ip
	if publicIP != nil {
		return net.IP(publicIP).String(), nil
	}

	return "", fmt.Errorf("No IP address found, and explicit IP not provided")
}

// IPs returns all known ips
func IPs() []string {
	ifaces, err := net.Interfaces()
	if err != nil {
		return nil
	}

	var ipAddrs []string

	for _, i := range ifaces {
		addrs, err := i.Addrs()
		if err != nil {
			continue
		}

		for _, addr := range addrs {
			var ip net.IP
			switch v := addr.(type) {
			case *net.IPNet:
				ip = v.IP
			case *net.IPAddr:
				ip = v.IP
			}

			if ip == nil {
				continue
			}

			// dont skip ipv6 addrs
			/*
				ip = ip.To4()
				if ip == nil {
					continue
				}
			*/

			ipAddrs = append(ipAddrs, ip.String())
		}
	}

	return ipAddrs
}
<commit_msg>ignore Loopback Address (LVS,DR mode)<commit_after>package addr

import (
	"fmt"
	"net"
)

var (
	privateBlocks []*net.IPNet
)

func init() {
	for _, b := range []string{"10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "100.64.0.0/10", "fd00::/8"} {
		if _, block, err := net.ParseCIDR(b); err == nil {
			privateBlocks = append(privateBlocks, block)
		}
	}
}

func isPrivateIP(ipAddr string) bool {
	ip := net.ParseIP(ipAddr)
	for _, priv := range privateBlocks {
		if priv.Contains(ip) {
			return true
		}
	}
	return false
}

// Extract returns a real ip
func Extract(addr string) (string, error) {
	// if addr specified then its returned
	if len(addr) > 0 && (addr != "0.0.0.0" && addr != "[::]" && addr != "::") {
		return addr, nil
	}

	ifaces, err := net.Interfaces()
	if err != nil {
		return "", fmt.Errorf("Failed to get interfaces! Err: %v", err)
	}

	var addrs []net.Addr
	for _, iface := range ifaces {
		ifaceAddrs, err := iface.Addrs()
		if err != nil {
			// ignore error, interface can dissapear from system
			continue
		}
		if iface.Flags&net.FlagLoopback != 0 {
			continue
		}
		addrs = append(addrs, ifaceAddrs...)
	}

	var ipAddr []byte
	var publicIP []byte

	for _, rawAddr := range addrs {
		var ip net.IP
		switch addr := rawAddr.(type) {
		case *net.IPAddr:
			ip = addr.IP
		case *net.IPNet:
			ip = addr.IP
		default:
			continue
		}

		if !isPrivateIP(ip.String()) {
			publicIP = ip
			continue
		}

		ipAddr = ip
		break
	}

	// return private ip
	if ipAddr != nil {
		return net.IP(ipAddr).String(), nil
	}

	// return public or virtual ip
	if publicIP != nil {
		return net.IP(publicIP).String(), nil
	}

	return "", fmt.Errorf("No IP address found, and explicit IP not provided")
}

// IPs returns all known ips
func IPs() []string {
	ifaces, err := net.Interfaces()
	if err != nil {
		return nil
	}

	var ipAddrs []string

	for _, i := range ifaces {
		addrs, err := i.Addrs()
		if err != nil {
			continue
		}

		for _, addr := range addrs {
			var ip net.IP
			switch v := addr.(type) {
			case *net.IPNet:
				ip = v.IP
			case *net.IPAddr:
				ip = v.IP
			}

			if ip == nil {
				continue
			}

			// dont skip ipv6 addrs
			/*
				ip = ip.To4()
				if ip == nil {
					continue
				}
			*/

			ipAddrs = append(ipAddrs, ip.String())
		}
	}

	return ipAddrs
}
<|endoftext|>


<commit_before>// Copyright 2017 Microsoft. All rights reserved.
// MIT License

package network

import (
	"net"

	"github.com/Azure/azure-container-networking/cni"
	"github.com/Azure/azure-container-networking/common"
	"github.com/Azure/azure-container-networking/log"
	"github.com/Azure/azure-container-networking/network"
	"github.com/Azure/azure-container-networking/platform"

	cniInvoke "github.com/containernetworking/cni/pkg/invoke"
	cniSkel "github.com/containernetworking/cni/pkg/skel"
	cniTypes "github.com/containernetworking/cni/pkg/types"
	cniTypesImpl "github.com/containernetworking/cni/pkg/types/020"
)

const (
	// Plugin name.
	name = "azure-vnet"
)

// NetPlugin represents the CNI network plugin.
type netPlugin struct {
	*cni.Plugin
	nm network.NetworkManager
}

// NewPlugin creates a new netPlugin object.
func NewPlugin(config *common.PluginConfig) (*netPlugin, error) {
	// Setup base plugin.
	plugin, err := cni.NewPlugin(name, config.Version)
	if err != nil {
		return nil, err
	}

	// Setup network manager.
	nm, err := network.NewNetworkManager()
	if err != nil {
		return nil, err
	}

	config.NetApi = nm

	return &netPlugin{
		Plugin: plugin,
		nm:     nm,
	}, nil
}

// Starts the plugin.
func (plugin *netPlugin) Start(config *common.PluginConfig) error {
	// Initialize base plugin.
	err := plugin.Initialize(config)
	if err != nil {
		log.Printf("[cni-net] Failed to initialize base plugin, err:%v.", err)
		return err
	}

	// Log platform information.
	log.Printf("[cni-net] Plugin %v version %v.", plugin.Name, plugin.Version)
	log.Printf("[cni-net] Running on %v", platform.GetOSInfo())
	common.LogNetworkInterfaces()

	// Initialize network manager.
	err = plugin.nm.Initialize(config)
	if err != nil {
		log.Printf("[cni-net] Failed to initialize network manager, err:%v.", err)
		return err
	}

	log.Printf("[cni-net] Plugin started.")

	return nil
}

// Stops the plugin.
func (plugin *netPlugin) Stop() {
	plugin.nm.Uninitialize()
	plugin.Uninitialize()
	log.Printf("[cni-net] Plugin stopped.")
}

// GetEndpointID returns a unique endpoint ID based on the CNI args.
func (plugin *netPlugin) getEndpointID(args *cniSkel.CmdArgs) string {
	return args.ContainerID[:8] + "-" + args.IfName
}

// FindMasterInterface returns the name of the master interface.
func (plugin *netPlugin) findMasterInterface(nwCfg *cni.NetworkConfig, subnetPrefix *net.IPNet) string {
	// An explicit master configuration wins. Explicitly specifying a master is
	// useful if host has multiple interfaces with addresses in the same subnet.
	if nwCfg.Master != "" {
		return nwCfg.Master
	}

	// Otherwise, pick the first interface with an IP address in the given subnet.
	subnetPrefixString := subnetPrefix.String()
	interfaces, _ := net.Interfaces()
	for _, iface := range interfaces {
		addrs, _ := iface.Addrs()
		for _, addr := range addrs {
			_, ipnet, err := net.ParseCIDR(addr.String())
			if err != nil {
				continue
			}
			if subnetPrefixString == ipnet.String() {
				return iface.Name
			}
		}
	}

	// Failed to find a suitable interface.
	return ""
}

//
// CNI implementation
// https://github.com/containernetworking/cni/blob/master/SPEC.md
//

// Add handles CNI add commands.
func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
	log.Printf("[cni-net] Processing ADD command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v}.",
		args.ContainerID, args.Netns, args.IfName, args.Args, args.Path)

	// Parse network configuration from stdin.
	nwCfg, err := cni.ParseNetworkConfig(args.StdinData)
	if err != nil {
		return plugin.Errorf("Failed to parse network configuration: %v.", err)
	}

	log.Printf("[cni-net] Read network configuration %+v.", nwCfg)

	// Initialize values from network config.
	var result cniTypes.Result
	var resultImpl *cniTypesImpl.Result
	networkId := nwCfg.Name
	endpointId := plugin.getEndpointID(args)

	// Check whether the network already exists.
	nwInfo, err := plugin.nm.GetNetworkInfo(networkId)
	if err != nil {
		// Network does not exist.
		log.Printf("[cni-net] Creating network %v.", networkId)

		// Call into IPAM plugin to allocate an address pool for the network.
		result, err = cniInvoke.DelegateAdd(nwCfg.Ipam.Type, nwCfg.Serialize())
		if err != nil {
			return plugin.Errorf("Failed to allocate pool: %v", err)
		}

		resultImpl, err = cniTypesImpl.GetResult(result)

		log.Printf("[cni-net] IPAM plugin returned result %v.", resultImpl)

		// Derive the subnet prefix from allocated IP address.
		subnetPrefix := resultImpl.IP4.IP
		subnetPrefix.IP = subnetPrefix.IP.Mask(subnetPrefix.Mask)

		// Find the master interface.
		masterIfName := plugin.findMasterInterface(nwCfg, &subnetPrefix)
		if masterIfName == "" {
			return plugin.Errorf("Failed to find the master interface")
		}
		log.Printf("[cni-net] Found master interface %v.", masterIfName)

		// Add the master as an external interface.
		err = plugin.nm.AddExternalInterface(masterIfName, subnetPrefix.String())
		if err != nil {
			return plugin.Errorf("Failed to add external interface: %v", err)
		}

		// Create the network.
		nwInfo := network.NetworkInfo{
			Id:   networkId,
			Mode: nwCfg.Mode,
			Subnets: []network.SubnetInfo{
				network.SubnetInfo{
					Family:  platform.AfINET,
					Prefix:  subnetPrefix,
					Gateway: resultImpl.IP4.Gateway,
				},
			},
			BridgeName: nwCfg.Bridge,
		}

		err = plugin.nm.CreateNetwork(&nwInfo)
		if err != nil {
			return plugin.Errorf("Failed to create network: %v", err)
		}

		log.Printf("[cni-net] Created network %v with subnet %v.", networkId, subnetPrefix.String())
	} else {
		// Network already exists.
		subnetPrefix := nwInfo.Subnets[0].Prefix.String()
		log.Printf("[cni-net] Found network %v with subnet %v.", networkId, subnetPrefix)

		// Call into IPAM plugin to allocate an address for the endpoint.
		nwCfg.Ipam.Subnet = subnetPrefix
		result, err = cniInvoke.DelegateAdd(nwCfg.Ipam.Type, nwCfg.Serialize())
		if err != nil {
			return plugin.Errorf("Failed to allocate address: %v", err)
		}

		resultImpl, err = cniTypesImpl.GetResult(result)

		log.Printf("[cni-net] IPAM plugin returned result %v.", resultImpl)
	}

	// Initialize endpoint info.
	epInfo := &network.EndpointInfo{
		Id:          endpointId,
		ContainerID: args.ContainerID,
		NetNsPath:   args.Netns,
		IfName:      args.IfName,
	}

	// Populate addresses and routes.
	if resultImpl.IP4 != nil {
		epInfo.IPAddresses = append(epInfo.IPAddresses, resultImpl.IP4.IP)

		for _, route := range resultImpl.IP4.Routes {
			epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: route.Dst, Gw: route.GW})
		}
	}

	// Populate DNS info.
	epInfo.DNS.Suffix = resultImpl.DNS.Domain
	epInfo.DNS.Servers = resultImpl.DNS.Nameservers

	// Create the endpoint.
	log.Printf("[cni-net] Creating endpoint %v.", epInfo.Id)
	err = plugin.nm.CreateEndpoint(networkId, epInfo)
	if err != nil {
		return plugin.Errorf("Failed to create endpoint: %v", err)
	}

	// Convert result to the requested CNI version.
	result, err = resultImpl.GetAsVersion(nwCfg.CniVersion)
	if err != nil {
		return plugin.Error(err)
	}

	// Output the result to stdout.
	result.Print()

	log.Printf("[cni-net] ADD succeeded with output %+v.", result)

	return nil
}

// Delete handles CNI delete commands.
func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error {
	log.Printf("[cni-net] Processing DEL command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v}.",
		args.ContainerID, args.Netns, args.IfName, args.Args, args.Path)

	// Parse network configuration from stdin.
	nwCfg, err := cni.ParseNetworkConfig(args.StdinData)
	if err != nil {
		return plugin.Errorf("Failed to parse network configuration: %v", err)
	}

	log.Printf("[cni-net] Read network configuration %+v.", nwCfg)

	// Initialize values from network config.
	networkId := nwCfg.Name
	endpointId := plugin.getEndpointID(args)

	// Query the network.
	nwInfo, err := plugin.nm.GetNetworkInfo(networkId)
	if err != nil {
		return plugin.Errorf("Failed to query network: %v", err)
	}

	// Query the endpoint.
	epInfo, err := plugin.nm.GetEndpointInfo(networkId, endpointId)
	if err != nil {
		return plugin.Errorf("Failed to query endpoint: %v", err)
	}

	// Delete the endpoint.
	err = plugin.nm.DeleteEndpoint(networkId, endpointId)
	if err != nil {
		return plugin.Errorf("Failed to delete endpoint: %v", err)
	}

	// Call into IPAM plugin to release the endpoint's addresses.
	nwCfg.Ipam.Subnet = nwInfo.Subnets[0].Prefix.String()
	for _, address := range epInfo.IPAddresses {
		nwCfg.Ipam.Address = address.IP.String()
		err = cniInvoke.DelegateDel(nwCfg.Ipam.Type, nwCfg.Serialize())
		if err != nil {
			return plugin.Errorf("Failed to release address: %v", err)
		}
	}

	log.Printf("[cni-net] DEL succeeded.")

	return nil
}
<commit_msg>Bounds check the CNI ContainerID before using it (#59)<commit_after>// Copyright 2017 Microsoft. All rights reserved.
// MIT License

package network

import (
	"net"

	"github.com/Azure/azure-container-networking/cni"
	"github.com/Azure/azure-container-networking/common"
	"github.com/Azure/azure-container-networking/log"
	"github.com/Azure/azure-container-networking/network"
	"github.com/Azure/azure-container-networking/platform"

	cniInvoke "github.com/containernetworking/cni/pkg/invoke"
	cniSkel "github.com/containernetworking/cni/pkg/skel"
	cniTypes "github.com/containernetworking/cni/pkg/types"
	cniTypesImpl "github.com/containernetworking/cni/pkg/types/020"
)

const (
	// Plugin name.
	name = "azure-vnet"
)

// NetPlugin represents the CNI network plugin.
type netPlugin struct {
	*cni.Plugin
	nm network.NetworkManager
}

// NewPlugin creates a new netPlugin object.
func NewPlugin(config *common.PluginConfig) (*netPlugin, error) {
	// Setup base plugin.
	plugin, err := cni.NewPlugin(name, config.Version)
	if err != nil {
		return nil, err
	}

	// Setup network manager.
	nm, err := network.NewNetworkManager()
	if err != nil {
		return nil, err
	}

	config.NetApi = nm

	return &netPlugin{
		Plugin: plugin,
		nm:     nm,
	}, nil
}

// Starts the plugin.
func (plugin *netPlugin) Start(config *common.PluginConfig) error {
	// Initialize base plugin.
	err := plugin.Initialize(config)
	if err != nil {
		log.Printf("[cni-net] Failed to initialize base plugin, err:%v.", err)
		return err
	}

	// Log platform information.
	log.Printf("[cni-net] Plugin %v version %v.", plugin.Name, plugin.Version)
	log.Printf("[cni-net] Running on %v", platform.GetOSInfo())
	common.LogNetworkInterfaces()

	// Initialize network manager.
	err = plugin.nm.Initialize(config)
	if err != nil {
		log.Printf("[cni-net] Failed to initialize network manager, err:%v.", err)
		return err
	}

	log.Printf("[cni-net] Plugin started.")

	return nil
}

// Stops the plugin.
func (plugin *netPlugin) Stop() {
	plugin.nm.Uninitialize()
	plugin.Uninitialize()
	log.Printf("[cni-net] Plugin stopped.")
}

// GetEndpointID returns a unique endpoint ID based on the CNI args.
func (plugin *netPlugin) getEndpointID(args *cniSkel.CmdArgs) string {
	var containerID string
	if len(args.ContainerID) >= 8 {
		containerID = args.ContainerID[:8] + "-" + args.IfName
	} else {
		containerID = args.ContainerID + "-" + args.IfName
	}
	return containerID
}

// FindMasterInterface returns the name of the master interface.
func (plugin *netPlugin) findMasterInterface(nwCfg *cni.NetworkConfig, subnetPrefix *net.IPNet) string {
	// An explicit master configuration wins. Explicitly specifying a master is
	// useful if host has multiple interfaces with addresses in the same subnet.
	if nwCfg.Master != "" {
		return nwCfg.Master
	}

	// Otherwise, pick the first interface with an IP address in the given subnet.
	subnetPrefixString := subnetPrefix.String()
	interfaces, _ := net.Interfaces()
	for _, iface := range interfaces {
		addrs, _ := iface.Addrs()
		for _, addr := range addrs {
			_, ipnet, err := net.ParseCIDR(addr.String())
			if err != nil {
				continue
			}
			if subnetPrefixString == ipnet.String() {
				return iface.Name
			}
		}
	}

	// Failed to find a suitable interface.
	return ""
}

//
// CNI implementation
// https://github.com/containernetworking/cni/blob/master/SPEC.md
//

// Add handles CNI add commands.
func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
	log.Printf("[cni-net] Processing ADD command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v}.",
		args.ContainerID, args.Netns, args.IfName, args.Args, args.Path)

	// Parse network configuration from stdin.
	nwCfg, err := cni.ParseNetworkConfig(args.StdinData)
	if err != nil {
		return plugin.Errorf("Failed to parse network configuration: %v.", err)
	}

	log.Printf("[cni-net] Read network configuration %+v.", nwCfg)

	// Initialize values from network config.
	var result cniTypes.Result
	var resultImpl *cniTypesImpl.Result
	networkId := nwCfg.Name
	endpointId := plugin.getEndpointID(args)

	// Check whether the network already exists.
	nwInfo, err := plugin.nm.GetNetworkInfo(networkId)
	if err != nil {
		// Network does not exist.
		log.Printf("[cni-net] Creating network %v.", networkId)

		// Call into IPAM plugin to allocate an address pool for the network.
		result, err = cniInvoke.DelegateAdd(nwCfg.Ipam.Type, nwCfg.Serialize())
		if err != nil {
			return plugin.Errorf("Failed to allocate pool: %v", err)
		}

		resultImpl, err = cniTypesImpl.GetResult(result)

		log.Printf("[cni-net] IPAM plugin returned result %v.", resultImpl)

		// Derive the subnet prefix from allocated IP address.
		subnetPrefix := resultImpl.IP4.IP
		subnetPrefix.IP = subnetPrefix.IP.Mask(subnetPrefix.Mask)

		// Find the master interface.
		masterIfName := plugin.findMasterInterface(nwCfg, &subnetPrefix)
		if masterIfName == "" {
			return plugin.Errorf("Failed to find the master interface")
		}
		log.Printf("[cni-net] Found master interface %v.", masterIfName)

		// Add the master as an external interface.
		err = plugin.nm.AddExternalInterface(masterIfName, subnetPrefix.String())
		if err != nil {
			return plugin.Errorf("Failed to add external interface: %v", err)
		}

		// Create the network.
		nwInfo := network.NetworkInfo{
			Id:   networkId,
			Mode: nwCfg.Mode,
			Subnets: []network.SubnetInfo{
				network.SubnetInfo{
					Family:  platform.AfINET,
					Prefix:  subnetPrefix,
					Gateway: resultImpl.IP4.Gateway,
				},
			},
			BridgeName: nwCfg.Bridge,
		}

		err = plugin.nm.CreateNetwork(&nwInfo)
		if err != nil {
			return plugin.Errorf("Failed to create network: %v", err)
		}

		log.Printf("[cni-net] Created network %v with subnet %v.", networkId, subnetPrefix.String())
	} else {
		// Network already exists.
		subnetPrefix := nwInfo.Subnets[0].Prefix.String()
		log.Printf("[cni-net] Found network %v with subnet %v.", networkId, subnetPrefix)

		// Call into IPAM plugin to allocate an address for the endpoint.
		nwCfg.Ipam.Subnet = subnetPrefix
		result, err = cniInvoke.DelegateAdd(nwCfg.Ipam.Type, nwCfg.Serialize())
		if err != nil {
			return plugin.Errorf("Failed to allocate address: %v", err)
		}

		resultImpl, err = cniTypesImpl.GetResult(result)

		log.Printf("[cni-net] IPAM plugin returned result %v.", resultImpl)
	}

	// Initialize endpoint info.
	epInfo := &network.EndpointInfo{
		Id:          endpointId,
		ContainerID: args.ContainerID,
		NetNsPath:   args.Netns,
		IfName:      args.IfName,
	}

	// Populate addresses and routes.
	if resultImpl.IP4 != nil {
		epInfo.IPAddresses = append(epInfo.IPAddresses, resultImpl.IP4.IP)

		for _, route := range resultImpl.IP4.Routes {
			epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: route.Dst, Gw: route.GW})
		}
	}

	// Populate DNS info.
	epInfo.DNS.Suffix = resultImpl.DNS.Domain
	epInfo.DNS.Servers = resultImpl.DNS.Nameservers

	// Create the endpoint.
	log.Printf("[cni-net] Creating endpoint %v.", epInfo.Id)
	err = plugin.nm.CreateEndpoint(networkId, epInfo)
	if err != nil {
		return plugin.Errorf("Failed to create endpoint: %v", err)
	}

	// Convert result to the requested CNI version.
	result, err = resultImpl.GetAsVersion(nwCfg.CniVersion)
	if err != nil {
		return plugin.Error(err)
	}

	// Output the result to stdout.
	result.Print()

	log.Printf("[cni-net] ADD succeeded with output %+v.", result)

	return nil
}

// Delete handles CNI delete commands.
func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error {
	log.Printf("[cni-net] Processing DEL command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v}.",
		args.ContainerID, args.Netns, args.IfName, args.Args, args.Path)

	// Parse network configuration from stdin.
	nwCfg, err := cni.ParseNetworkConfig(args.StdinData)
	if err != nil {
		return plugin.Errorf("Failed to parse network configuration: %v", err)
	}

	log.Printf("[cni-net] Read network configuration %+v.", nwCfg)

	// Initialize values from network config.
	networkId := nwCfg.Name
	endpointId := plugin.getEndpointID(args)

	// Query the network.
	nwInfo, err := plugin.nm.GetNetworkInfo(networkId)
	if err != nil {
		return plugin.Errorf("Failed to query network: %v", err)
	}

	// Query the endpoint.
	epInfo, err := plugin.nm.GetEndpointInfo(networkId, endpointId)
	if err != nil {
		return plugin.Errorf("Failed to query endpoint: %v", err)
	}

	// Delete the endpoint.
	err = plugin.nm.DeleteEndpoint(networkId, endpointId)
	if err != nil {
		return plugin.Errorf("Failed to delete endpoint: %v", err)
	}

	// Call into IPAM plugin to release the endpoint's addresses.
	nwCfg.Ipam.Subnet = nwInfo.Subnets[0].Prefix.String()
	for _, address := range epInfo.IPAddresses {
		nwCfg.Ipam.Address = address.IP.String()
		err = cniInvoke.DelegateDel(nwCfg.Ipam.Type, nwCfg.Serialize())
		if err != nil {
			return plugin.Errorf("Failed to release address: %v", err)
		}
	}

	log.Printf("[cni-net] DEL succeeded.")

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"strings"
	"sync"
	"time"

	docker "github.com/fsouza/go-dockerclient"
	"github.com/litl/galaxy/log"
	"github.com/litl/galaxy/registry"
	"github.com/litl/galaxy/runtime"
	"github.com/litl/galaxy/utils"
)

var (
	stopCutoff      int64
	apps            []string
	redisHost       string
	env             string
	pool            string
	loop            bool
	shuttleHost     string
	statsdHost      string
	debug           bool
	runOnce         bool
	version         bool
	buildVersion    string
	serviceConfigs  []*registry.ServiceConfig
	serviceRegistry *registry.ServiceRegistry
	serviceRuntime  *runtime.ServiceRuntime
	workerChans     map[string]chan string
	wg              sync.WaitGroup
)

func initOrDie() {

	serviceRegistry = registry.NewServiceRegistry(
		"",
		registry.DefaultTTL,
		"",
	)

	serviceRegistry.Connect(redisHost)
	serviceRuntime = runtime.NewServiceRuntime(serviceRegistry, shuttleHost, statsdHost)

	apps, err := serviceRegistry.ListAssignments(env, pool)
	if err != nil {
		log.Fatalf("ERROR: Could not retrieve service configs for /%s/%s: %s\n", env, pool, err)
	}

	workerChans = make(map[string]chan string)
	for _, app := range apps {
		serviceConfig, err := serviceRegistry.GetServiceConfig(app, env)
		if err != nil {
			log.Fatalf("ERROR: Could not retrieve service config for /%s/%s: %s\n", env, pool, err)
		}

		workerChans[serviceConfig.Name] = make(chan string)
	}
}

func pullImageAsync(serviceConfig registry.ServiceConfig, errChan chan error) {
	// err logged via pullImage
	_, err := pullImage(&serviceConfig)
	if err != nil {
		errChan <- err
		return
	}
	errChan <- nil
}

func pullImage(serviceConfig *registry.ServiceConfig) (*docker.Image, error) {

	image, err := serviceRuntime.InspectImage(serviceConfig.Version())
	if image != nil && image.ID == serviceConfig.VersionID() || serviceConfig.VersionID() == "" {
		return image, nil
	}

	log.Printf("Pulling %s version %s\n", serviceConfig.Name, serviceConfig.Version())
	image, err = serviceRuntime.PullImage(serviceConfig.Version(),
		serviceConfig.VersionID(), true)
	if image == nil || err != nil {
		log.Errorf("ERROR: Could not pull image %s: %s",
			serviceConfig.Version(), err)
		return nil, err
	}

	if image.ID != serviceConfig.VersionID() && len(serviceConfig.VersionID()) > 12 {
		log.Errorf("ERROR: Pulled image for %s does not match expected ID. Expected: %s: Got: %s",
			serviceConfig.Version(),
			image.ID[0:12], serviceConfig.VersionID()[0:12])
		return nil, errors.New(fmt.Sprintf("failed to pull image ID %s", serviceConfig.VersionID()[0:12]))
	}

	log.Printf("Pulled %s\n", serviceConfig.Version())
	return image, nil
}

func startService(serviceConfig *registry.ServiceConfig, logStatus bool) {
	started, container, err := serviceRuntime.StartIfNotRunning(env, serviceConfig)
	if err != nil {
		log.Errorf("ERROR: Could not start container for %s: %s", serviceConfig.Version(), err)
		return
	}

	if started {
		log.Printf("Started %s version %s as %s\n", serviceConfig.Name, serviceConfig.Version(), container.ID[0:12])
	}

	if logStatus && !debug {
		log.Printf("%s version %s running as %s\n", serviceConfig.Name, serviceConfig.Version(), container.ID[0:12])
	}

	log.Debugf("%s version %s running as %s\n", serviceConfig.Name, serviceConfig.Version(), container.ID[0:12])

	err = serviceRuntime.StopAllButLatestService(serviceConfig, stopCutoff)
	if err != nil {
		log.Errorf("ERROR: Could not stop containers: %s", err)
	}
}

func appAssigned(app string) (bool, error) {
	assignments, err := serviceRegistry.ListAssignments(env, pool)
	if err != nil {
		return false, err
	}

	if !utils.StringInSlice(app, assignments) {
		return false, nil
	}
	return true, nil
}

func restartContainers(app string, cmdChan chan string) {
	defer wg.Done()
	logOnce := true

	ticker := time.NewTicker(10 * time.Second)

	for {

		select {

		case cmd := <-cmdChan:

			assigned, err := appAssigned(app)
			if err != nil {
				log.Errorf("ERROR: Error retrieving assignments for %s: %s", app, err)
				if !loop {
					return
				}
				continue
			}

			if !assigned {
				continue
			}

			serviceConfig, err := serviceRegistry.GetServiceConfig(app, env)
			if err != nil {
				log.Errorf("ERROR: Error retrieving service config for %s: %s", app, err)
				if !loop {
					return
				}
				continue
			}

			if serviceConfig.Version() == "" {
				if !loop {
					return
				}
				continue
			}

			if cmd == "deploy" {
				_, err = pullImage(serviceConfig)
				if err != nil {
					if !loop {
						return
					}
					continue
				}
				startService(serviceConfig, logOnce)
			}

			if cmd == "restart" {
				err := serviceRuntime.Stop(serviceConfig)
				if err != nil {
					log.Errorf("ERROR: Could not stop %s: %s",
						serviceConfig.Version(), err)
					if !loop {
						return
					}

					startService(serviceConfig, logOnce)
					continue
				}
			}

			logOnce = false
		case <-ticker.C:

			serviceConfig, err := serviceRegistry.GetServiceConfig(app, env)
			if err != nil {
				log.Errorf("ERROR: Error retrieving service config for %s: %s", app, err)
				continue
			}

			assigned, err := appAssigned(app)
			if err != nil {
				log.Errorf("ERROR: Error retrieving service config for %s: %s", app, err)
				if !loop {
					return
				}

				continue
			}

			if serviceConfig == nil || !assigned {
				log.Errorf("%s no longer exists.  Stopping worker.", app)
				serviceRuntime.StopAllMatching(app)
				delete(workerChans, app)
				return
			}

			if serviceConfig.Version() == "" {
				continue
			}

			_, err = pullImage(serviceConfig)
			if err != nil {
				if !loop {
					return
				}
				log.Errorf("ERROR: Could not pull images: %s", err)
				continue
			}

			started, container, err := serviceRuntime.StartIfNotRunning(env, serviceConfig)
			if err != nil {
				log.Errorf("ERROR: Could not start containers: %s", err)
				continue
			}

			if started {
				log.Printf("Started %s version %s as %s\n", serviceConfig.Name, serviceConfig.Version(), container.ID[0:12])
			}

			log.Debugf("%s version %s running as %s\n", serviceConfig.Name, serviceConfig.Version(), container.ID[0:12])

			err = serviceRuntime.StopAllButCurrentVersion(serviceConfig)
			if err != nil {
				log.Errorf("ERROR: Could not stop containers: %s", err)
			}
		}

		if !loop {
			return
		}

	}
}

func monitorService(changedConfigs chan *registry.ConfigChange) {

	for {

		var changedConfig *registry.ConfigChange
		select {

		case changedConfig = <-changedConfigs:

			if changedConfig.Error != nil {
				log.Errorf("ERROR: Error watching changes: %s", changedConfig.Error)
				continue
			}

			if changedConfig.ServiceConfig == nil {
				continue
			}

			assigned, err := appAssigned(changedConfig.ServiceConfig.Name)
			if err != nil {
				log.Errorf("ERROR: Error retrieving service config for %s: %s", changedConfig.ServiceConfig.Name, err)
				if !loop {
					return
				}
				continue
			}

			if !assigned {
				continue
			}

			ch, ok := workerChans[changedConfig.ServiceConfig.Name]
			if !ok {
				name := changedConfig.ServiceConfig.Name
				ch := make(chan string)
				workerChans[name] = ch
				wg.Add(1)
				go restartContainers(name, ch)
				ch <- "deploy"

				log.Printf("Started new worker for %s\n", name)
				continue
			}

			if changedConfig.Restart {
				log.Printf("Restarting %s", changedConfig.ServiceConfig.Name)
				ch <- "restart"
			} else {
				ch <- "deploy"
			}
		}
	}

}

func main() {
	flag.Int64Var(&stopCutoff, "cutoff", 10, "Seconds to wait before stopping old containers")
	flag.StringVar(&redisHost, "redis", utils.GetEnv("GALAXY_REDIS_HOST", utils.DefaultRedisHost), "redis host")
	flag.StringVar(&env, "env", utils.GetEnv("GALAXY_ENV", ""), "Environment namespace")
	flag.StringVar(&pool, "pool", utils.GetEnv("GALAXY_POOL", ""), "Pool namespace")
	flag.StringVar(&shuttleHost, "shuttleAddr", "", "IP where containers can reach shuttle proxy. Defaults to docker0 IP.")
	flag.StringVar(&statsdHost, "statsdAddr", utils.GetEnv("GALAXY_STATSD_HOST", ""), "IP where containers can reach a statsd service. Defaults to docker0 IP:8125.")
	flag.BoolVar(&debug, "debug", false, "verbose logging")
	flag.BoolVar(&version, "v", false, "display version info")

	flag.Parse()

	if version {
		fmt.Println(buildVersion)
		return
	}

	if strings.TrimSpace(env) == "" {
		fmt.Println("Need an env")
		flag.PrintDefaults()
		os.Exit(1)
	}

	if strings.TrimSpace(pool) == "" {
		fmt.Println("Need a pool")
		flag.PrintDefaults()
		os.Exit(1)
	}

	if debug {
		log.DefaultLogger.Level = log.DEBUG
	}

	if flag.NArg() < 1 {
		fmt.Println("Need a command")
		flag.PrintDefaults()
		os.Exit(1)
	}

	initOrDie()
	serviceRegistry.CreatePool(pool, env)

	switch flag.Args()[0] {
	case "agent":
		loop = true
	case "start":
		if flag.NArg() >= 2 {
			apps = flag.Args()[1:]
		}
		break
	case "stop":
		if flag.NArg() >= 2 {
			apps = flag.Args()[1:]
			for _, app := range apps {
				err := serviceRuntime.StopAllMatching(app)
				if err != nil {
					log.Fatalf("ERROR: Unable able to stop all containers: %s", err)
				}
			}
			return
		}

		err := serviceRuntime.StopAll(env)
		if err != nil {
			log.Fatalf("ERROR: Unable able to stop all containers: %s", err)
		}
		return
	}

	log.Printf("Starting commander %s", buildVersion)
	log.Printf("Using env = %s, pool = %s",
		env, pool)

	for app, ch := range workerChans {
		if len(apps) == 0 || utils.StringInSlice(app, apps) {
			wg.Add(1)
			go restartContainers(app, ch)
			ch <- "deploy"
		}
	}

	if loop {

		cancelChan := make(chan struct{})
		// do we need to cancel ever?

		restartChan := serviceRegistry.Watch(env, cancelChan)
		monitorService(restartChan)
	}

	wg.Wait()
}
<commit_msg>commander: Add command-line usage<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"strings"
	"sync"
	"time"

	docker "github.com/fsouza/go-dockerclient"
	"github.com/litl/galaxy/log"
	"github.com/litl/galaxy/registry"
	"github.com/litl/galaxy/runtime"
	"github.com/litl/galaxy/utils"
)

var (
	stopCutoff      int64
	apps            []string
	redisHost       string
	env             string
	pool            string
	loop            bool
	shuttleHost     string
	statsdHost      string
	debug           bool
	runOnce         bool
	version         bool
	buildVersion    string
	serviceConfigs  []*registry.ServiceConfig
	serviceRegistry *registry.ServiceRegistry
	serviceRuntime  *runtime.ServiceRuntime
	workerChans     map[string]chan string
	wg              sync.WaitGroup
)

func initOrDie() {

	serviceRegistry = registry.NewServiceRegistry(
		"",
		registry.DefaultTTL,
		"",
	)

	serviceRegistry.Connect(redisHost)
	serviceRuntime = runtime.NewServiceRuntime(serviceRegistry, shuttleHost, statsdHost)

	apps, err := serviceRegistry.ListAssignments(env, pool)
	if err != nil {
		log.Fatalf("ERROR: Could not retrieve service configs for /%s/%s: %s\n", env, pool, err)
	}

	workerChans = make(map[string]chan string)
	for _, app := range apps {
		serviceConfig, err := serviceRegistry.GetServiceConfig(app, env)
		if err != nil {
			log.Fatalf("ERROR: Could not retrieve service config for /%s/%s: %s\n", env, pool, err)
		}

		workerChans[serviceConfig.Name] = make(chan string)
	}
}

func pullImageAsync(serviceConfig registry.ServiceConfig, errChan chan error) {
	// err logged via pullImage
	_, err := pullImage(&serviceConfig)
	if err != nil {
		errChan <- err
		return
	}
	errChan <- nil
}

func pullImage(serviceConfig *registry.ServiceConfig) (*docker.Image, error) {

	image, err := serviceRuntime.InspectImage(serviceConfig.Version())
	if image != nil && image.ID == serviceConfig.VersionID() || serviceConfig.VersionID() == "" {
		return image, nil
	}

	log.Printf("Pulling %s version %s\n", serviceConfig.Name, serviceConfig.Version())
	image, err = serviceRuntime.PullImage(serviceConfig.Version(),
		serviceConfig.VersionID(), true)
	if image == nil || err != nil {
		log.Errorf("ERROR: Could not pull image %s: %s",
			serviceConfig.Version(), err)
		return nil, err
	}

	if image.ID != serviceConfig.VersionID() && len(serviceConfig.VersionID()) > 12 {
		log.Errorf("ERROR: Pulled image for %s does not match expected ID. Expected: %s: Got: %s",
			serviceConfig.Version(),
			image.ID[0:12], serviceConfig.VersionID()[0:12])
		return nil, errors.New(fmt.Sprintf("failed to pull image ID %s", serviceConfig.VersionID()[0:12]))
	}

	log.Printf("Pulled %s\n", serviceConfig.Version())
	return image, nil
}

func startService(serviceConfig *registry.ServiceConfig, logStatus bool) {
	started, container, err := serviceRuntime.StartIfNotRunning(env, serviceConfig)
	if err != nil {
		log.Errorf("ERROR: Could not start container for %s: %s", serviceConfig.Version(), err)
		return
	}

	if started {
		log.Printf("Started %s version %s as %s\n", serviceConfig.Name, serviceConfig.Version(), container.ID[0:12])
	}

	if logStatus && !debug {
		log.Printf("%s version %s running as %s\n", serviceConfig.Name, serviceConfig.Version(), container.ID[0:12])
	}

	log.Debugf("%s version %s running as %s\n", serviceConfig.Name, serviceConfig.Version(), container.ID[0:12])

	err = serviceRuntime.StopAllButLatestService(serviceConfig, stopCutoff)
	if err != nil {
		log.Errorf("ERROR: Could not stop containers: %s", err)
	}
}

func appAssigned(app string) (bool, error) {
	assignments, err := serviceRegistry.ListAssignments(env, pool)
	if err != nil {
		return false, err
	}

	if !utils.StringInSlice(app, assignments) {
		return false, nil
	}
	return true, nil
}

func restartContainers(app string, cmdChan chan string) {
	defer wg.Done()
	logOnce := true

	ticker := time.NewTicker(10 * time.Second)

	for {

		select {

		case cmd := <-cmdChan:

			assigned, err := appAssigned(app)
			if err != nil {
				log.Errorf("ERROR: Error retrieving assignments for %s: %s", app, err)
				if !loop {
					return
				}
				continue
			}

			if !assigned {
				continue
			}

			serviceConfig, err := serviceRegistry.GetServiceConfig(app, env)
			if err != nil {
				log.Errorf("ERROR: Error retrieving service config for %s: %s", app, err)
				if !loop {
					return
				}
				continue
			}

			if serviceConfig.Version() == "" {
				if !loop {
					return
				}
				continue
			}

			if cmd == "deploy" {
				_, err = pullImage(serviceConfig)
				if err != nil {
					if !loop {
						return
					}
					continue
				}
				startService(serviceConfig, logOnce)
			}

			if cmd == "restart" {
				err := serviceRuntime.Stop(serviceConfig)
				if err != nil {
					log.Errorf("ERROR: Could not stop %s: %s",
						serviceConfig.Version(), err)
					if !loop {
						return
					}

					startService(serviceConfig, logOnce)
					continue
				}
			}

			logOnce = false
		case <-ticker.C:

			serviceConfig, err := serviceRegistry.GetServiceConfig(app, env)
			if err != nil {
				log.Errorf("ERROR: Error retrieving service config for %s: %s", app, err)
				continue
			}

			assigned, err := appAssigned(app)
			if err != nil {
				log.Errorf("ERROR: Error retrieving service config for %s: %s", app, err)
				if !loop {
					return
				}

				continue
			}

			if serviceConfig == nil || !assigned {
				log.Errorf("%s no longer exists.  Stopping worker.", app)
				serviceRuntime.StopAllMatching(app)
				delete(workerChans, app)
				return
			}

			if serviceConfig.Version() == "" {
				continue
			}

			_, err = pullImage(serviceConfig)
			if err != nil {
				if !loop {
					return
				}
				log.Errorf("ERROR: Could not pull images: %s", err)
				continue
			}

			started, container, err := serviceRuntime.StartIfNotRunning(env, serviceConfig)
			if err != nil {
				log.Errorf("ERROR: Could not start containers: %s", err)
				continue
			}

			if started {
				log.Printf("Started %s version %s as %s\n", serviceConfig.Name, serviceConfig.Version(), container.ID[0:12])
			}

			log.Debugf("%s version %s running as %s\n", serviceConfig.Name, serviceConfig.Version(), container.ID[0:12])

			err = serviceRuntime.StopAllButCurrentVersion(serviceConfig)
			if err != nil {
				log.Errorf("ERROR: Could not stop containers: %s", err)
			}
		}

		if !loop {
			return
		}

	}
}

func monitorService(changedConfigs chan *registry.ConfigChange) {

	for {

		var changedConfig *registry.ConfigChange
		select {

		case changedConfig = <-changedConfigs:

			if changedConfig.Error != nil {
				log.Errorf("ERROR: Error watching changes: %s", changedConfig.Error)
				continue
			}

			if changedConfig.ServiceConfig == nil {
				continue
			}

			assigned, err := appAssigned(changedConfig.ServiceConfig.Name)
			if err != nil {
				log.Errorf("ERROR: Error retrieving service config for %s: %s", changedConfig.ServiceConfig.Name, err)
				if !loop {
					return
				}
				continue
			}

			if !assigned {
				continue
			}

			ch, ok := workerChans[changedConfig.ServiceConfig.Name]
			if !ok {
				name := changedConfig.ServiceConfig.Name
				ch := make(chan string)
				workerChans[name] = ch
				wg.Add(1)
				go restartContainers(name, ch)
				ch <- "deploy"

				log.Printf("Started new worker for %s\n", name)
				continue
			}

			if changedConfig.Restart {
				log.Printf("Restarting %s", changedConfig.ServiceConfig.Name)
				ch <- "restart"
			} else {
				ch <- "deploy"
			}
		}
	}

}

func main() {
	flag.Int64Var(&stopCutoff, "cutoff", 10, "Seconds to wait before stopping old containers")
	flag.StringVar(&redisHost, "redis", utils.GetEnv("GALAXY_REDIS_HOST", utils.DefaultRedisHost), "redis host")
	flag.StringVar(&env, "env", utils.GetEnv("GALAXY_ENV", ""), "Environment namespace")
	flag.StringVar(&pool, "pool", utils.GetEnv("GALAXY_POOL", ""), "Pool namespace")
	flag.StringVar(&shuttleHost, "shuttleAddr", "", "IP where containers can reach shuttle proxy. Defaults to docker0 IP.")
	flag.StringVar(&statsdHost, "statsdAddr", utils.GetEnv("GALAXY_STATSD_HOST", ""), "IP where containers can reach a statsd service. Defaults to docker0 IP:8125.")
	flag.BoolVar(&debug, "debug", false, "verbose logging")
	flag.BoolVar(&version, "v", false, "display version info")

	flag.Usage = func() {
		println("Usage: commander [options] <command> [<args>]\n")
		println("Available commands are:")
		println("   agent           Runs commander agent")
		println("   start           Starts one or more apps")
		println("   stop            Stops one or more apps")
		println("\nOptions:\n")
		flag.PrintDefaults()

	}

	flag.Parse()

	if version {
		fmt.Println(buildVersion)
		return
	}

	if strings.TrimSpace(env) == "" {
		fmt.Println("Need an env")
		flag.PrintDefaults()
		os.Exit(1)
	}

	if strings.TrimSpace(pool) == "" {
		fmt.Println("Need a pool")
		flag.PrintDefaults()
		os.Exit(1)
	}

	if debug {
		log.DefaultLogger.Level = log.DEBUG
	}

	if flag.NArg() < 1 {
		fmt.Println("Need a command")
		flag.PrintDefaults()
		os.Exit(1)
	}

	initOrDie()
	serviceRegistry.CreatePool(pool, env)

	switch flag.Args()[0] {
	case "agent":
		loop = true
		agentFs := flag.NewFlagSet("agent", flag.ExitOnError)
		agentFs.Usage = func() {
			println("Usage: commander agent [options]\n")
			println("    Runs commander continuously\n\n")
			println("Options:\n\n")
			agentFs.PrintDefaults()
		}
		agentFs.Parse(flag.Args()[1:])
	case "start":

		startFs := flag.NewFlagSet("start", flag.ExitOnError)
		startFs.Usage = func() {
			println("Usage: commander start [options] [<app>]*\n")
			println("    Starts one or more apps. If no apps are specified, starts all apps.\n\n")
			println("Options:\n\n")
			startFs.PrintDefaults()
		}
		startFs.Parse(flag.Args()[1:])

		apps = startFs.Args()

		break
	case "stop":
		stopFs := flag.NewFlagSet("stop", flag.ExitOnError)
		stopFs.Usage = func() {
			println("Usage: commander stop [options] [<app>]*\n")
			println("    Stops one or more apps. If no apps are specified, stops all apps.\n\n")
			println("Options:\n\n")
			stopFs.PrintDefaults()
		}
		stopFs.Parse(flag.Args()[1:])

		apps = stopFs.Args()

		for _, app := range apps {
			err := serviceRuntime.StopAllMatching(app)
			if err != nil {
				log.Fatalf("ERROR: Unable able to stop all containers: %s", err)
			}
		}
		if len(apps) > 0 {
			return
		}

		err := serviceRuntime.StopAll(env)
		if err != nil {
			log.Fatalf("ERROR: Unable able to stop all containers: %s", err)
		}
		return
	}

	log.Printf("Starting commander %s", buildVersion)
	log.Printf("Using env = %s, pool = %s",
		env, pool)

	for app, ch := range workerChans {
		if len(apps) == 0 || utils.StringInSlice(app, apps) {
			wg.Add(1)
			go restartContainers(app, ch)
			ch <- "deploy"
		}
	}

	if loop {

		cancelChan := make(chan struct{})
		// do we need to cancel ever?

		restartChan := serviceRegistry.Watch(env, cancelChan)
		monitorService(restartChan)
	}

	wg.Wait()
}
<|endoftext|>


<commit_before>package fixity

import (
	"encoding/json"
	"io"
	"time"

	"github.com/leeola/fixity/q"
)

// Fixity implements writing, indexing and reading with a Fixity store.
//
// This interface will be implemented for multiple stores, such as a local on
// disk store and a remote over network store.
type Fixity interface {
	// Blob returns a raw blob of the given hash.
	//
	// Mainly useful for inspecting the underlying data structure.
	Blob(hash string) ([]byte, error)

	// ReadHash unmarshals the given hash contents into a Version.
	//
	// Included in the Version is the Json and MultiBlob, if any exist. If no
	// Json exists the Json struct will be zero value, and if no MultiBlob
	// exists the ReadCloser will be nil.
	//
	// ReadHash will return ErrNotVersion if the given hash is not a valid hash.
	ReadHash(hash string) (Version, error)

	// ReadId unmarshals the given id into a Version struct.
	//
	// Included in the Version is the Json and MultiBlob, if any exist. If no
	// Json exists the Json struct will be zero value, and if no MultiBlob
	// exists the ReadCloser will be nil.
	ReadId(id string) (Version, error)

	// Search for documents matching the given query.
	Search(*q.Query) ([]string, error)

	// Write the given Commit, MultiJson, and Reader to the Fixity store.
	//
	// A single write can support an arbitrary number of Json documents
	// via the MultiJson map. The reasoning behind this is documented in
	// the MultiJson docstring.
	Write(Commit, MultiJson, io.Reader) ([]string, error)

	// TODO(leeola): Enable a close method to shutdown any
	//
	// // Close shuts down any connections that may need to be closed.
	// Close() error
}

// Commit is a higher level Version, allowing simple and high level writes.
// Commit is metadata about the Json/Bytes being written.
// Commit contains ordering and mutation info for the data being written.
//
// Eg, the Id to group writes together, the PreviousVersionHash to load
// mutations and/or order, and the CreatedAt  to represent timed order.
//
// Most fields are optional, depending on the Fixity and Index implementations.
type Commit struct {
	// Id is a unique string which allows Versions to be linked.
	//
	// Since Fixity is immutable, Versions allow a single piece of data to be
	// mutated over time and history. Each version represents a single state
	// of mutation for the given Json and Blob hash. The Id, allows each
	// version of, say, a single File or Wiki page to have the same identifier
	// and represent the same item.
	//
	// Ids can be random or contain meaning, the usage is entirely up to the
	// user.
	Id string `json:"id,omitempty"`

	// UploadedAt is used to track the Version over time, and sort the most recent.
	//
	// This is important, as many versions of a single id have to be sorted somehow.
	// Sorting them by PreviousVersionCount and PreviousVersionHash is possible,
	// but that leads itself to conflicts which then have to be resolved, merged,
	// etc.
	//
	// Sorting by time allows for automatic resolution of any conflict, and is
	// the most hands-free method of conflict resolution. Not guaranteed to be
	// correct, but guaranteed to be easy.
	UploadedAt *time.Time `json:"uploadedAt,omitempty"`

	// PreviousVersionHash stores the Version preceeding this Version, if any.
	//
	// This not only provides a historical record of each mutation, but it can
	// help identify version forks. A fork in this case, is when multiple
	// writers write based off of the same PreviousVersionHash. Since Fixity
	// stores data by content address, forks and "conflicts" are not
	// problematic, but can cause confusion to the actual writer of the data.
	PreviousVersionHash string `json:"previousVersion,omitempty"`

	// ChangeLog is a simple human friendly message about this Version.
	ChangeLog string `json:"changeLog,omitempty"`
}

// MultiJson is a JsonWithMetas map, keyed for unordered unmarshalling.
//
// MultiJson differs from MultiJsonHash in that MultiJson is supplied by
// users, and contains the JsonBytes. MultiJsonHash is stored within
// the Fixity.Store, and does *not* contain the JsonBytes. The Bytes are
// stored separately, as to separate the Meta from the actual Content.
//
// MultiJson and MultiJsonHash allow a writer to store multiple json structs
// together, within a single Commit.
//
// A single Commit Write can support an arbitrary number of Json documents via
// the MultiJson map. Each Json value within the JsonWithMeta is stored as
// it's own content address.
//
// This allows the caller to optimize how the data is stored. Ensuring that
// frequently changing data is not stored with infrequently changing data,
// effectively manually deduplicating the json.
//
// This method of deduplication, vs rolling checksums as seen in Blobs,
// is chosen because the caller of Write is able to effectively choose
// the rolling splits by seperating Json out into separate objects.
// Furthermore, for rolling checksums to be effective with smaller documents
// the rolling algorithm would need to chunk at very small intervals,
// introducing a lot of extra documents in the store with little gain.
//
// Finally, and most importantly, storing Json as chunked bytes would cause
// the json to effectively be encoded. No longer is the content "just json",
// but rather you need to join bytes together to construct your actual data,
// as is the case with binary blobs. Blobs don't have a choice on this, as
// Binary isn't Json, but Json does. Keeping the storage model easy to reason
// about and easy to migrate away from, analyze with external tools, etc,
// is a core philosophy of Fixity.
type MultiJson map[string]JsonWithMeta

// JsonWithMeta stores the bytes and meta of a Json struct.
type JsonWithMeta struct {
	Json

	// JsonMeta stores information about the raw Json being stored.
	//
	// This is primarily used to provide insights on how to index and unmarshal
	// the Json struct.
	//
	// See JsonMeta docstring for further details.
	JsonMeta *JsonMeta `json:"jsonMeta,omitempty"`
}

// Json is a struct which stores text data in Json form.
//
// This data is often indexed, and is the method by which Blob data stores
// and indexes metadata about that blob data. It does not require or imply
// that blob data exists with the given Json, as the Json may be the primary
// data being stored. As is the case with a Wiki, etc.
type Json struct {
	// JsonBytes is the actual json data being stored.
	JsonBytes json.RawMessage `json:"json"`
}

// JsonMeta stores information about the raw Json being stored.
//
// This serves, for example, to ensure that if the Index is rebuilt,
// it always knows which fields of the Json data need to be indexed.
// As well as mappings for json fields, etc.
//
// Without Metadata about the Json data, Json data would become a black box
// with no information to help Fixity rebuild indexes and etc.
type JsonMeta struct {
	// IndexFields are the fields of the Json data to be indexed.
	//
	// These can include the value if the indexer cannot assert the real
	// value to be indexed from the Json.Json []byte slice.
	IndexedFields Fields `json:"indexedFields"`
}
<commit_msg>docs: removed mistaken doc lines<commit_after>package fixity

import (
	"encoding/json"
	"io"
	"time"

	"github.com/leeola/fixity/q"
)

// Fixity implements writing, indexing and reading with a Fixity store.
//
// This interface will be implemented for multiple stores, such as a local on
// disk store and a remote over network store.
type Fixity interface {
	// Blob returns a raw blob of the given hash.
	//
	// Mainly useful for inspecting the underlying data structure.
	Blob(hash string) ([]byte, error)

	// ReadHash unmarshals the given hash contents into a Version.
	//
	// Included in the Version is the Json and MultiBlob, if any exist. If no
	// Json exists the Json struct will be zero value, and if no MultiBlob
	// exists the ReadCloser will be nil.
	//
	// ReadHash will return ErrNotVersion if the given hash is not a valid hash.
	ReadHash(hash string) (Version, error)

	// ReadId unmarshals the given id into a Version struct.
	//
	// Included in the Version is the Json and MultiBlob, if any exist. If no
	// Json exists the Json struct will be zero value, and if no MultiBlob
	// exists the ReadCloser will be nil.
	ReadId(id string) (Version, error)

	// Search for documents matching the given query.
	Search(*q.Query) ([]string, error)

	// Write the given Commit, MultiJson, and Reader to the Fixity store.
	//
	// A single write can support an arbitrary number of Json documents
	// via the MultiJson map. The reasoning behind this is documented in
	// the MultiJson docstring.
	Write(Commit, MultiJson, io.Reader) ([]string, error)

	// TODO(leeola): Enable a close method to shutdown any
	//
	// // Close shuts down any connections that may need to be closed.
	// Close() error
}

// Commit contains ordering and mutation info for the data being written.
//
// Eg, the Id to group writes together, the PreviousVersionHash to load
// mutations and/or order, and the CreatedAt  to represent timed order.
//
// Most fields are optional, depending on the Fixity and Index implementations.
type Commit struct {
	// Id is a unique string which allows Versions to be linked.
	//
	// Since Fixity is immutable, Versions allow a single piece of data to be
	// mutated over time and history. Each version represents a single state
	// of mutation for the given Json and Blob hash. The Id, allows each
	// version of, say, a single File or Wiki page to have the same identifier
	// and represent the same item.
	//
	// Ids can be random or contain meaning, the usage is entirely up to the
	// user.
	Id string `json:"id,omitempty"`

	// UploadedAt is used to track the Version over time, and sort the most recent.
	//
	// This is important, as many versions of a single id have to be sorted somehow.
	// Sorting them by PreviousVersionCount and PreviousVersionHash is possible,
	// but that leads itself to conflicts which then have to be resolved, merged,
	// etc.
	//
	// Sorting by time allows for automatic resolution of any conflict, and is
	// the most hands-free method of conflict resolution. Not guaranteed to be
	// correct, but guaranteed to be easy.
	UploadedAt *time.Time `json:"uploadedAt,omitempty"`

	// PreviousVersionHash stores the Version preceeding this Version, if any.
	//
	// This not only provides a historical record of each mutation, but it can
	// help identify version forks. A fork in this case, is when multiple
	// writers write based off of the same PreviousVersionHash. Since Fixity
	// stores data by content address, forks and "conflicts" are not
	// problematic, but can cause confusion to the actual writer of the data.
	PreviousVersionHash string `json:"previousVersion,omitempty"`

	// ChangeLog is a simple human friendly message about this Version.
	ChangeLog string `json:"changeLog,omitempty"`
}

// MultiJson is a JsonWithMetas map, keyed for unordered unmarshalling.
//
// MultiJson differs from MultiJsonHash in that MultiJson is supplied by
// users, and contains the JsonBytes. MultiJsonHash is stored within
// the Fixity.Store, and does *not* contain the JsonBytes. The Bytes are
// stored separately, as to separate the Meta from the actual Content.
//
// MultiJson and MultiJsonHash allow a writer to store multiple json structs
// together, within a single Commit.
//
// A single Commit Write can support an arbitrary number of Json documents via
// the MultiJson map. Each Json value within the JsonWithMeta is stored as
// it's own content address.
//
// This allows the caller to optimize how the data is stored. Ensuring that
// frequently changing data is not stored with infrequently changing data,
// effectively manually deduplicating the json.
//
// This method of deduplication, vs rolling checksums as seen in Blobs,
// is chosen because the caller of Write is able to effectively choose
// the rolling splits by seperating Json out into separate objects.
// Furthermore, for rolling checksums to be effective with smaller documents
// the rolling algorithm would need to chunk at very small intervals,
// introducing a lot of extra documents in the store with little gain.
//
// Finally, and most importantly, storing Json as chunked bytes would cause
// the json to effectively be encoded. No longer is the content "just json",
// but rather you need to join bytes together to construct your actual data,
// as is the case with binary blobs. Blobs don't have a choice on this, as
// Binary isn't Json, but Json does. Keeping the storage model easy to reason
// about and easy to migrate away from, analyze with external tools, etc,
// is a core philosophy of Fixity.
type MultiJson map[string]JsonWithMeta

// JsonWithMeta stores the bytes and meta of a Json struct.
type JsonWithMeta struct {
	Json

	// JsonMeta stores information about the raw Json being stored.
	//
	// This is primarily used to provide insights on how to index and unmarshal
	// the Json struct.
	//
	// See JsonMeta docstring for further details.
	JsonMeta *JsonMeta `json:"jsonMeta,omitempty"`
}

// Json is a struct which stores text data in Json form.
//
// This data is often indexed, and is the method by which Blob data stores
// and indexes metadata about that blob data. It does not require or imply
// that blob data exists with the given Json, as the Json may be the primary
// data being stored. As is the case with a Wiki, etc.
type Json struct {
	// JsonBytes is the actual json data being stored.
	JsonBytes json.RawMessage `json:"json"`
}

// JsonMeta stores information about the raw Json being stored.
//
// This serves, for example, to ensure that if the Index is rebuilt,
// it always knows which fields of the Json data need to be indexed.
// As well as mappings for json fields, etc.
//
// Without Metadata about the Json data, Json data would become a black box
// with no information to help Fixity rebuild indexes and etc.
type JsonMeta struct {
	// IndexFields are the fields of the Json data to be indexed.
	//
	// These can include the value if the indexer cannot assert the real
	// value to be indexed from the Json.Json []byte slice.
	IndexedFields Fields `json:"indexedFields"`
}
<|endoftext|>


<commit_before>package ics

import (
	"io"
	"unicode/utf8"
)

type folder struct {
	w    io.Writer
	err  error
	line uint8
}

const maxLineLength = 75

var eol = [...]byte{'\r', '\n', ' '}

func (f *folder) Write(q []byte) (int, error) {
	if f.err != nil {
		return 0, f.err
	}
	var (
		r       rune
		s, n, m int
	)
	for pos := 0; pos < len(q); pos += s {
		r, s = utf8.DecodeRune(q[pos:])
		f.line += uint8(s)
		if r == '\n' {
			f.line = 0
		} else if r == '\r' {
		} else if f.line > maxLineLength {
			if pos > 0 {
				m, f.err = f.w.Write(q[:pos])
				n += m
				if f.err != nil {
					return n, f.err
				}
				q = q[pos:]
			}
			_, f.err = f.w.Write(eol[:])
			if f.err != nil {
				return n, f.err
			}

			pos = 0
			f.line = uint8(s)
		}
	}
	if len(q) > 0 {
		m, f.err = f.w.Write(q)
		n += m
	}
	return n, f.err
}

func (f *folder) WriteString(q string) (int, error) {
	return f.Write([]byte(q))
}
<commit_msg>folder incorrectly ignores space at beginning of continuation lines when counting line length<commit_after>package ics

import (
	"io"
	"unicode/utf8"
)

type folder struct {
	w    io.Writer
	err  error
	line uint8
}

const maxLineLength = 75

var eol = [...]byte{'\r', '\n', ' '}

func (f *folder) Write(q []byte) (int, error) {
	if f.err != nil {
		return 0, f.err
	}
	var (
		r       rune
		s, n, m int
	)
	for pos := 0; pos < len(q); pos += s {
		r, s = utf8.DecodeRune(q[pos:])
		f.line += uint8(s)
		if r == '\n' {
			f.line = 0
		} else if r == '\r' {
		} else if f.line > maxLineLength {
			if pos > 0 {
				m, f.err = f.w.Write(q[:pos])
				n += m
				if f.err != nil {
					return n, f.err
				}
				q = q[pos:]
			}
			_, f.err = f.w.Write(eol[:])
			if f.err != nil {
				return n, f.err
			}

			pos = 0
			f.line = uint8(s) + 1
		}
	}
	if len(q) > 0 {
		m, f.err = f.w.Write(q)
		n += m
	}
	return n, f.err
}

func (f *folder) WriteString(q string) (int, error) {
	return f.Write([]byte(q))
}
<|endoftext|>


<commit_before>package commands

import (
	"strings"
	"testing"
)

// TestStacks - tests stack type and methods
func TestStack(t *testing.T) {

	teststack := stack{
		name: "sqs",
	}

	// Define sources
	testConfigSrc := `s3://daidokoro-dev/qaz/test/config.yml`
	testTemplateSrc := `s3://daidokoro-dev/qaz/test/sqs.yml`

	// Get Config
	if err := configReader(testConfigSrc); err != nil {
		t.Error(err)
	}

	// create session
	sess, err := awsSession()
	if err != nil {
		t.Error(err)
	}

	// Set stack name
	teststack.setStackName()
	if teststack.stackname != "github-release-sqs" {
		t.Errorf("StackName Failed, Expected: github-release-sqs, Received: %s", teststack.stackname)
	}

	// Get Stack template - test s3Read
	teststack.template, err = genTimeParser(testTemplateSrc)
	if err != nil {
		t.Error(err)
	}

	// Test Stack status method
	if err := teststack.status(sess); err != nil {
		t.Error(err)
	}

	// Test Stack output method
	if err := teststack.outputs(sess); err != nil {
		t.Error(err)
	}

	// Test Stack output length
	if len(teststack.output.Stacks) < 1 {
		t.Errorf("Expected Output Length to be greater than 0: Got: %s", teststack.output.Stacks)
	}

	// Test Check/Validate template
	if err := teststack.check(sess); err != nil {
		t.Error(err, "\n", teststack.template)
	}

	// Test State method
	if _, err := teststack.state(sess); err != nil {
		t.Error(err)
	}

	// Test stackExists method
	if ok := teststack.stackExists(sess); !ok {
		t.Error("Expected True for StackExists but got:", ok)
	}

	// Test UpdateStack
	teststack.template = strings.Replace(teststack.template, "MySecret", "Secret", -1)
	if err := teststack.update(sess); err != nil {
		t.Error(err)
	}

	// Test ChangeSets
	teststack.template = strings.Replace(teststack.template, "Secret", "MySecret", -1)
	job.changeName = "gotest"

	for _, c := range []string{"create", "list", "desc", "execute"} {
		if err := teststack.change(sess, c); err != nil {
			t.Error(err)
		}
	}

}

// TestDeploy - test deploy and terminate stack.
func TestDeploy(t *testing.T) {
	teststack := stack{
		name: "vpc",
	}

	// Define sources
	deployTemplateSrc := `https://raw.githubusercontent.com/daidokoro/qaz/master/examples/vpc/templates/vpc.yml`
	deployConfSource := `https://raw.githubusercontent.com/daidokoro/qaz/master/examples/vpc/config.yml`

	// Get Config
	if err := configReader(deployConfSource); err != nil {
		t.Error(err)
	}

	// create session
	sess, err := awsSession()
	if err != nil {
		t.Error(err)
	}

	teststack.setStackName()

	// Get Stack template - test s3Read
	teststack.template, err = genTimeParser(deployTemplateSrc)
	if err != nil {
		t.Error(err)
	}

	// Test Deploy Stack
	if err := teststack.deploy(sess); err != nil {
		t.Error(err)
	}

	// Test Set Stack Policy
	// if err := teststack.stackPolicy(sess); err != nil {
	// 	t.Errorf("%s - [%s]", err, teststack.policy)
	// }

	// Test Terminate Stack
	if err := teststack.terminate(sess); err != nil {
		t.Error(err)
	}
}
<commit_msg>added debug flag to tests<commit_after>package commands

import (
	"strings"
	"testing"
)

// TestStacks - tests stack type and methods
func TestStack(t *testing.T) {

	teststack := stack{
		name: "sqs",
	}

	// Define sources
	testConfigSrc := `s3://daidokoro-dev/qaz/test/config.yml`
	testTemplateSrc := `s3://daidokoro-dev/qaz/test/sqs.yml`

	// Get Config
	if err := configReader(testConfigSrc); err != nil {
		t.Error(err)
	}

	// create session
	sess, err := awsSession()
	if err != nil {
		t.Error(err)
	}

	// Set stack name
	teststack.setStackName()
	if teststack.stackname != "github-release-sqs" {
		t.Errorf("StackName Failed, Expected: github-release-sqs, Received: %s", teststack.stackname)
	}

	// Get Stack template - test s3Read
	teststack.template, err = genTimeParser(testTemplateSrc)
	if err != nil {
		t.Error(err)
	}

	// Test Stack status method
	if err := teststack.status(sess); err != nil {
		t.Error(err)
	}

	// Test Stack output method
	if err := teststack.outputs(sess); err != nil {
		t.Error(err)
	}

	// Test Stack output length
	if len(teststack.output.Stacks) < 1 {
		t.Errorf("Expected Output Length to be greater than 0: Got: %s", teststack.output.Stacks)
	}

	// Test Check/Validate template
	if err := teststack.check(sess); err != nil {
		t.Error(err, "\n", teststack.template)
	}

	// Test State method
	if _, err := teststack.state(sess); err != nil {
		t.Error(err)
	}

	// Test stackExists method
	if ok := teststack.stackExists(sess); !ok {
		t.Error("Expected True for StackExists but got:", ok)
	}

	// Test UpdateStack
	teststack.template = strings.Replace(teststack.template, "MySecret", "Secret", -1)
	if err := teststack.update(sess); err != nil {
		t.Error(err)
	}

	// Test ChangeSets
	teststack.template = strings.Replace(teststack.template, "Secret", "MySecret", -1)
	job.changeName = "gotest"

	for _, c := range []string{"create", "list", "desc", "execute"} {
		if err := teststack.change(sess, c); err != nil {
			t.Error(err)
		}
	}

	return

}

// TestDeploy - test deploy and terminate stack.
func TestDeploy(t *testing.T) {
	job.debug = true
	teststack := stack{
		name: "vpc",
	}

	// Define sources
	deployTemplateSrc := `https://raw.githubusercontent.com/daidokoro/qaz/master/examples/vpc/templates/vpc.yml`
	deployConfSource := `https://raw.githubusercontent.com/daidokoro/qaz/master/examples/vpc/config.yml`

	// Get Config
	if err := configReader(deployConfSource); err != nil {
		t.Error(err)
	}

	// create session
	sess, err := awsSession()
	if err != nil {
		t.Error(err)
	}

	teststack.setStackName()

	// Get Stack template - test s3Read
	teststack.template, err = genTimeParser(deployTemplateSrc)
	if err != nil {
		t.Error(err)
	}

	// Test Deploy Stack
	if err := teststack.deploy(sess); err != nil {
		t.Error(err)
	}

	// Test Set Stack Policy
	if err := teststack.stackPolicy(sess); err != nil {
		t.Errorf("%s - [%s]", err, teststack.policy)
	}

	// Test Terminate Stack
	if err := teststack.terminate(sess); err != nil {
		t.Error(err)
	}

	return
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file implements the host side of CGI (being the webserver
// parent process).

// Package cgi implements CGI (Common Gateway Interface) as specified
// in RFC 3875.
//
// Note that using CGI means starting a new process to handle each
// request, which is typically less efficient than using a
// long-running server.  This package is intended primarily for
// compatibility with existing systems.
package cgi

import (
	"bufio"
	"exec"
	"fmt"
	"http"
	"io"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
)

var trailingPort = regexp.MustCompile(`:([0-9]+)$`)

var osDefaultInheritEnv = map[string][]string{
	"darwin":  {"DYLD_LIBRARY_PATH"},
	"freebsd": {"LD_LIBRARY_PATH"},
	"hpux":    {"LD_LIBRARY_PATH", "SHLIB_PATH"},
	"irix":    {"LD_LIBRARY_PATH", "LD_LIBRARYN32_PATH", "LD_LIBRARY64_PATH"},
	"linux":   {"LD_LIBRARY_PATH"},
	"solaris": {"LD_LIBRARY_PATH", "LD_LIBRARY_PATH_32", "LD_LIBRARY_PATH_64"},
	"windows": {"SystemRoot", "COMSPEC", "PATHEXT", "WINDIR"},
}

// Handler runs an executable in a subprocess with a CGI environment.
type Handler struct {
	Path string // path to the CGI executable
	Root string // root URI prefix of handler or empty for "/"

	// Dir specifies the CGI executable's working directory.
	// If Dir is empty, the base directory of Path is used.
	// If Path has no base directory, the current working
	// directory is used.
	Dir string

	Env        []string    // extra environment variables to set, if any, as "key=value"
	InheritEnv []string    // environment variables to inherit from host, as "key"
	Logger     *log.Logger // optional log for errors or nil to use log.Print
	Args       []string    // optional arguments to pass to child process

	// PathLocationHandler specifies the root http Handler that
	// should handle internal redirects when the CGI process
	// returns a Location header value starting with a "/", as
	// specified in RFC 3875 § 6.3.2. This will likely be
	// http.DefaultServeMux.
	//
	// If nil, a CGI response with a local URI path is instead sent
	// back to the client and not redirected internally.
	PathLocationHandler http.Handler
}

func (h *Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	root := h.Root
	if root == "" {
		root = "/"
	}

	if len(req.TransferEncoding) > 0 && req.TransferEncoding[0] == "chunked" {
		rw.WriteHeader(http.StatusBadRequest)
		rw.Write([]byte("Chunked request bodies are not supported by CGI."))
		return
	}

	pathInfo := req.URL.Path
	if root != "/" && strings.HasPrefix(pathInfo, root) {
		pathInfo = pathInfo[len(root):]
	}

	port := "80"
	if matches := trailingPort.FindStringSubmatch(req.Host); len(matches) != 0 {
		port = matches[1]
	}

	env := []string{
		"SERVER_SOFTWARE=go",
		"SERVER_NAME=" + req.Host,
		"SERVER_PROTOCOL=HTTP/1.1",
		"HTTP_HOST=" + req.Host,
		"GATEWAY_INTERFACE=CGI/1.1",
		"REQUEST_METHOD=" + req.Method,
		"QUERY_STRING=" + req.URL.RawQuery,
		"REQUEST_URI=" + req.URL.RawPath,
		"PATH_INFO=" + pathInfo,
		"SCRIPT_NAME=" + root,
		"SCRIPT_FILENAME=" + h.Path,
		"REMOTE_ADDR=" + req.RemoteAddr,
		"REMOTE_HOST=" + req.RemoteAddr,
		"SERVER_PORT=" + port,
	}

	if req.TLS != nil {
		env = append(env, "HTTPS=on")
	}

	for k, v := range req.Header {
		k = strings.Map(upperCaseAndUnderscore, k)
		joinStr := ", "
		if k == "COOKIE" {
			joinStr = "; "
		}
		env = append(env, "HTTP_"+k+"="+strings.Join(v, joinStr))
	}

	if req.ContentLength > 0 {
		env = append(env, fmt.Sprintf("CONTENT_LENGTH=%d", req.ContentLength))
	}
	if ctype := req.Header.Get("Content-Type"); ctype != "" {
		env = append(env, "CONTENT_TYPE="+ctype)
	}

	if h.Env != nil {
		env = append(env, h.Env...)
	}

	envPath := os.Getenv("PATH")
	if envPath == "" {
		envPath = "/bin:/usr/bin:/usr/ucb:/usr/bsd:/usr/local/bin"
	}
	env = append(env, "PATH="+envPath)

	for _, e := range h.InheritEnv {
		if v := os.Getenv(e); v != "" {
			env = append(env, e+"="+v)
		}
	}

	for _, e := range osDefaultInheritEnv[runtime.GOOS] {
		if v := os.Getenv(e); v != "" {
			env = append(env, e+"="+v)
		}
	}

	var cwd, path string
	if h.Dir != "" {
		path = h.Path
		cwd = h.Dir
	} else {
		cwd, path = filepath.Split(h.Path)
	}
	if cwd == "" {
		cwd = "."
	}

	internalError := func(err os.Error) {
		rw.WriteHeader(http.StatusInternalServerError)
		h.printf("CGI error: %v", err)
	}

	cmd := &exec.Cmd{
		Path:   path,
		Args:   append([]string{h.Path}, h.Args...),
		Dir:    cwd,
		Env:    env,
		Stderr: os.Stderr, // for now
	}
	if req.ContentLength != 0 {
		cmd.Stdin = req.Body
	}
	stdoutRead, err := cmd.StdoutPipe()
	if err != nil {
		internalError(err)
		return
	}

	err = cmd.Start()
	if err != nil {
		internalError(err)
		return
	}
	defer cmd.Wait()
	defer stdoutRead.Close()

	linebody, _ := bufio.NewReaderSize(stdoutRead, 1024)
	headers := make(http.Header)
	statusCode := 0
	for {
		line, isPrefix, err := linebody.ReadLine()
		if isPrefix {
			rw.WriteHeader(http.StatusInternalServerError)
			h.printf("cgi: long header line from subprocess.")
			return
		}
		if err == os.EOF {
			break
		}
		if err != nil {
			rw.WriteHeader(http.StatusInternalServerError)
			h.printf("cgi: error reading headers: %v", err)
			return
		}
		if len(line) == 0 {
			break
		}
		parts := strings.SplitN(string(line), ":", 2)
		if len(parts) < 2 {
			h.printf("cgi: bogus header line: %s", string(line))
			continue
		}
		header, val := parts[0], parts[1]
		header = strings.TrimSpace(header)
		val = strings.TrimSpace(val)
		switch {
		case header == "Status":
			if len(val) < 3 {
				h.printf("cgi: bogus status (short): %q", val)
				return
			}
			code, err := strconv.Atoi(val[0:3])
			if err != nil {
				h.printf("cgi: bogus status: %q", val)
				h.printf("cgi: line was %q", line)
				return
			}
			statusCode = code
		default:
			headers.Add(header, val)
		}
	}

	if loc := headers.Get("Location"); loc != "" {
		if strings.HasPrefix(loc, "/") && h.PathLocationHandler != nil {
			h.handleInternalRedirect(rw, req, loc)
			return
		}
		if statusCode == 0 {
			statusCode = http.StatusFound
		}
	}

	if statusCode == 0 {
		statusCode = http.StatusOK
	}

	// Copy headers to rw's headers, after we've decided not to
	// go into handleInternalRedirect, which won't want its rw
	// headers to have been touched.
	for k, vv := range headers {
		for _, v := range vv {
			rw.Header().Add(k, v)
		}
	}

	rw.WriteHeader(statusCode)

	_, err = io.Copy(rw, linebody)
	if err != nil {
		h.printf("cgi: copy error: %v", err)
	}
}

func (h *Handler) printf(format string, v ...interface{}) {
	if h.Logger != nil {
		h.Logger.Printf(format, v...)
	} else {
		log.Printf(format, v...)
	}
}

func (h *Handler) handleInternalRedirect(rw http.ResponseWriter, req *http.Request, path string) {
	url, err := req.URL.Parse(path)
	if err != nil {
		rw.WriteHeader(http.StatusInternalServerError)
		h.printf("cgi: error resolving local URI path %q: %v", path, err)
		return
	}
	// TODO: RFC 3875 isn't clear if only GET is supported, but it
	// suggests so: "Note that any message-body attached to the
	// request (such as for a POST request) may not be available
	// to the resource that is the target of the redirect."  We
	// should do some tests against Apache to see how it handles
	// POST, HEAD, etc. Does the internal redirect get the same
	// method or just GET? What about incoming headers?
	// (e.g. Cookies) Which headers, if any, are copied into the
	// second request?
	newReq := &http.Request{
		Method:     "GET",
		URL:        url,
		RawURL:     path,
		Proto:      "HTTP/1.1",
		ProtoMajor: 1,
		ProtoMinor: 1,
		Header:     make(http.Header),
		Host:       url.Host,
		RemoteAddr: req.RemoteAddr,
		TLS:        req.TLS,
	}
	h.PathLocationHandler.ServeHTTP(rw, newReq)
}

func upperCaseAndUnderscore(rune int) int {
	switch {
	case rune >= 'a' && rune <= 'z':
		return rune - ('a' - 'A')
	case rune == '-':
		return '_'
	case rune == '=':
		// Maybe not part of the CGI 'spec' but would mess up
		// the environment in any case, as Go represents the
		// environment as a slice of "key=value" strings.
		return '_'
	}
	// TODO: other transformations in spec or practice?
	return rune
}
<commit_msg>http/cgi: add openbsd environment configuration<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file implements the host side of CGI (being the webserver
// parent process).

// Package cgi implements CGI (Common Gateway Interface) as specified
// in RFC 3875.
//
// Note that using CGI means starting a new process to handle each
// request, which is typically less efficient than using a
// long-running server.  This package is intended primarily for
// compatibility with existing systems.
package cgi

import (
	"bufio"
	"exec"
	"fmt"
	"http"
	"io"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
)

var trailingPort = regexp.MustCompile(`:([0-9]+)$`)

var osDefaultInheritEnv = map[string][]string{
	"darwin":  {"DYLD_LIBRARY_PATH"},
	"freebsd": {"LD_LIBRARY_PATH"},
	"hpux":    {"LD_LIBRARY_PATH", "SHLIB_PATH"},
	"irix":    {"LD_LIBRARY_PATH", "LD_LIBRARYN32_PATH", "LD_LIBRARY64_PATH"},
	"linux":   {"LD_LIBRARY_PATH"},
	"openbsd": {"LD_LIBRARY_PATH"},
	"solaris": {"LD_LIBRARY_PATH", "LD_LIBRARY_PATH_32", "LD_LIBRARY_PATH_64"},
	"windows": {"SystemRoot", "COMSPEC", "PATHEXT", "WINDIR"},
}

// Handler runs an executable in a subprocess with a CGI environment.
type Handler struct {
	Path string // path to the CGI executable
	Root string // root URI prefix of handler or empty for "/"

	// Dir specifies the CGI executable's working directory.
	// If Dir is empty, the base directory of Path is used.
	// If Path has no base directory, the current working
	// directory is used.
	Dir string

	Env        []string    // extra environment variables to set, if any, as "key=value"
	InheritEnv []string    // environment variables to inherit from host, as "key"
	Logger     *log.Logger // optional log for errors or nil to use log.Print
	Args       []string    // optional arguments to pass to child process

	// PathLocationHandler specifies the root http Handler that
	// should handle internal redirects when the CGI process
	// returns a Location header value starting with a "/", as
	// specified in RFC 3875 § 6.3.2. This will likely be
	// http.DefaultServeMux.
	//
	// If nil, a CGI response with a local URI path is instead sent
	// back to the client and not redirected internally.
	PathLocationHandler http.Handler
}

func (h *Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	root := h.Root
	if root == "" {
		root = "/"
	}

	if len(req.TransferEncoding) > 0 && req.TransferEncoding[0] == "chunked" {
		rw.WriteHeader(http.StatusBadRequest)
		rw.Write([]byte("Chunked request bodies are not supported by CGI."))
		return
	}

	pathInfo := req.URL.Path
	if root != "/" && strings.HasPrefix(pathInfo, root) {
		pathInfo = pathInfo[len(root):]
	}

	port := "80"
	if matches := trailingPort.FindStringSubmatch(req.Host); len(matches) != 0 {
		port = matches[1]
	}

	env := []string{
		"SERVER_SOFTWARE=go",
		"SERVER_NAME=" + req.Host,
		"SERVER_PROTOCOL=HTTP/1.1",
		"HTTP_HOST=" + req.Host,
		"GATEWAY_INTERFACE=CGI/1.1",
		"REQUEST_METHOD=" + req.Method,
		"QUERY_STRING=" + req.URL.RawQuery,
		"REQUEST_URI=" + req.URL.RawPath,
		"PATH_INFO=" + pathInfo,
		"SCRIPT_NAME=" + root,
		"SCRIPT_FILENAME=" + h.Path,
		"REMOTE_ADDR=" + req.RemoteAddr,
		"REMOTE_HOST=" + req.RemoteAddr,
		"SERVER_PORT=" + port,
	}

	if req.TLS != nil {
		env = append(env, "HTTPS=on")
	}

	for k, v := range req.Header {
		k = strings.Map(upperCaseAndUnderscore, k)
		joinStr := ", "
		if k == "COOKIE" {
			joinStr = "; "
		}
		env = append(env, "HTTP_"+k+"="+strings.Join(v, joinStr))
	}

	if req.ContentLength > 0 {
		env = append(env, fmt.Sprintf("CONTENT_LENGTH=%d", req.ContentLength))
	}
	if ctype := req.Header.Get("Content-Type"); ctype != "" {
		env = append(env, "CONTENT_TYPE="+ctype)
	}

	if h.Env != nil {
		env = append(env, h.Env...)
	}

	envPath := os.Getenv("PATH")
	if envPath == "" {
		envPath = "/bin:/usr/bin:/usr/ucb:/usr/bsd:/usr/local/bin"
	}
	env = append(env, "PATH="+envPath)

	for _, e := range h.InheritEnv {
		if v := os.Getenv(e); v != "" {
			env = append(env, e+"="+v)
		}
	}

	for _, e := range osDefaultInheritEnv[runtime.GOOS] {
		if v := os.Getenv(e); v != "" {
			env = append(env, e+"="+v)
		}
	}

	var cwd, path string
	if h.Dir != "" {
		path = h.Path
		cwd = h.Dir
	} else {
		cwd, path = filepath.Split(h.Path)
	}
	if cwd == "" {
		cwd = "."
	}

	internalError := func(err os.Error) {
		rw.WriteHeader(http.StatusInternalServerError)
		h.printf("CGI error: %v", err)
	}

	cmd := &exec.Cmd{
		Path:   path,
		Args:   append([]string{h.Path}, h.Args...),
		Dir:    cwd,
		Env:    env,
		Stderr: os.Stderr, // for now
	}
	if req.ContentLength != 0 {
		cmd.Stdin = req.Body
	}
	stdoutRead, err := cmd.StdoutPipe()
	if err != nil {
		internalError(err)
		return
	}

	err = cmd.Start()
	if err != nil {
		internalError(err)
		return
	}
	defer cmd.Wait()
	defer stdoutRead.Close()

	linebody, _ := bufio.NewReaderSize(stdoutRead, 1024)
	headers := make(http.Header)
	statusCode := 0
	for {
		line, isPrefix, err := linebody.ReadLine()
		if isPrefix {
			rw.WriteHeader(http.StatusInternalServerError)
			h.printf("cgi: long header line from subprocess.")
			return
		}
		if err == os.EOF {
			break
		}
		if err != nil {
			rw.WriteHeader(http.StatusInternalServerError)
			h.printf("cgi: error reading headers: %v", err)
			return
		}
		if len(line) == 0 {
			break
		}
		parts := strings.SplitN(string(line), ":", 2)
		if len(parts) < 2 {
			h.printf("cgi: bogus header line: %s", string(line))
			continue
		}
		header, val := parts[0], parts[1]
		header = strings.TrimSpace(header)
		val = strings.TrimSpace(val)
		switch {
		case header == "Status":
			if len(val) < 3 {
				h.printf("cgi: bogus status (short): %q", val)
				return
			}
			code, err := strconv.Atoi(val[0:3])
			if err != nil {
				h.printf("cgi: bogus status: %q", val)
				h.printf("cgi: line was %q", line)
				return
			}
			statusCode = code
		default:
			headers.Add(header, val)
		}
	}

	if loc := headers.Get("Location"); loc != "" {
		if strings.HasPrefix(loc, "/") && h.PathLocationHandler != nil {
			h.handleInternalRedirect(rw, req, loc)
			return
		}
		if statusCode == 0 {
			statusCode = http.StatusFound
		}
	}

	if statusCode == 0 {
		statusCode = http.StatusOK
	}

	// Copy headers to rw's headers, after we've decided not to
	// go into handleInternalRedirect, which won't want its rw
	// headers to have been touched.
	for k, vv := range headers {
		for _, v := range vv {
			rw.Header().Add(k, v)
		}
	}

	rw.WriteHeader(statusCode)

	_, err = io.Copy(rw, linebody)
	if err != nil {
		h.printf("cgi: copy error: %v", err)
	}
}

func (h *Handler) printf(format string, v ...interface{}) {
	if h.Logger != nil {
		h.Logger.Printf(format, v...)
	} else {
		log.Printf(format, v...)
	}
}

func (h *Handler) handleInternalRedirect(rw http.ResponseWriter, req *http.Request, path string) {
	url, err := req.URL.Parse(path)
	if err != nil {
		rw.WriteHeader(http.StatusInternalServerError)
		h.printf("cgi: error resolving local URI path %q: %v", path, err)
		return
	}
	// TODO: RFC 3875 isn't clear if only GET is supported, but it
	// suggests so: "Note that any message-body attached to the
	// request (such as for a POST request) may not be available
	// to the resource that is the target of the redirect."  We
	// should do some tests against Apache to see how it handles
	// POST, HEAD, etc. Does the internal redirect get the same
	// method or just GET? What about incoming headers?
	// (e.g. Cookies) Which headers, if any, are copied into the
	// second request?
	newReq := &http.Request{
		Method:     "GET",
		URL:        url,
		RawURL:     path,
		Proto:      "HTTP/1.1",
		ProtoMajor: 1,
		ProtoMinor: 1,
		Header:     make(http.Header),
		Host:       url.Host,
		RemoteAddr: req.RemoteAddr,
		TLS:        req.TLS,
	}
	h.PathLocationHandler.ServeHTTP(rw, newReq)
}

func upperCaseAndUnderscore(rune int) int {
	switch {
	case rune >= 'a' && rune <= 'z':
		return rune - ('a' - 'A')
	case rune == '-':
		return '_'
	case rune == '=':
		// Maybe not part of the CGI 'spec' but would mess up
		// the environment in any case, as Go represents the
		// environment as a slice of "key=value" strings.
		return '_'
	}
	// TODO: other transformations in spec or practice?
	return rune
}
<|endoftext|>