<commit_before>package cli

import (
	"bufio"
	"bytes"
	"encoding/json"
	"io"
	"os"

	"errors"

	"os/exec"

	"io/ioutil"

	"path"

	"github.com/tucnak/climax"
	"github.com/vsco/dcdr/cli/api"
	"github.com/vsco/dcdr/cli/api/stores"
	"github.com/vsco/dcdr/cli/models"
	"github.com/vsco/dcdr/cli/printer"
	"github.com/vsco/dcdr/cli/ui"
	"github.com/vsco/dcdr/client"
	"github.com/vsco/dcdr/config"
	"github.com/vsco/dcdr/server"
	"github.com/zenazn/goji"
)

const FilePerms = 0755

var (
	InvalidFeatureTypeError = errors.New("invalid -value format. use -value=[0.0-1.0] or [true|false]")
	InvalidRangeError       = errors.New("invalid -value for percentile. use -value=[0.0-1.0]")
)

type Controller struct {
	Config *config.Config
	Client api.ClientIFace
}

func NewController(cfg *config.Config, kv api.ClientIFace) (cc *Controller) {
	cc = &Controller{
		Config: cfg,
		Client: kv,
	}

	return
}

func (cc *Controller) List(ctx climax.Context) int {
	pf, _ := ctx.Get("prefix")
	scope, _ := ctx.Get("scope")

	if pf != "" && scope == "" {
		scope = models.DefaultScope
	}

	features, err := cc.Client.List(pf, scope)

	if err != nil {
		printer.SayErr("%v", err)
		return 1
	}

	if len(features) == 0 {
		printer.Say("no feature flags found in namespace: %s",
			cc.Client.Namespace())
		return 1
	}

	ui.New().DrawFeatures(features)

	return 0
}

func (cc *Controller) Set(ctx climax.Context) int {
	ft, err := cc.ParseContext(ctx)

	if err != nil {
		printer.SayErr("parse error: %v", err)
		return 1
	}

	err = cc.Client.Set(ft)

	if err != nil {
		printer.SayErr("set error: %v", err)
		return 1
	}

	printer.Say("set flag '%s'", ft.ScopedKey())

	return cc.CommitFeatures(ft, false)
}

func (cc *Controller) Delete(ctx climax.Context) int {
	name, _ := ctx.Get("name")
	scope, _ := ctx.Get("scope")

	if name == "" {
		printer.Say("name is required")
		return 1
	}

	if scope == "" {
		scope = models.DefaultScope
	}

	err := cc.Client.Delete(name, scope)

	if err != nil {
		printer.SayErr("%v", err)
		return 1
	}

	printer.Say("deleted flag %s/%s/%s",
		cc.Config.Namespace, scope, name)

	ft := &models.Feature{
		Key:       name,
		Scope:     scope,
		UpdatedBy: cc.Config.Username,
	}

	return cc.CommitFeatures(ft, true)
}

func (cc *Controller) CommitFeatures(ft *models.Feature, deleted bool) int {
	if cc.Config.GitEnabled() {
		printer.Say("committing changes")
		err := cc.Client.Commit(ft, deleted)

		if err != nil {
			printer.SayErr("%v", err)
			return 1
		}

		sha, err := cc.Client.UpdateCurrentSha()
		printer.Say("set info/current_sha: %s", sha)

		if err != nil {
			printer.SayErr("%v", err)
			return 1
		}

		if cc.Config.PushEnabled() {
			printer.Say("pushing commit to origin")
			err = cc.Client.Push()

			if err != nil {
				printer.SayErr("%v", err)
				return 1
			}
		}

	}

	return 0
}

func (cc *Controller) Init(ctx climax.Context) int {
	if _, err := os.Stat(config.ConfigPath()); os.IsNotExist(err) {
		err = os.MkdirAll(path.Dir(config.ConfigPath()), FilePerms)

		printer.Say("creating %s", path.Dir(config.ConfigPath()))

		if err != nil {
			printer.SayErr("could not create config directory: %v", err)
			return 1
		}

		err = ioutil.WriteFile(config.ConfigPath(), config.ExampleConfig, FilePerms)
		printer.Say("%s not found. creating example config", config.ConfigPath())

		if err != nil {
			printer.SayErr("could not write config.hcl %v", err)
			return 1
		}
	}

	if !cc.Config.GitEnabled() {
		printer.Say("no repository configured. skipping")
		return 0
	}

	create := ctx.Is("create")

	err := cc.Client.InitRepo(create)

	if err != nil {
		printer.SayErr("%v", err)
		return 1
	}

	if create {
		printer.Say("initialized new repo in %s and pushed to %s",
			cc.Config.Git.RepoPath, cc.Config.Git.RepoURL)
	} else {
		printer.Say("cloned %s into %s",
			cc.Config.Git.RepoURL, cc.Config.Git.RepoPath)
	}

	return 0
}

func (cc *Controller) Import(ctx climax.Context) int {
	bts, err := ioutil.ReadAll(os.Stdin)

	if err != nil {
		printer.SayErr("%v", err)
		return 1
	}

	var kvs map[string]interface{}
	err = json.Unmarshal(bts, &kvs)

	if err != nil {
		printer.SayErr("%v", err)
		return 1
	}

	scope, _ := ctx.Get("scope")

	if scope == "" {
		scope = models.DefaultScope
	}

	for k, v := range kvs {
		f := models.NewFeature(k, v, "", "", scope, cc.Config.Namespace)
		err = cc.Client.Set(f)

		if err != nil {
			printer.SayErr("%v", err)
			return 1
		}

		printer.Say("set %s to %+v", k, v)
	}

	return 1
}

func (cc *Controller) Info(ctx climax.Context) int {

	ui.New().DrawConfig(cc.Config)

	return 0
}

func (cc *Controller) Serve(ctx climax.Context) int {
	c, err := client.New(cc.Config).Watch()

	if err != nil {
		panic(err)
	}

	s := server.New(cc.Config, goji.DefaultMux, c)
	s.Serve()

	return 0
}

func (cc *Controller) Watch(ctx climax.Context) int {
	cmd := exec.Command(
		"consul",
		"watch",
		"-type",
		"keyprefix",
		"-prefix",
		cc.Config.Namespace,
		"cat")

	pr, pw := io.Pipe()
	cmd.Stdout = pw
	b := &bytes.Buffer{}
	cmd.Stderr = b

	scanner := bufio.NewScanner(pr)
	scanner.Split(bufio.ScanLines)

	go func() {
		for scanner.Scan() {
			kvb, err := stores.KvPairsBytesToKvBytes(scanner.Bytes())

			if err != nil {
				printer.LogErr("parse kv error: %v", err)
				os.Exit(1)
			}

			fts, err := models.KVsToFeatureMap(kvb)

			if err != nil {
				printer.LogErr("parse features error: %v", err)
				os.Exit(1)
			}

			bts, err := json.MarshalIndent(fts, "", "  ")

			if err != nil {
				printer.LogErr("%v", err)
				os.Exit(1)
			}

			err = ioutil.WriteFile(cc.Config.Watcher.OutputPath, bts, 0644)

			if err != nil {
				printer.LogErr("%v", err)
				os.Exit(1)
			}

			printer.Log("wrote changes to %s",
				cc.Config.Watcher.OutputPath)
		}

		if scanner.Err() != nil {
			printer.LogErr("%v", scanner.Err())
			os.Exit(1)
		}
	}()

	printer.Log("watching namespace: %s", cc.Config.Namespace)

	if err := cmd.Run(); err != nil {
		printer.LogErr("%v", err)
	}

	return 0
}

func (cc *Controller) ParseContext(ctx climax.Context) (*models.Feature, error) {
	name, _ := ctx.Get("name")
	val, _ := ctx.Get("value")
	cmt, _ := ctx.Get("comment")
	scp, _ := ctx.Get("scope")

	var v interface{}
	var ft models.FeatureType

	if val != "" {
		v, ft = models.ParseValueAndFeatureType(val)

		if ft == models.Invalid {
			return nil, InvalidFeatureTypeError
		}

		if ft == models.Percentile {
			if v.(float64) > 1.0 || v.(float64) < 0 {
				return nil, InvalidRangeError
			}
		}
	}

	f := models.NewFeature(name, v, cmt, cc.Config.Username, scp, cc.Config.Namespace)
	f.FeatureType = ft

	return f, nil
}
<commit_msg>Adding -name guard to set command<commit_after>package cli

import (
	"bufio"
	"bytes"
	"encoding/json"
	"io"
	"os"

	"errors"

	"os/exec"

	"io/ioutil"

	"path"

	"github.com/tucnak/climax"
	"github.com/vsco/dcdr/cli/api"
	"github.com/vsco/dcdr/cli/api/stores"
	"github.com/vsco/dcdr/cli/models"
	"github.com/vsco/dcdr/cli/printer"
	"github.com/vsco/dcdr/cli/ui"
	"github.com/vsco/dcdr/client"
	"github.com/vsco/dcdr/config"
	"github.com/vsco/dcdr/server"
	"github.com/zenazn/goji"
)

const FilePerms = 0755

var (
	InvalidFeatureTypeError = errors.New("invalid -value format. use -value=[0.0-1.0] or [true|false]")
	InvalidRangeError       = errors.New("invalid -value for percentile. use -value=[0.0-1.0]")
	NameRequiredError       = errors.New("-name is required")
)

type Controller struct {
	Config *config.Config
	Client api.ClientIFace
}

func NewController(cfg *config.Config, kv api.ClientIFace) (cc *Controller) {
	cc = &Controller{
		Config: cfg,
		Client: kv,
	}

	return
}

func (cc *Controller) List(ctx climax.Context) int {
	pf, _ := ctx.Get("prefix")
	scope, _ := ctx.Get("scope")

	if pf != "" && scope == "" {
		scope = models.DefaultScope
	}

	features, err := cc.Client.List(pf, scope)

	if err != nil {
		printer.SayErr("%v", err)
		return 1
	}

	if len(features) == 0 {
		printer.Say("no feature flags found in namespace: %s",
			cc.Client.Namespace())
		return 1
	}

	ui.New().DrawFeatures(features)

	return 0
}

func (cc *Controller) Set(ctx climax.Context) int {
	ft, err := cc.ParseContext(ctx)

	if err != nil {
		printer.SayErr("parse error: %v", err)
		return 1
	}

	err = cc.Client.Set(ft)

	if err != nil {
		printer.SayErr("set error: %v", err)
		return 1
	}

	printer.Say("set flag '%s'", ft.ScopedKey())

	return cc.CommitFeatures(ft, false)
}

func (cc *Controller) Delete(ctx climax.Context) int {
	name, _ := ctx.Get("name")
	scope, _ := ctx.Get("scope")

	if name == "" {
		printer.Say("name is required")
		return 1
	}

	if scope == "" {
		scope = models.DefaultScope
	}

	err := cc.Client.Delete(name, scope)

	if err != nil {
		printer.SayErr("%v", err)
		return 1
	}

	printer.Say("deleted flag %s/%s/%s",
		cc.Config.Namespace, scope, name)

	ft := &models.Feature{
		Key:       name,
		Scope:     scope,
		UpdatedBy: cc.Config.Username,
	}

	return cc.CommitFeatures(ft, true)
}

func (cc *Controller) CommitFeatures(ft *models.Feature, deleted bool) int {
	if cc.Config.GitEnabled() {
		printer.Say("committing changes")
		err := cc.Client.Commit(ft, deleted)

		if err != nil {
			printer.SayErr("%v", err)
			return 1
		}

		sha, err := cc.Client.UpdateCurrentSha()
		printer.Say("set info/current_sha: %s", sha)

		if err != nil {
			printer.SayErr("%v", err)
			return 1
		}

		if cc.Config.PushEnabled() {
			printer.Say("pushing commit to origin")
			err = cc.Client.Push()

			if err != nil {
				printer.SayErr("%v", err)
				return 1
			}
		}

	}

	return 0
}

func (cc *Controller) Init(ctx climax.Context) int {
	if _, err := os.Stat(config.ConfigPath()); os.IsNotExist(err) {
		err = os.MkdirAll(path.Dir(config.ConfigPath()), FilePerms)

		printer.Say("creating %s", path.Dir(config.ConfigPath()))

		if err != nil {
			printer.SayErr("could not create config directory: %v", err)
			return 1
		}

		err = ioutil.WriteFile(config.ConfigPath(), config.ExampleConfig, FilePerms)
		printer.Say("%s not found. creating example config", config.ConfigPath())

		if err != nil {
			printer.SayErr("could not write config.hcl %v", err)
			return 1
		}
	}

	if !cc.Config.GitEnabled() {
		printer.Say("no repository configured. skipping")
		return 0
	}

	create := ctx.Is("create")

	err := cc.Client.InitRepo(create)

	if err != nil {
		printer.SayErr("%v", err)
		return 1
	}

	if create {
		printer.Say("initialized new repo in %s and pushed to %s",
			cc.Config.Git.RepoPath, cc.Config.Git.RepoURL)
	} else {
		printer.Say("cloned %s into %s",
			cc.Config.Git.RepoURL, cc.Config.Git.RepoPath)
	}

	return 0
}

func (cc *Controller) Import(ctx climax.Context) int {
	bts, err := ioutil.ReadAll(os.Stdin)

	if err != nil {
		printer.SayErr("%v", err)
		return 1
	}

	var kvs map[string]interface{}
	err = json.Unmarshal(bts, &kvs)

	if err != nil {
		printer.SayErr("%v", err)
		return 1
	}

	scope, _ := ctx.Get("scope")

	if scope == "" {
		scope = models.DefaultScope
	}

	for k, v := range kvs {
		f := models.NewFeature(k, v, "", "", scope, cc.Config.Namespace)
		err = cc.Client.Set(f)

		if err != nil {
			printer.SayErr("%v", err)
			return 1
		}

		printer.Say("set %s to %+v", k, v)
	}

	return 1
}

func (cc *Controller) Info(ctx climax.Context) int {

	ui.New().DrawConfig(cc.Config)

	return 0
}

func (cc *Controller) Serve(ctx climax.Context) int {
	c, err := client.New(cc.Config).Watch()

	if err != nil {
		panic(err)
	}

	s := server.New(cc.Config, goji.DefaultMux, c)
	s.Serve()

	return 0
}

func (cc *Controller) Watch(ctx climax.Context) int {
	cmd := exec.Command(
		"consul",
		"watch",
		"-type",
		"keyprefix",
		"-prefix",
		cc.Config.Namespace,
		"cat")

	pr, pw := io.Pipe()
	cmd.Stdout = pw
	b := &bytes.Buffer{}
	cmd.Stderr = b

	scanner := bufio.NewScanner(pr)
	scanner.Split(bufio.ScanLines)

	go func() {
		for scanner.Scan() {
			kvb, err := stores.KvPairsBytesToKvBytes(scanner.Bytes())

			if err != nil {
				printer.LogErr("parse kv error: %v", err)
				os.Exit(1)
			}

			fts, err := models.KVsToFeatureMap(kvb)

			if err != nil {
				printer.LogErr("parse features error: %v", err)
				os.Exit(1)
			}

			bts, err := json.MarshalIndent(fts, "", "  ")

			if err != nil {
				printer.LogErr("%v", err)
				os.Exit(1)
			}

			err = ioutil.WriteFile(cc.Config.Watcher.OutputPath, bts, 0644)

			if err != nil {
				printer.LogErr("%v", err)
				os.Exit(1)
			}

			printer.Log("wrote changes to %s",
				cc.Config.Watcher.OutputPath)
		}

		if scanner.Err() != nil {
			printer.LogErr("%v", scanner.Err())
			os.Exit(1)
		}
	}()

	printer.Log("watching namespace: %s", cc.Config.Namespace)

	if err := cmd.Run(); err != nil {
		printer.LogErr("%v", err)
	}

	return 0
}

func (cc *Controller) ParseContext(ctx climax.Context) (*models.Feature, error) {
	name, _ := ctx.Get("name")
	val, _ := ctx.Get("value")
	cmt, _ := ctx.Get("comment")
	scp, _ := ctx.Get("scope")

	if name == "" {
		return nil, NameRequiredError
	}

	var v interface{}
	var ft models.FeatureType

	if val != "" {
		v, ft = models.ParseValueAndFeatureType(val)

		if ft == models.Invalid {
			return nil, InvalidFeatureTypeError
		}

		if ft == models.Percentile {
			if v.(float64) > 1.0 || v.(float64) < 0 {
				return nil, InvalidRangeError
			}
		}
	}

	f := models.NewFeature(name, v, cmt, cc.Config.Username, scp, cc.Config.Namespace)
	f.FeatureType = ft

	return f, nil
}
<|endoftext|>


<commit_before>package libvirt

import (
	"encoding/xml"
	"fmt"
	libvirt "github.com/dmacvicar/libvirt-go"
	"github.com/hashicorp/terraform/helper/schema"
	"log"
	"time"
)

func resourceLibvirtDomain() *schema.Resource {
	return &schema.Resource{
		Create: resourceLibvirtDomainCreate,
		Read:   resourceLibvirtDomainRead,
		Delete: resourceLibvirtDomainDelete,
		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"vcpu": &schema.Schema{
				Type:     schema.TypeInt,
				Optional: true,
				Default:  1,
				ForceNew: true,
			},
			"memory": &schema.Schema{
				Type:     schema.TypeInt,
				Optional: true,
				Default:  512,
				ForceNew: true,
			},
			"disk": &schema.Schema{
				Type:     schema.TypeList,
				Optional: true,
				Required: false,
				ForceNew: true,
				Elem: &schema.Resource{
					Schema: diskCommonSchema(),
				},
			},
			"network_interface": &schema.Schema{
				Type:     schema.TypeList,
				Optional: true,
				Required: false,
				ForceNew: true,
				Elem: &schema.Resource{
					Schema: networkInterfaceCommonSchema(),
				},
			},
		},
	}
}

func resourceLibvirtDomainCreate(d *schema.ResourceData, meta interface{}) error {
	virConn := meta.(*Client).libvirt
	if virConn == nil {
		return fmt.Errorf("The libvirt connection was nil.")
	}

	disksCount := d.Get("disk.#").(int)
	disks := make([]defDisk, 0, disksCount)
	for i := 0; i < disksCount; i++ {
		prefix := fmt.Sprintf("disk.%d", i)
		disk := newDefDisk()
		disk.Target.Dev = fmt.Sprintf("vd%s", DiskLetterForIndex(i))

		volumeKey := d.Get(prefix + ".volume_id").(string)
		diskVolume, err := virConn.LookupStorageVolByKey(volumeKey)
		if err != nil {
			return fmt.Errorf("Can't retrieve volume %s", volumeKey)
		}
		diskVolumeName, err := diskVolume.GetName()
		if err != nil {
			return fmt.Errorf("Error retrieving volume name: %s", err)
		}
		diskPool, err := diskVolume.LookupPoolByVolume()
		if err != nil {
			return fmt.Errorf("Error retrieving pool for volume: %s", err)
		}
		diskPoolName, err := diskPool.GetName()
		if err != nil {
			return fmt.Errorf("Error retrieving pool name: %s", err)
		}

		disk.Source.Volume = diskVolumeName
		disk.Source.Pool = diskPoolName

		disks = append(disks, disk)
	}

	netIfacesCount := d.Get("network_interface.#").(int)
	netIfaces := make([]defNetworkInterface, 0, netIfacesCount)
	for i := 0; i < netIfacesCount; i++ {
		prefix := fmt.Sprintf("network_interface.%d", i)
		netIface := newDefNetworkInterface()

		if mac, ok := d.GetOk(prefix + ".mac"); ok {
			netIface.Mac.Address = mac.(string)
		} else {
			var err error
			netIface.Mac.Address, err = RandomMACAddress()
			if err != nil {
				return fmt.Errorf("Error generating mac address: %s", err)
			}
		}

		// this is not passes to libvirt, but used by waitForAddress
		if waitForLease, ok := d.GetOk(prefix + ".wait_for_lease"); ok {
			netIface.waitForLease = waitForLease.(bool)
		}

		if network, ok := d.GetOk(prefix + ".network"); ok {
			netIface.Source.Network = network.(string)
		}
		netIfaces = append(netIfaces, netIface)
	}

	domainDef := newDomainDef()
	if name, ok := d.GetOk("name"); ok {
		domainDef.Name = name.(string)
	}

	domainDef.Memory.Amount = d.Get("memory").(int)
	domainDef.VCpu.Amount = d.Get("vcpu").(int)
	domainDef.Devices.Disks = disks
	domainDef.Devices.NetworkInterfaces = netIfaces

	connectURI, err := virConn.GetURI()
	if err != nil {
		return fmt.Errorf("Error retrieving libvirt connection URI: %s", err)
	}
	log.Printf("[INFO] Creating libvirt domain at %s", connectURI)

	data, err := xml.Marshal(domainDef)
	if err != nil {
		return fmt.Errorf("Error serializing libvirt domain: %s", err)
	}

	domain, err := virConn.DomainDefineXML(string(data))
	if err != nil {
		return fmt.Errorf("Error defining libvirt domain: %s", err)
	}

	err = domain.Create()
	if err != nil {
		return fmt.Errorf("Error crearing libvirt domain: %s", err)
	}
	defer domain.Free()

	id, err := domain.GetUUIDString()
	if err != nil {
		return fmt.Errorf("Error retrieving libvirt domain id: %s", err)
	}
	d.SetId(id)

	log.Printf("[INFO] Domain ID: %s", d.Id())

	err = waitForDomainUp(domain)
	if err != nil {
		return fmt.Errorf("Error waiting for domain to reach RUNNING state: %s", err)
	}

	err = waitForNetworkAddresses(netIfaces, domain)
	if err != nil {
		return err
	}

	return resourceLibvirtDomainRead(d, meta)
}

func resourceLibvirtDomainRead(d *schema.ResourceData, meta interface{}) error {
	virConn := meta.(*Client).libvirt
	if virConn == nil {
		return fmt.Errorf("The libvirt connection was nil.")
	}

	domain, err := virConn.LookupByUUIDString(d.Id())
	if err != nil {
		return fmt.Errorf("Error retrieving libvirt domain: %s", err)
	}
	defer domain.Free()

	xmlDesc, err := domain.GetXMLDesc(0)
	if err != nil {
		return fmt.Errorf("Error retrieving libvirt domain XML description: %s", err)
	}

	domainDef := newDomainDef()
	err = xml.Unmarshal([]byte(xmlDesc), &domainDef)
	if err != nil {
		return fmt.Errorf("Error reading libvirt domain XML description: %s", err)
	}

	d.Set("name", domainDef.Name)
	d.Set("vpu", domainDef.VCpu)
	d.Set("memory", domainDef.Memory)

	disks := make([]map[string]interface{}, 0)
	for _, diskDef := range domainDef.Devices.Disks {
		virPool, err := virConn.LookupStoragePoolByName(diskDef.Source.Pool)
		if err != nil {
			return fmt.Errorf("Error retrieving pool for disk: %s", err)
		}
		defer virPool.Free()

		virVol, err := virPool.LookupStorageVolByName(diskDef.Source.Volume)
		if err != nil {
			return fmt.Errorf("Error retrieving volume for disk: %s", err)
		}
		defer virVol.Free()

		virVolKey, err := virVol.GetKey()
		if err != nil {
			return fmt.Errorf("Error retrieving volume ke for disk: %s", err)
		}

		disk := map[string]interface{}{
			"volume_id": virVolKey,
		}
		disks = append(disks, disk)
	}
	d.Set("disks", disks)

	// look interfaces with addresses
	ifacesWithAddr, err := domain.ListAllInterfaceAddresses(libvirt.VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_LEASE)
	if err != nil {
		switch err.(type) {
		default:
			return fmt.Errorf("Error retrieving interface addresses: %s", err)
		case libvirt.VirError:
			virErr := err.(libvirt.VirError)
			if virErr.Code != libvirt.VIR_ERR_OPERATION_INVALID || virErr.Domain != libvirt.VIR_FROM_QEMU {
				return fmt.Errorf("Error retrieving interface addresses: %s", err)
			}
		}
	}

	netIfaces := make([]map[string]interface{}, 0)
	for i, networkInterfaceDef := range domainDef.Devices.NetworkInterfaces {
		// we need it to read old values
		prefix := fmt.Sprintf("network_interface.%d", i)

		if networkInterfaceDef.Type != "network" {
			log.Printf("[DEBUG] ignoring interface of type '%s'", networkInterfaceDef.Type)
			continue
		}

		netIface := map[string]interface{}{
			"network": networkInterfaceDef.Source.Network,
			"mac":     networkInterfaceDef.Mac.Address,
		}

		netIfaceAddrs := make([]map[string]interface{}, 0)
		// look for an ip address and try to match it with the mac address
		// not sure if using the target device name is a better idea here
		for _, ifaceWithAddr := range ifacesWithAddr {
			if ifaceWithAddr.Hwaddr == networkInterfaceDef.Mac.Address {
				for _, addr := range ifaceWithAddr.Addrs {
					netIfaceAddr := map[string]interface{}{
						"type": func() string {
							switch addr.Type {
							case libvirt.VIR_IP_ADDR_TYPE_IPV4:
								return "ipv4"
							case libvirt.VIR_IP_ADDR_TYPE_IPV6:
								return "ipv6"
							default:
								return "other"
							}
						}(),
						"address": addr.Addr,
						"prefix":  addr.Prefix,
					}
					netIfaceAddrs = append(netIfaceAddrs, netIfaceAddr)
				}
			}
		}

		log.Printf("[DEBUG] %d addresses for %s\n", len(netIfaceAddrs), networkInterfaceDef.Mac.Address)
		netIface["address"] = netIfaceAddrs

		// pass on old wait_for_lease value
		if waitForLease, ok := d.GetOk(prefix + ".wait_for_lease"); ok {
			netIface["wait_for_lease"] = waitForLease
		}

		netIfaces = append(netIfaces, netIface)
	}
	d.Set("network_interface", netIfaces)

	if len(ifacesWithAddr) > 0 {
		d.SetConnInfo(map[string]string{
			"type": "ssh",
			"host": ifacesWithAddr[0].Addrs[0].Addr,
		})
	}
	return nil
}

func resourceLibvirtDomainDelete(d *schema.ResourceData, meta interface{}) error {
	virConn := meta.(*Client).libvirt
	if virConn == nil {
		return fmt.Errorf("The libvirt connection was nil.")
	}

	domain, err := virConn.LookupByUUIDString(d.Id())
	if err != nil {
		return fmt.Errorf("Error retrieving libvirt domain: %s", err)
	}
	defer domain.Free()

	state, err := domain.GetState()
	if err != nil {
		return fmt.Errorf("Couldn't get info about domain: %s", err)
	}

	if state[0] == libvirt.VIR_DOMAIN_RUNNING || state[0] == libvirt.VIR_DOMAIN_PAUSED {
		if err := domain.Destroy(); err != nil {
			return fmt.Errorf("Couldn't destroy libvirt domain: %s", err)
		}
	}

	if err := domain.Undefine(); err != nil {
		return fmt.Errorf("Couldn't undefine libvirt domain: %s", err)
	}

	return nil
}

// wait for domain to be up and timeout after 5 minutes.
func waitForDomainUp(domain libvirt.VirDomain) error {
	start := time.Now()
	for {
		state, err := domain.GetState()
		if err != nil {
			return err
		}

		running := true
		if state[0] != libvirt.VIR_DOMAIN_RUNNING {
			running = false
		}

		if running {
			return nil
		}
		time.Sleep(1 * time.Second)
		if time.Since(start) > 5*time.Minute {
			return fmt.Errorf("Domain didn't switch to state RUNNING in 5 minutes")
		}
	}
}

func waitForNetworkAddresses(ifaces []defNetworkInterface, domain libvirt.VirDomain) error {
	log.Printf("[DEBUG] waiting for network addresses.\n")
	// wait for network interfaces with 'wait_for_lease' to get an address
	for _, iface := range ifaces {
		if !iface.waitForLease {
			continue
		}

		if iface.Mac.Address == "" {
			log.Printf("[DEBUG] Can't wait without a mac address.\n")
			// we can't get the ip without a mac address
			continue
		}

		// loop until address appear, with timeout
		start := time.Now()
	waitLoop:
		for {
			log.Printf("[DEBUG] waiting for network address for interface with hwaddr: '%s'\n", iface.Mac.Address)
			ifacesWithAddr, err := domain.ListAllInterfaceAddresses(libvirt.VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_LEASE)
			if err != nil {
				return fmt.Errorf("Error retrieving interface addresses: %s", err)
			}

			for _, ifaceWithAddr := range ifacesWithAddr {
				// found
				if iface.Mac.Address == ifaceWithAddr.Hwaddr {
					break waitLoop
				}
			}

			time.Sleep(1 * time.Second)
			if time.Since(start) > 5*time.Minute {
				return fmt.Errorf("Timeout waiting for interface addresses")
			}
		}
	}

	return nil
}
<commit_msg>Ensure all the domains are running<commit_after>package libvirt

import (
	"encoding/xml"
	"fmt"
	libvirt "github.com/dmacvicar/libvirt-go"
	"github.com/hashicorp/terraform/helper/schema"
	"log"
	"time"
)

func resourceLibvirtDomain() *schema.Resource {
	return &schema.Resource{
		Create: resourceLibvirtDomainCreate,
		Read:   resourceLibvirtDomainRead,
		Delete: resourceLibvirtDomainDelete,
		Update: resourceLibvirtDomainUpdate,
		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"vcpu": &schema.Schema{
				Type:     schema.TypeInt,
				Optional: true,
				Default:  1,
				ForceNew: true,
			},
			"memory": &schema.Schema{
				Type:     schema.TypeInt,
				Optional: true,
				Default:  512,
				ForceNew: true,
			},
			"running": &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				Default:  true,
				ForceNew: false,
			},
			"disk": &schema.Schema{
				Type:     schema.TypeList,
				Optional: true,
				Required: false,
				ForceNew: true,
				Elem: &schema.Resource{
					Schema: diskCommonSchema(),
				},
			},
			"network_interface": &schema.Schema{
				Type:     schema.TypeList,
				Optional: true,
				Required: false,
				ForceNew: true,
				Elem: &schema.Resource{
					Schema: networkInterfaceCommonSchema(),
				},
			},
		},
	}
}

func resourceLibvirtDomainCreate(d *schema.ResourceData, meta interface{}) error {
	virConn := meta.(*Client).libvirt
	if virConn == nil {
		return fmt.Errorf("The libvirt connection was nil.")
	}

	disksCount := d.Get("disk.#").(int)
	disks := make([]defDisk, 0, disksCount)
	for i := 0; i < disksCount; i++ {
		prefix := fmt.Sprintf("disk.%d", i)
		disk := newDefDisk()
		disk.Target.Dev = fmt.Sprintf("vd%s", DiskLetterForIndex(i))

		volumeKey := d.Get(prefix + ".volume_id").(string)
		diskVolume, err := virConn.LookupStorageVolByKey(volumeKey)
		if err != nil {
			return fmt.Errorf("Can't retrieve volume %s", volumeKey)
		}
		diskVolumeName, err := diskVolume.GetName()
		if err != nil {
			return fmt.Errorf("Error retrieving volume name: %s", err)
		}
		diskPool, err := diskVolume.LookupPoolByVolume()
		if err != nil {
			return fmt.Errorf("Error retrieving pool for volume: %s", err)
		}
		diskPoolName, err := diskPool.GetName()
		if err != nil {
			return fmt.Errorf("Error retrieving pool name: %s", err)
		}

		disk.Source.Volume = diskVolumeName
		disk.Source.Pool = diskPoolName

		disks = append(disks, disk)
	}

	netIfacesCount := d.Get("network_interface.#").(int)
	netIfaces := make([]defNetworkInterface, 0, netIfacesCount)
	for i := 0; i < netIfacesCount; i++ {
		prefix := fmt.Sprintf("network_interface.%d", i)
		netIface := newDefNetworkInterface()

		if mac, ok := d.GetOk(prefix + ".mac"); ok {
			netIface.Mac.Address = mac.(string)
		} else {
			var err error
			netIface.Mac.Address, err = RandomMACAddress()
			if err != nil {
				return fmt.Errorf("Error generating mac address: %s", err)
			}
		}

		// this is not passes to libvirt, but used by waitForAddress
		if waitForLease, ok := d.GetOk(prefix + ".wait_for_lease"); ok {
			netIface.waitForLease = waitForLease.(bool)
		}

		if network, ok := d.GetOk(prefix + ".network"); ok {
			netIface.Source.Network = network.(string)
		}
		netIfaces = append(netIfaces, netIface)
	}

	domainDef := newDomainDef()
	if name, ok := d.GetOk("name"); ok {
		domainDef.Name = name.(string)
	}

	domainDef.Memory.Amount = d.Get("memory").(int)
	domainDef.VCpu.Amount = d.Get("vcpu").(int)
	domainDef.Devices.Disks = disks
	domainDef.Devices.NetworkInterfaces = netIfaces

	connectURI, err := virConn.GetURI()
	if err != nil {
		return fmt.Errorf("Error retrieving libvirt connection URI: %s", err)
	}
	log.Printf("[INFO] Creating libvirt domain at %s", connectURI)

	data, err := xml.Marshal(domainDef)
	if err != nil {
		return fmt.Errorf("Error serializing libvirt domain: %s", err)
	}

	domain, err := virConn.DomainDefineXML(string(data))
	if err != nil {
		return fmt.Errorf("Error defining libvirt domain: %s", err)
	}

	err = domain.Create()
	if err != nil {
		return fmt.Errorf("Error crearing libvirt domain: %s", err)
	}
	defer domain.Free()

	id, err := domain.GetUUIDString()
	if err != nil {
		return fmt.Errorf("Error retrieving libvirt domain id: %s", err)
	}
	d.SetId(id)

	log.Printf("[INFO] Domain ID: %s", d.Id())

	err = waitForDomainUp(domain)
	if err != nil {
		return fmt.Errorf("Error waiting for domain to reach RUNNING state: %s", err)
	}

	err = waitForNetworkAddresses(netIfaces, domain)
	if err != nil {
		return err
	}

	return resourceLibvirtDomainRead(d, meta)
}

func resourceLibvirtDomainUpdate(d *schema.ResourceData, meta interface{}) error {
	virConn := meta.(*Client).libvirt
	if virConn == nil {
		return fmt.Errorf("The libvirt connection was nil.")
	}

	domain, err := virConn.LookupByUUIDString(d.Id())
	if err != nil {
		return fmt.Errorf("Error retrieving libvirt domain: %s", err)
	}
	defer domain.Free()

	running, err := isDomainRunning(domain)
	if err != nil {
		return err
	}
	if !running {
		err = domain.Create()
		if err != nil {
			return fmt.Errorf("Error crearing libvirt domain: %s", err)
		}
	}

	return nil
}
func resourceLibvirtDomainRead(d *schema.ResourceData, meta interface{}) error {
	virConn := meta.(*Client).libvirt
	if virConn == nil {
		return fmt.Errorf("The libvirt connection was nil.")
	}

	domain, err := virConn.LookupByUUIDString(d.Id())
	if err != nil {
		return fmt.Errorf("Error retrieving libvirt domain: %s", err)
	}
	defer domain.Free()

	xmlDesc, err := domain.GetXMLDesc(0)
	if err != nil {
		return fmt.Errorf("Error retrieving libvirt domain XML description: %s", err)
	}

	domainDef := newDomainDef()
	err = xml.Unmarshal([]byte(xmlDesc), &domainDef)
	if err != nil {
		return fmt.Errorf("Error reading libvirt domain XML description: %s", err)
	}

	d.Set("name", domainDef.Name)
	d.Set("vpu", domainDef.VCpu)
	d.Set("memory", domainDef.Memory)

	running, err := isDomainRunning(domain)
	if err != nil {
		return err
	}
	d.Set("running", running)

	disks := make([]map[string]interface{}, 0)
	for _, diskDef := range domainDef.Devices.Disks {
		virPool, err := virConn.LookupStoragePoolByName(diskDef.Source.Pool)
		if err != nil {
			return fmt.Errorf("Error retrieving pool for disk: %s", err)
		}
		defer virPool.Free()

		virVol, err := virPool.LookupStorageVolByName(diskDef.Source.Volume)
		if err != nil {
			return fmt.Errorf("Error retrieving volume for disk: %s", err)
		}
		defer virVol.Free()

		virVolKey, err := virVol.GetKey()
		if err != nil {
			return fmt.Errorf("Error retrieving volume ke for disk: %s", err)
		}

		disk := map[string]interface{}{
			"volume_id": virVolKey,
		}
		disks = append(disks, disk)
	}
	d.Set("disks", disks)

	// look interfaces with addresses
	ifacesWithAddr, err := domain.ListAllInterfaceAddresses(libvirt.VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_LEASE)
	if err != nil {
		switch err.(type) {
		default:
			return fmt.Errorf("Error retrieving interface addresses: %s", err)
		case libvirt.VirError:
			virErr := err.(libvirt.VirError)
			if virErr.Code != libvirt.VIR_ERR_OPERATION_INVALID || virErr.Domain != libvirt.VIR_FROM_QEMU {
				return fmt.Errorf("Error retrieving interface addresses: %s", err)
			}
		}
	}

	netIfaces := make([]map[string]interface{}, 0)
	for i, networkInterfaceDef := range domainDef.Devices.NetworkInterfaces {
		// we need it to read old values
		prefix := fmt.Sprintf("network_interface.%d", i)

		if networkInterfaceDef.Type != "network" {
			log.Printf("[DEBUG] ignoring interface of type '%s'", networkInterfaceDef.Type)
			continue
		}

		netIface := map[string]interface{}{
			"network": networkInterfaceDef.Source.Network,
			"mac":     networkInterfaceDef.Mac.Address,
		}

		netIfaceAddrs := make([]map[string]interface{}, 0)
		// look for an ip address and try to match it with the mac address
		// not sure if using the target device name is a better idea here
		for _, ifaceWithAddr := range ifacesWithAddr {
			if ifaceWithAddr.Hwaddr == networkInterfaceDef.Mac.Address {
				for _, addr := range ifaceWithAddr.Addrs {
					netIfaceAddr := map[string]interface{}{
						"type": func() string {
							switch addr.Type {
							case libvirt.VIR_IP_ADDR_TYPE_IPV4:
								return "ipv4"
							case libvirt.VIR_IP_ADDR_TYPE_IPV6:
								return "ipv6"
							default:
								return "other"
							}
						}(),
						"address": addr.Addr,
						"prefix":  addr.Prefix,
					}
					netIfaceAddrs = append(netIfaceAddrs, netIfaceAddr)
				}
			}
		}

		log.Printf("[DEBUG] %d addresses for %s\n", len(netIfaceAddrs), networkInterfaceDef.Mac.Address)
		netIface["address"] = netIfaceAddrs

		// pass on old wait_for_lease value
		if waitForLease, ok := d.GetOk(prefix + ".wait_for_lease"); ok {
			netIface["wait_for_lease"] = waitForLease
		}

		netIfaces = append(netIfaces, netIface)
	}
	d.Set("network_interface", netIfaces)

	if len(ifacesWithAddr) > 0 {
		d.SetConnInfo(map[string]string{
			"type": "ssh",
			"host": ifacesWithAddr[0].Addrs[0].Addr,
		})
	}
	return nil
}

func resourceLibvirtDomainDelete(d *schema.ResourceData, meta interface{}) error {
	virConn := meta.(*Client).libvirt
	if virConn == nil {
		return fmt.Errorf("The libvirt connection was nil.")
	}

	domain, err := virConn.LookupByUUIDString(d.Id())
	if err != nil {
		return fmt.Errorf("Error retrieving libvirt domain: %s", err)
	}
	defer domain.Free()

	state, err := domain.GetState()
	if err != nil {
		return fmt.Errorf("Couldn't get info about domain: %s", err)
	}

	if state[0] == libvirt.VIR_DOMAIN_RUNNING || state[0] == libvirt.VIR_DOMAIN_PAUSED {
		if err := domain.Destroy(); err != nil {
			return fmt.Errorf("Couldn't destroy libvirt domain: %s", err)
		}
	}

	if err := domain.Undefine(); err != nil {
		return fmt.Errorf("Couldn't undefine libvirt domain: %s", err)
	}

	return nil
}

// wait for domain to be up and timeout after 5 minutes.
func waitForDomainUp(domain libvirt.VirDomain) error {
	start := time.Now()
	for {
		state, err := domain.GetState()
		if err != nil {
			return err
		}

		running := true
		if state[0] != libvirt.VIR_DOMAIN_RUNNING {
			running = false
		}

		if running {
			return nil
		}
		time.Sleep(1 * time.Second)
		if time.Since(start) > 5*time.Minute {
			return fmt.Errorf("Domain didn't switch to state RUNNING in 5 minutes")
		}
	}
}

func waitForNetworkAddresses(ifaces []defNetworkInterface, domain libvirt.VirDomain) error {
	log.Printf("[DEBUG] waiting for network addresses.\n")
	// wait for network interfaces with 'wait_for_lease' to get an address
	for _, iface := range ifaces {
		if !iface.waitForLease {
			continue
		}

		if iface.Mac.Address == "" {
			log.Printf("[DEBUG] Can't wait without a mac address.\n")
			// we can't get the ip without a mac address
			continue
		}

		// loop until address appear, with timeout
		start := time.Now()
	waitLoop:
		for {
			log.Printf("[DEBUG] waiting for network address for interface with hwaddr: '%s'\n", iface.Mac.Address)
			ifacesWithAddr, err := domain.ListAllInterfaceAddresses(libvirt.VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_LEASE)
			if err != nil {
				return fmt.Errorf("Error retrieving interface addresses: %s", err)
			}

			for _, ifaceWithAddr := range ifacesWithAddr {
				// found
				if iface.Mac.Address == ifaceWithAddr.Hwaddr {
					break waitLoop
				}
			}

			time.Sleep(1 * time.Second)
			if time.Since(start) > 5*time.Minute {
				return fmt.Errorf("Timeout waiting for interface addresses")
			}
		}
	}

	return nil
}

func isDomainRunning(domain libvirt.VirDomain) (bool, error) {
	state, err := domain.GetState()
	if err != nil {
		return false, fmt.Errorf("Couldn't get state of domain: %s", err)
	}

	return state[0] == libvirt.VIR_DOMAIN_RUNNING, nil
}
<|endoftext|>


<commit_before>package libvirt

import (
	"encoding/xml"
	"fmt"
	libvirt "github.com/dmacvicar/libvirt-go"
	"github.com/hashicorp/terraform/helper/schema"
	"io"
	"log"
	"net/http"
	"strconv"
)

func volumeCommonSchema() map[string]*schema.Schema {
	return map[string]*schema.Schema{
		"name": &schema.Schema{
			Type:     schema.TypeString,
			Required: true,
			ForceNew: true,
		},
		"pool": &schema.Schema{
			Type:     schema.TypeString,
			Optional: true,
			Default:  "default",
			ForceNew: true,
		},
		"source": &schema.Schema{
			Type:     schema.TypeString,
			Optional: true,
			ForceNew: true,
		},
		"size": &schema.Schema{
			Type:     schema.TypeInt,
			Optional: true,
			Computed: true,
			ForceNew: true,
		},
		"base_volume": &schema.Schema{
			Type:     schema.TypeString,
			Optional: true,
			ForceNew: true,
		},
	}
}

func resourceLibvirtVolume() *schema.Resource {
	return &schema.Resource{
		Create: resourceLibvirtVolumeCreate,
		Read:   resourceLibvirtVolumeRead,
		Delete: resourceLibvirtVolumeDelete,
		Schema: volumeCommonSchema(),
	}
}

func remoteImageSize(url string) (int, error) {
	response, err := http.Head(url)
	if err != nil {
		return 0, err
	}
	length, err := strconv.Atoi(response.Header.Get("Content-Length"))
	if err != nil {
		return 0, err
	}
	return length, nil
}

func resourceLibvirtVolumeCreate(d *schema.ResourceData, meta interface{}) error {
	virConn := meta.(*Client).libvirt
	if virConn == nil {
		return fmt.Errorf("The libvirt connection was nil.")
	}

	poolName := "default"
	if _, ok := d.GetOk("pool"); ok {
		poolName = d.Get("pool").(string)
	}

	pool, err := virConn.LookupStoragePoolByName(poolName)
	if err != nil {
		return fmt.Errorf("can't find storage pool '%s'", poolName)
	}
	defer pool.Free()

	// Refresh the pool of the volume so that libvirt knows it is
	// not longer in use.
	err = pool.Refresh(0)
	if err != nil {
		return fmt.Errorf("Error refreshing pool for volume: %s", err)
	}

	volumeDef := newDefVolume()

	if name, ok := d.GetOk("name"); ok {
		volumeDef.Name = name.(string)
	}

	// an existing image was given, this mean we can't choose size
	if url, ok := d.GetOk("source"); ok {
		// source and size conflict
		if _, ok := d.GetOk("size"); ok {
			return fmt.Errorf("'size' can't be specified when also 'source' is given (the size will be set to the size of the source image.")
		}

		size, err := remoteImageSize(url.(string))
		if err != nil {
			return err
		}
		log.Printf("Remote image is: %d bytes", size)
		volumeDef.Capacity.Unit = "B"
		volumeDef.Capacity.Amount = size
	} else {
		_, noSize := d.GetOk("size")
		_, noBaseVol := d.GetOk("base_volume")

		if noSize && noBaseVol {
			return fmt.Errorf("'size' needs to be specified if no 'source' or 'base_vol' is given.")
		}
		volumeDef.Capacity.Amount = d.Get("size").(int)
	}

	if baseVolumeId, ok := d.GetOk("base_volume"); ok {
		if _, ok := d.GetOk("size"); ok {
			return fmt.Errorf("'size' can't be specified when also 'base_volume' is given (the size will be set to the size of the backing image.")
		}

		volumeDef.BackingStore = new(defBackingStore)
		volumeDef.BackingStore.Format.Type = "qcow2"
		baseVolume, err := virConn.LookupStorageVolByKey(baseVolumeId.(string))
		if err != nil {
			return fmt.Errorf("Can't retrieve volume %s", baseVolumeId.(string))
		}
		baseVolPath, err := baseVolume.GetPath()
		if err != nil {
			return fmt.Errorf("can't get name for base image '%s'", baseVolumeId)
		}
		volumeDef.BackingStore.Path = baseVolPath
	}

	volumeDefXml, err := xml.Marshal(volumeDef)
	if err != nil {
		return fmt.Errorf("Error serializing libvirt volume: %s", err)
	}

	// create the volume
	volume, err := pool.StorageVolCreateXML(string(volumeDefXml), 0)
	if err != nil {
		return fmt.Errorf("Error creating libvirt volume: %s", err)
	}
	defer volume.Free()

	// we use the key as the id
	key, err := volume.GetKey()
	if err != nil {
		return fmt.Errorf("Error retrieving volume key: %s", err)
	}
	d.SetId(key)
	log.Printf("[INFO] Volume ID: %s", d.Id())

	// upload source if present
	if url, ok := d.GetOk("source"); ok {
		stream, err := libvirt.NewVirStream(virConn, 0)
		defer stream.Close()

		volume.Upload(stream, 0, uint64(volumeDef.Capacity.Amount), 0)
		response, err := http.Get(url.(string))
		defer response.Body.Close()
		if err != nil {
			return fmt.Errorf("Error while downloading %s: %s", url.(string), err)
		}

		n, err := io.Copy(stream, response.Body)
		if err != nil {
			return fmt.Errorf("Error while downloading %s: %s", url.(string), err)
		}
		log.Printf("%d bytes uploaded\n", n)
	}

	return resourceLibvirtVolumeRead(d, meta)
}

func resourceLibvirtVolumeRead(d *schema.ResourceData, meta interface{}) error {
	virConn := meta.(*Client).libvirt
	if virConn == nil {
		return fmt.Errorf("The libvirt connection was nil.")
	}

	volume, err := virConn.LookupStorageVolByKey(d.Id())
	if err != nil {
		return fmt.Errorf("Can't retrieve volume %s", d.Id())
	}
	defer volume.Free()

	volName, err := volume.GetName()
	if err != nil {
		return fmt.Errorf("Error retrieving volume name: %s", err)
	}

	volPool, err := volume.LookupPoolByVolume()
	if err != nil {
		return fmt.Errorf("Error retrieving pool for volume: %s", err)
	}
	defer volPool.Free()

	volPoolName, err := volPool.GetName()
	if err != nil {
		return fmt.Errorf("Error retrieving pool name: %s", err)
	}

	d.Set("pool", volPoolName)
	d.Set("name", volName)

	info, err := volume.GetInfo()
	if err != nil {
		return fmt.Errorf("Error retrieving volume name: %s", err)
	}
	d.Set("size", info.GetCapacityInBytes())

	return nil
}

func resourceLibvirtVolumeDelete(d *schema.ResourceData, meta interface{}) error {
	virConn := meta.(*Client).libvirt
	if virConn == nil {
		return fmt.Errorf("The libvirt connection was nil.")
	}

	volume, err := virConn.LookupStorageVolByKey(d.Id())
	if err != nil {
		return fmt.Errorf("Can't retrieve volume %s", d.Id())
	}
	defer volume.Free()

	// Refresh the pool of the volume so that libvirt knows it is
	// not longer in use.
	volPool, err := volume.LookupPoolByVolume()
	if err != nil {
		return fmt.Errorf("Error retrieving pool for volume: %s", err)
	}
	defer volPool.Free()

	err = volPool.Refresh(0)
	if err != nil {
		return fmt.Errorf("Error refreshing pool for volume: %s", err)
	}

	err = volume.Delete(0)
	if err != nil {
		return fmt.Errorf("Can't delete volume %s", d.Id())
	}

	return nil
}
<commit_msg>include previous error in new error<commit_after>package libvirt

import (
	"encoding/xml"
	"fmt"
	libvirt "github.com/dmacvicar/libvirt-go"
	"github.com/hashicorp/terraform/helper/schema"
	"io"
	"log"
	"net/http"
	"strconv"
)

func volumeCommonSchema() map[string]*schema.Schema {
	return map[string]*schema.Schema{
		"name": &schema.Schema{
			Type:     schema.TypeString,
			Required: true,
			ForceNew: true,
		},
		"pool": &schema.Schema{
			Type:     schema.TypeString,
			Optional: true,
			Default:  "default",
			ForceNew: true,
		},
		"source": &schema.Schema{
			Type:     schema.TypeString,
			Optional: true,
			ForceNew: true,
		},
		"size": &schema.Schema{
			Type:     schema.TypeInt,
			Optional: true,
			Computed: true,
			ForceNew: true,
		},
		"base_volume": &schema.Schema{
			Type:     schema.TypeString,
			Optional: true,
			ForceNew: true,
		},
	}
}

func resourceLibvirtVolume() *schema.Resource {
	return &schema.Resource{
		Create: resourceLibvirtVolumeCreate,
		Read:   resourceLibvirtVolumeRead,
		Delete: resourceLibvirtVolumeDelete,
		Schema: volumeCommonSchema(),
	}
}

func remoteImageSize(url string) (int, error) {
	response, err := http.Head(url)
	if err != nil {
		return 0, err
	}
	length, err := strconv.Atoi(response.Header.Get("Content-Length"))
	if err != nil {
		return 0, err
	}
	return length, nil
}

func resourceLibvirtVolumeCreate(d *schema.ResourceData, meta interface{}) error {
	virConn := meta.(*Client).libvirt
	if virConn == nil {
		return fmt.Errorf("The libvirt connection was nil.")
	}

	poolName := "default"
	if _, ok := d.GetOk("pool"); ok {
		poolName = d.Get("pool").(string)
	}

	pool, err := virConn.LookupStoragePoolByName(poolName)
	if err != nil {
		return fmt.Errorf("can't find storage pool '%s'", poolName)
	}
	defer pool.Free()

	// Refresh the pool of the volume so that libvirt knows it is
	// not longer in use.
	err = pool.Refresh(0)
	if err != nil {
		return fmt.Errorf("Error refreshing pool for volume: %s", err)
	}

	volumeDef := newDefVolume()

	if name, ok := d.GetOk("name"); ok {
		volumeDef.Name = name.(string)
	}

	// an existing image was given, this mean we can't choose size
	if url, ok := d.GetOk("source"); ok {
		// source and size conflict
		if _, ok := d.GetOk("size"); ok {
			return fmt.Errorf("'size' can't be specified when also 'source' is given (the size will be set to the size of the source image.")
		}

		size, err := remoteImageSize(url.(string))
		if err != nil {
			return err
		}
		log.Printf("Remote image is: %d bytes", size)
		volumeDef.Capacity.Unit = "B"
		volumeDef.Capacity.Amount = size
	} else {
		_, noSize := d.GetOk("size")
		_, noBaseVol := d.GetOk("base_volume")

		if noSize && noBaseVol {
			return fmt.Errorf("'size' needs to be specified if no 'source' or 'base_vol' is given.")
		}
		volumeDef.Capacity.Amount = d.Get("size").(int)
	}

	if baseVolumeId, ok := d.GetOk("base_volume"); ok {
		if _, ok := d.GetOk("size"); ok {
			return fmt.Errorf("'size' can't be specified when also 'base_volume' is given (the size will be set to the size of the backing image.")
		}

		volumeDef.BackingStore = new(defBackingStore)
		volumeDef.BackingStore.Format.Type = "qcow2"
		baseVolume, err := virConn.LookupStorageVolByKey(baseVolumeId.(string))
		if err != nil {
			return fmt.Errorf("Can't retrieve volume %s", baseVolumeId.(string))
		}
		baseVolPath, err := baseVolume.GetPath()
		if err != nil {
			return fmt.Errorf("can't get name for base image '%s'", baseVolumeId)
		}
		volumeDef.BackingStore.Path = baseVolPath
	}

	volumeDefXml, err := xml.Marshal(volumeDef)
	if err != nil {
		return fmt.Errorf("Error serializing libvirt volume: %s", err)
	}

	// create the volume
	volume, err := pool.StorageVolCreateXML(string(volumeDefXml), 0)
	if err != nil {
		return fmt.Errorf("Error creating libvirt volume: %s", err)
	}
	defer volume.Free()

	// we use the key as the id
	key, err := volume.GetKey()
	if err != nil {
		return fmt.Errorf("Error retrieving volume key: %s", err)
	}
	d.SetId(key)
	log.Printf("[INFO] Volume ID: %s", d.Id())

	// upload source if present
	if url, ok := d.GetOk("source"); ok {
		stream, err := libvirt.NewVirStream(virConn, 0)
		defer stream.Close()

		volume.Upload(stream, 0, uint64(volumeDef.Capacity.Amount), 0)
		response, err := http.Get(url.(string))
		defer response.Body.Close()
		if err != nil {
			return fmt.Errorf("Error while downloading %s: %s", url.(string), err)
		}

		n, err := io.Copy(stream, response.Body)
		if err != nil {
			return fmt.Errorf("Error while downloading %s: %s", url.(string), err)
		}
		log.Printf("%d bytes uploaded\n", n)
	}

	return resourceLibvirtVolumeRead(d, meta)
}

func resourceLibvirtVolumeRead(d *schema.ResourceData, meta interface{}) error {
	virConn := meta.(*Client).libvirt
	if virConn == nil {
		return fmt.Errorf("The libvirt connection was nil.")
	}

	volume, err := virConn.LookupStorageVolByKey(d.Id())
	if err != nil {
		return fmt.Errorf("Can't retrieve volume %s", d.Id())
	}
	defer volume.Free()

	volName, err := volume.GetName()
	if err != nil {
		return fmt.Errorf("Error retrieving volume name: %s", err)
	}

	volPool, err := volume.LookupPoolByVolume()
	if err != nil {
		return fmt.Errorf("Error retrieving pool for volume: %s", err)
	}
	defer volPool.Free()

	volPoolName, err := volPool.GetName()
	if err != nil {
		return fmt.Errorf("Error retrieving pool name: %s", err)
	}

	d.Set("pool", volPoolName)
	d.Set("name", volName)

	info, err := volume.GetInfo()
	if err != nil {
		return fmt.Errorf("Error retrieving volume name: %s", err)
	}
	d.Set("size", info.GetCapacityInBytes())

	return nil
}

func resourceLibvirtVolumeDelete(d *schema.ResourceData, meta interface{}) error {
	virConn := meta.(*Client).libvirt
	if virConn == nil {
		return fmt.Errorf("The libvirt connection was nil.")
	}

	volume, err := virConn.LookupStorageVolByKey(d.Id())
	if err != nil {
		return fmt.Errorf("Can't retrieve volume %s", d.Id())
	}
	defer volume.Free()

	// Refresh the pool of the volume so that libvirt knows it is
	// not longer in use.
	volPool, err := volume.LookupPoolByVolume()
	if err != nil {
		return fmt.Errorf("Error retrieving pool for volume: %s", err)
	}
	defer volPool.Free()

	err = volPool.Refresh(0)
	if err != nil {
		return fmt.Errorf("Error refreshing pool for volume: %s", err)
	}

	err = volume.Delete(0)
	if err != nil {
		return fmt.Errorf("Can't delete volume %s: %s", d.Id(), err)
	}

	return nil
}
<|endoftext|>


<commit_before>package drive

import (
    "fmt"
    "io"
    "os"
    "time"
    "sort"
    "bytes"
    "path/filepath"
    "google.golang.org/api/googleapi"
    "google.golang.org/api/drive/v3"
)

type UploadSyncArgs struct {
    Out io.Writer
    Progress io.Writer
    Path string
    RootId string
    DryRun bool
    DeleteExtraneous bool
    ChunkSize int64
    Resolution ConflictResolution
    Comparer FileComparer
}

func (self *Drive) UploadSync(args UploadSyncArgs) error {
    if args.ChunkSize > intMax() - 1 {
        return fmt.Errorf("Chunk size is to big, max chunk size for this computer is %d", intMax() - 1)
    }

    fmt.Fprintln(args.Out, "Starting sync...")
    started := time.Now()

    // Create root directory if it does not exist
    rootDir, err := self.prepareSyncRoot(args)
    if err != nil {
        return err
    }

    fmt.Fprintln(args.Out, "Collecting local and remote file information...")
    files, err := self.prepareSyncFiles(args.Path, rootDir, args.Comparer)
    if err != nil {
        return err
    }

    // Find changed files
    changedFiles := files.filterChangedLocalFiles()

    fmt.Fprintf(args.Out, "Found %d local files and %d remote files\n", len(files.local), len(files.remote))

    // Ensure that that we don't overwrite any remote changes
    if args.Resolution == NoResolution {
        err = ensureNoRemoteModifications(changedFiles)
        if err != nil {
            return fmt.Errorf("Conflict detected!\nThe following files have changed and the remote file are newer than it's local counterpart:\n\n%s\nNo conflict resolution was given, aborting...", err)
        }
    }

    // Create missing directories
    files, err = self.createMissingRemoteDirs(files, args)
    if err != nil {
        return err
    }

    // Upload missing files
    err = self.uploadMissingFiles(files, args)
    if err != nil {
        return err
    }

    // Update modified files
    err = self.updateChangedFiles(changedFiles, rootDir, args)
    if err != nil {
        return err
    }

    // Delete extraneous files on drive
    if args.DeleteExtraneous {
        err = self.deleteExtraneousRemoteFiles(files, args)
        if err != nil {
            return err
        }
    }
    fmt.Fprintf(args.Out, "Sync finished in %s\n", time.Since(started))

    return nil
}

func (self *Drive) prepareSyncRoot(args UploadSyncArgs) (*drive.File, error) {
    fields := []googleapi.Field{"id", "name", "mimeType", "appProperties"}
    f, err := self.service.Files.Get(args.RootId).Fields(fields...).Do()
    if err != nil {
        return nil, fmt.Errorf("Failed to find root dir: %s", err)
    }

    // Ensure file is a directory
    if !isDir(f) {
        return nil, fmt.Errorf("Provided root id is not a directory")
    }

    // Return directory if syncRoot property is already set
    if _, ok := f.AppProperties["isSyncRoot"]; ok {
        return f, nil
    }

    // This is the first time this directory have been used for sync
    // Check if the directory is empty
    isEmpty, err := self.dirIsEmpty(f.Id)
    if err != nil {
        return nil, fmt.Errorf("Failed to check if root dir is empty: %s", err)
    }

    // Ensure that the directory is empty
    if !isEmpty {
        return nil, fmt.Errorf("Root directoy is not empty, the initial sync requires an empty directory")
    }

    // Update directory with syncRoot property
    dstFile := &drive.File{
        AppProperties: map[string]string{"isSyncRoot": "true"},
    }

    f, err = self.service.Files.Update(f.Id, dstFile).Fields(fields...).Do()
    if err != nil {
        return nil, fmt.Errorf("Failed to update root directory: %s", err)
    }

    return f, nil
}

func (self *Drive) createMissingRemoteDirs(files *syncFiles, args UploadSyncArgs) (*syncFiles, error) {
    missingDirs := files.filterMissingRemoteDirs()
    missingCount := len(missingDirs)

    if missingCount > 0 {
        fmt.Fprintf(args.Out, "\n%d remote directories are missing\n", missingCount)
    }

    // Sort directories so that the dirs with the shortest path comes first
    sort.Sort(byLocalPathLength(missingDirs))

    for i, lf := range missingDirs {
        parentPath := parentFilePath(lf.relPath)
        parent, ok := files.findRemoteByPath(parentPath)
        if !ok {
            return nil, fmt.Errorf("Could not find remote directory with path '%s'", parentPath)
        }

        fmt.Fprintf(args.Out, "[%04d/%04d] Creating directory %s\n", i + 1, missingCount, filepath.Join(files.root.file.Name, lf.relPath))

        f, err := self.createMissingRemoteDir(createMissingRemoteDirArgs{
            name: lf.info.Name(),
            parentId: parent.file.Id,
            rootId: args.RootId,
            dryRun: args.DryRun,
            try: 0,
        })
        if err != nil {
            return nil, err
        }

        files.remote = append(files.remote, &RemoteFile{
            relPath: lf.relPath,
            file: f,
        })
    }

    return files, nil
}

type createMissingRemoteDirArgs struct {
    name string
    parentId string
    rootId string
    dryRun bool
    try int
}

func (self *Drive) uploadMissingFiles(files *syncFiles, args UploadSyncArgs) error {
    missingFiles := files.filterMissingRemoteFiles()
    missingCount := len(missingFiles)

    if missingCount > 0 {
        fmt.Fprintf(args.Out, "\n%d remote files are missing\n", missingCount)
    }

    for i, lf := range missingFiles {
        parentPath := parentFilePath(lf.relPath)
        parent, ok := files.findRemoteByPath(parentPath)
        if !ok {
            return fmt.Errorf("Could not find remote directory with path '%s'", parentPath)
        }

        fmt.Fprintf(args.Out, "[%04d/%04d] Uploading %s -> %s\n", i + 1, missingCount, lf.relPath, filepath.Join(files.root.file.Name, lf.relPath))

        err := self.uploadMissingFile(parent.file.Id, lf, args, 0)
        if err != nil {
            return err
        }
    }

    return nil
}

func (self *Drive) updateChangedFiles(changedFiles []*changedFile, root *drive.File, args UploadSyncArgs) error {
    changedCount := len(changedFiles)

    if changedCount > 0 {
        fmt.Fprintf(args.Out, "\n%d local files has changed\n", changedCount)
    }

    for i, cf := range changedFiles {
        if skip, reason := checkRemoteConflict(cf, args.Resolution); skip {
            fmt.Fprintf(args.Out, "[%04d/%04d] Skipping %s (%s)\n", i + 1, changedCount, cf.local.relPath, reason)
            continue
        }

        fmt.Fprintf(args.Out, "[%04d/%04d] Updating %s -> %s\n", i + 1, changedCount, cf.local.relPath, filepath.Join(root.Name, cf.local.relPath))

        err := self.updateChangedFile(cf, args, 0)
        if err != nil {
            return err
        }
    }

    return nil
}

func (self *Drive) deleteExtraneousRemoteFiles(files *syncFiles, args UploadSyncArgs) error {
    extraneousFiles := files.filterExtraneousRemoteFiles()
    extraneousCount := len(extraneousFiles)

    if extraneousCount > 0 {
        fmt.Fprintf(args.Out, "\n%d remote files are extraneous\n", extraneousCount)
    }

    // Sort files so that the files with the longest path comes first
    sort.Sort(sort.Reverse(byRemotePathLength(extraneousFiles)))

    for i, rf := range extraneousFiles {
        fmt.Fprintf(args.Out, "[%04d/%04d] Deleting %s\n", i + 1, extraneousCount, filepath.Join(files.root.file.Name, rf.relPath))

        err := self.deleteRemoteFile(rf, args, 0)
        if err != nil {
            return err
        }
    }

    return nil
}

func (self *Drive) createMissingRemoteDir(args createMissingRemoteDirArgs) (*drive.File, error) {
    dstFile := &drive.File{
        Name: args.name,
        MimeType: DirectoryMimeType,
        Parents: []string{args.parentId},
        AppProperties: map[string]string{"syncRootId": args.rootId},
    }

    if args.dryRun {
        return dstFile, nil
    }

    f, err := self.service.Files.Create(dstFile).Do()
    if err != nil {
        if isBackendError(err) && args.try < MaxBackendErrorRetries {
            exponentialBackoffSleep(args.try)
            args.try++
            return self.createMissingRemoteDir(args)
        } else {
            return nil, fmt.Errorf("Failed to create directory: %s", err)
        }
    }

    return f, nil
}

func (self *Drive) uploadMissingFile(parentId string, lf *LocalFile, args UploadSyncArgs, try int) error {
    if args.DryRun {
        return nil
    }

    srcFile, err := os.Open(lf.absPath)
    if err != nil {
        return fmt.Errorf("Failed to open file: %s", err)
    }

    // Close file on function exit
    defer srcFile.Close()

    // Instantiate drive file
    dstFile := &drive.File{
        Name: lf.info.Name(),
        Parents: []string{parentId},
        AppProperties: map[string]string{"syncRootId": args.RootId},
    }

    // Chunk size option
    chunkSize := googleapi.ChunkSize(int(args.ChunkSize))

    // Wrap file in progress reader
    srcReader := getProgressReader(srcFile, args.Progress, lf.info.Size())

    _, err = self.service.Files.Create(dstFile).Fields("id", "name", "size", "md5Checksum").Media(srcReader, chunkSize).Do()
    if err != nil {
        if isBackendError(err) && try < MaxBackendErrorRetries {
            exponentialBackoffSleep(try)
            try++
            return self.uploadMissingFile(parentId, lf, args, try)
        } else {
            return fmt.Errorf("Failed to upload file: %s", err)
        }
    }

    return nil
}

func (self *Drive) updateChangedFile(cf *changedFile, args UploadSyncArgs, try int) error {
    if args.DryRun {
        return nil
    }

    srcFile, err := os.Open(cf.local.absPath)
    if err != nil {
        return fmt.Errorf("Failed to open file: %s", err)
    }

    // Close file on function exit
    defer srcFile.Close()

    // Instantiate drive file
    dstFile := &drive.File{}

    // Chunk size option
    chunkSize := googleapi.ChunkSize(int(args.ChunkSize))

    // Wrap file in progress reader
    srcReader := getProgressReader(srcFile, args.Progress, cf.local.info.Size())

    _, err = self.service.Files.Update(cf.remote.file.Id, dstFile).Media(srcReader, chunkSize).Do()
    if err != nil {
        if isBackendError(err) && try < MaxBackendErrorRetries {
            exponentialBackoffSleep(try)
            try++
            return self.updateChangedFile(cf, args, try)
        } else {
            return fmt.Errorf("Failed to update file: %s", err)
        }
    }

    return nil
}

func (self *Drive) deleteRemoteFile(rf *RemoteFile, args UploadSyncArgs, try int) error {
    if args.DryRun {
        return nil
    }

    err := self.service.Files.Delete(rf.file.Id).Do()
    if err != nil {
        if isBackendError(err) && try < MaxBackendErrorRetries {
            exponentialBackoffSleep(try)
            try++
            return self.deleteRemoteFile(rf, args, try)
        } else {
            return fmt.Errorf("Failed to delete file: %s", err)
        }
    }

    return nil
}

func (self *Drive) dirIsEmpty(id string) (bool, error) {
    query := fmt.Sprintf("'%s' in parents", id)
    fileList, err := self.service.Files.List().Q(query).Do()
    if err != nil {
        return false, fmt.Errorf("Empty dir check failed: ", err)
    }

    return len(fileList.Files) == 0, nil
}

func checkRemoteConflict(cf *changedFile, resolution ConflictResolution) (bool, string) {
    // No conflict unless remote file was last modified
    if cf.compareModTime() != RemoteLastModified {
        return false, ""
    }

    // Don't skip if want to keep the local file
    if resolution == KeepLocal {
        return false, ""
    }

    // Skip if we want to keep the remote file
    if resolution == KeepRemote {
        return true, "conflicting file, keeping remote file"
    }

    if resolution == KeepLargest {
        largest := cf.compareSize()

        // Skip if the remote file is largest
        if largest == RemoteLargestSize {
            return true, "conflicting file, remote file is largest, keeping remote"
        }

        // Don't skip if the local file is largest
        if largest == LocalLargestSize {
            return false, ""
        }

        // Keep remote if both files have the same size
        if largest == EqualSize {
            return true, "conflicting file, file sizes are equal, keeping remote"
        }
    }

    // The conditionals above should cover all cases,
    // unless the programmer did something wrong,
    // in which case we default to being non-destructive and skip the file
    return true, "conflicting file, unhandled case"
}

func ensureNoRemoteModifications(files []*changedFile) error {
    conflicts := findRemoteConflicts(files)
    if len(conflicts) == 0 {
        return nil
    }

    buffer := bytes.NewBufferString("")
    formatConflicts(conflicts, buffer)
    return fmt.Errorf(buffer.String())
}
<commit_msg>Ensure that there is enough free space on drive<commit_after>package drive

import (
    "fmt"
    "io"
    "os"
    "time"
    "sort"
    "bytes"
    "path/filepath"
    "google.golang.org/api/googleapi"
    "google.golang.org/api/drive/v3"
)

type UploadSyncArgs struct {
    Out io.Writer
    Progress io.Writer
    Path string
    RootId string
    DryRun bool
    DeleteExtraneous bool
    ChunkSize int64
    Resolution ConflictResolution
    Comparer FileComparer
}

func (self *Drive) UploadSync(args UploadSyncArgs) error {
    if args.ChunkSize > intMax() - 1 {
        return fmt.Errorf("Chunk size is to big, max chunk size for this computer is %d", intMax() - 1)
    }

    fmt.Fprintln(args.Out, "Starting sync...")
    started := time.Now()

    // Create root directory if it does not exist
    rootDir, err := self.prepareSyncRoot(args)
    if err != nil {
        return err
    }

    fmt.Fprintln(args.Out, "Collecting local and remote file information...")
    files, err := self.prepareSyncFiles(args.Path, rootDir, args.Comparer)
    if err != nil {
        return err
    }

    // Find missing and changed files
    changedFiles := files.filterChangedLocalFiles()
    missingFiles := files.filterMissingRemoteFiles()

    fmt.Fprintf(args.Out, "Found %d local files and %d remote files\n", len(files.local), len(files.remote))

    // Ensure that there is enough free space on drive
    if ok, msg := self.checkRemoteFreeSpace(missingFiles, changedFiles); !ok {
        return fmt.Errorf(msg)
    }

    // Ensure that we don't overwrite any remote changes
    if args.Resolution == NoResolution {
        err = ensureNoRemoteModifications(changedFiles)
        if err != nil {
            return fmt.Errorf("Conflict detected!\nThe following files have changed and the remote file are newer than it's local counterpart:\n\n%s\nNo conflict resolution was given, aborting...", err)
        }
    }

    // Create missing directories
    files, err = self.createMissingRemoteDirs(files, args)
    if err != nil {
        return err
    }

    // Upload missing files
    err = self.uploadMissingFiles(missingFiles, files, args)
    if err != nil {
        return err
    }

    // Update modified files
    err = self.updateChangedFiles(changedFiles, rootDir, args)
    if err != nil {
        return err
    }

    // Delete extraneous files on drive
    if args.DeleteExtraneous {
        err = self.deleteExtraneousRemoteFiles(files, args)
        if err != nil {
            return err
        }
    }
    fmt.Fprintf(args.Out, "Sync finished in %s\n", time.Since(started))

    return nil
}

func (self *Drive) prepareSyncRoot(args UploadSyncArgs) (*drive.File, error) {
    fields := []googleapi.Field{"id", "name", "mimeType", "appProperties"}
    f, err := self.service.Files.Get(args.RootId).Fields(fields...).Do()
    if err != nil {
        return nil, fmt.Errorf("Failed to find root dir: %s", err)
    }

    // Ensure file is a directory
    if !isDir(f) {
        return nil, fmt.Errorf("Provided root id is not a directory")
    }

    // Return directory if syncRoot property is already set
    if _, ok := f.AppProperties["isSyncRoot"]; ok {
        return f, nil
    }

    // This is the first time this directory have been used for sync
    // Check if the directory is empty
    isEmpty, err := self.dirIsEmpty(f.Id)
    if err != nil {
        return nil, fmt.Errorf("Failed to check if root dir is empty: %s", err)
    }

    // Ensure that the directory is empty
    if !isEmpty {
        return nil, fmt.Errorf("Root directoy is not empty, the initial sync requires an empty directory")
    }

    // Update directory with syncRoot property
    dstFile := &drive.File{
        AppProperties: map[string]string{"isSyncRoot": "true"},
    }

    f, err = self.service.Files.Update(f.Id, dstFile).Fields(fields...).Do()
    if err != nil {
        return nil, fmt.Errorf("Failed to update root directory: %s", err)
    }

    return f, nil
}

func (self *Drive) createMissingRemoteDirs(files *syncFiles, args UploadSyncArgs) (*syncFiles, error) {
    missingDirs := files.filterMissingRemoteDirs()
    missingCount := len(missingDirs)

    if missingCount > 0 {
        fmt.Fprintf(args.Out, "\n%d remote directories are missing\n", missingCount)
    }

    // Sort directories so that the dirs with the shortest path comes first
    sort.Sort(byLocalPathLength(missingDirs))

    for i, lf := range missingDirs {
        parentPath := parentFilePath(lf.relPath)
        parent, ok := files.findRemoteByPath(parentPath)
        if !ok {
            return nil, fmt.Errorf("Could not find remote directory with path '%s'", parentPath)
        }

        fmt.Fprintf(args.Out, "[%04d/%04d] Creating directory %s\n", i + 1, missingCount, filepath.Join(files.root.file.Name, lf.relPath))

        f, err := self.createMissingRemoteDir(createMissingRemoteDirArgs{
            name: lf.info.Name(),
            parentId: parent.file.Id,
            rootId: args.RootId,
            dryRun: args.DryRun,
            try: 0,
        })
        if err != nil {
            return nil, err
        }

        files.remote = append(files.remote, &RemoteFile{
            relPath: lf.relPath,
            file: f,
        })
    }

    return files, nil
}

type createMissingRemoteDirArgs struct {
    name string
    parentId string
    rootId string
    dryRun bool
    try int
}

func (self *Drive) uploadMissingFiles(missingFiles []*LocalFile, files *syncFiles, args UploadSyncArgs) error {
    missingCount := len(missingFiles)

    if missingCount > 0 {
        fmt.Fprintf(args.Out, "\n%d remote files are missing\n", missingCount)
    }

    for i, lf := range missingFiles {
        parentPath := parentFilePath(lf.relPath)
        parent, ok := files.findRemoteByPath(parentPath)
        if !ok {
            return fmt.Errorf("Could not find remote directory with path '%s'", parentPath)
        }

        fmt.Fprintf(args.Out, "[%04d/%04d] Uploading %s -> %s\n", i + 1, missingCount, lf.relPath, filepath.Join(files.root.file.Name, lf.relPath))

        err := self.uploadMissingFile(parent.file.Id, lf, args, 0)
        if err != nil {
            return err
        }
    }

    return nil
}

func (self *Drive) updateChangedFiles(changedFiles []*changedFile, root *drive.File, args UploadSyncArgs) error {
    changedCount := len(changedFiles)

    if changedCount > 0 {
        fmt.Fprintf(args.Out, "\n%d local files has changed\n", changedCount)
    }

    for i, cf := range changedFiles {
        if skip, reason := checkRemoteConflict(cf, args.Resolution); skip {
            fmt.Fprintf(args.Out, "[%04d/%04d] Skipping %s (%s)\n", i + 1, changedCount, cf.local.relPath, reason)
            continue
        }

        fmt.Fprintf(args.Out, "[%04d/%04d] Updating %s -> %s\n", i + 1, changedCount, cf.local.relPath, filepath.Join(root.Name, cf.local.relPath))

        err := self.updateChangedFile(cf, args, 0)
        if err != nil {
            return err
        }
    }

    return nil
}

func (self *Drive) deleteExtraneousRemoteFiles(files *syncFiles, args UploadSyncArgs) error {
    extraneousFiles := files.filterExtraneousRemoteFiles()
    extraneousCount := len(extraneousFiles)

    if extraneousCount > 0 {
        fmt.Fprintf(args.Out, "\n%d remote files are extraneous\n", extraneousCount)
    }

    // Sort files so that the files with the longest path comes first
    sort.Sort(sort.Reverse(byRemotePathLength(extraneousFiles)))

    for i, rf := range extraneousFiles {
        fmt.Fprintf(args.Out, "[%04d/%04d] Deleting %s\n", i + 1, extraneousCount, filepath.Join(files.root.file.Name, rf.relPath))

        err := self.deleteRemoteFile(rf, args, 0)
        if err != nil {
            return err
        }
    }

    return nil
}

func (self *Drive) createMissingRemoteDir(args createMissingRemoteDirArgs) (*drive.File, error) {
    dstFile := &drive.File{
        Name: args.name,
        MimeType: DirectoryMimeType,
        Parents: []string{args.parentId},
        AppProperties: map[string]string{"syncRootId": args.rootId},
    }

    if args.dryRun {
        return dstFile, nil
    }

    f, err := self.service.Files.Create(dstFile).Do()
    if err != nil {
        if isBackendError(err) && args.try < MaxBackendErrorRetries {
            exponentialBackoffSleep(args.try)
            args.try++
            return self.createMissingRemoteDir(args)
        } else {
            return nil, fmt.Errorf("Failed to create directory: %s", err)
        }
    }

    return f, nil
}

func (self *Drive) uploadMissingFile(parentId string, lf *LocalFile, args UploadSyncArgs, try int) error {
    if args.DryRun {
        return nil
    }

    srcFile, err := os.Open(lf.absPath)
    if err != nil {
        return fmt.Errorf("Failed to open file: %s", err)
    }

    // Close file on function exit
    defer srcFile.Close()

    // Instantiate drive file
    dstFile := &drive.File{
        Name: lf.info.Name(),
        Parents: []string{parentId},
        AppProperties: map[string]string{"syncRootId": args.RootId},
    }

    // Chunk size option
    chunkSize := googleapi.ChunkSize(int(args.ChunkSize))

    // Wrap file in progress reader
    srcReader := getProgressReader(srcFile, args.Progress, lf.info.Size())

    _, err = self.service.Files.Create(dstFile).Fields("id", "name", "size", "md5Checksum").Media(srcReader, chunkSize).Do()
    if err != nil {
        if isBackendError(err) && try < MaxBackendErrorRetries {
            exponentialBackoffSleep(try)
            try++
            return self.uploadMissingFile(parentId, lf, args, try)
        } else {
            return fmt.Errorf("Failed to upload file: %s", err)
        }
    }

    return nil
}

func (self *Drive) updateChangedFile(cf *changedFile, args UploadSyncArgs, try int) error {
    if args.DryRun {
        return nil
    }

    srcFile, err := os.Open(cf.local.absPath)
    if err != nil {
        return fmt.Errorf("Failed to open file: %s", err)
    }

    // Close file on function exit
    defer srcFile.Close()

    // Instantiate drive file
    dstFile := &drive.File{}

    // Chunk size option
    chunkSize := googleapi.ChunkSize(int(args.ChunkSize))

    // Wrap file in progress reader
    srcReader := getProgressReader(srcFile, args.Progress, cf.local.info.Size())

    _, err = self.service.Files.Update(cf.remote.file.Id, dstFile).Media(srcReader, chunkSize).Do()
    if err != nil {
        if isBackendError(err) && try < MaxBackendErrorRetries {
            exponentialBackoffSleep(try)
            try++
            return self.updateChangedFile(cf, args, try)
        } else {
            return fmt.Errorf("Failed to update file: %s", err)
        }
    }

    return nil
}

func (self *Drive) deleteRemoteFile(rf *RemoteFile, args UploadSyncArgs, try int) error {
    if args.DryRun {
        return nil
    }

    err := self.service.Files.Delete(rf.file.Id).Do()
    if err != nil {
        if isBackendError(err) && try < MaxBackendErrorRetries {
            exponentialBackoffSleep(try)
            try++
            return self.deleteRemoteFile(rf, args, try)
        } else {
            return fmt.Errorf("Failed to delete file: %s", err)
        }
    }

    return nil
}

func (self *Drive) dirIsEmpty(id string) (bool, error) {
    query := fmt.Sprintf("'%s' in parents", id)
    fileList, err := self.service.Files.List().Q(query).Do()
    if err != nil {
        return false, fmt.Errorf("Empty dir check failed: ", err)
    }

    return len(fileList.Files) == 0, nil
}

func checkRemoteConflict(cf *changedFile, resolution ConflictResolution) (bool, string) {
    // No conflict unless remote file was last modified
    if cf.compareModTime() != RemoteLastModified {
        return false, ""
    }

    // Don't skip if want to keep the local file
    if resolution == KeepLocal {
        return false, ""
    }

    // Skip if we want to keep the remote file
    if resolution == KeepRemote {
        return true, "conflicting file, keeping remote file"
    }

    if resolution == KeepLargest {
        largest := cf.compareSize()

        // Skip if the remote file is largest
        if largest == RemoteLargestSize {
            return true, "conflicting file, remote file is largest, keeping remote"
        }

        // Don't skip if the local file is largest
        if largest == LocalLargestSize {
            return false, ""
        }

        // Keep remote if both files have the same size
        if largest == EqualSize {
            return true, "conflicting file, file sizes are equal, keeping remote"
        }
    }

    // The conditionals above should cover all cases,
    // unless the programmer did something wrong,
    // in which case we default to being non-destructive and skip the file
    return true, "conflicting file, unhandled case"
}

func ensureNoRemoteModifications(files []*changedFile) error {
    conflicts := findRemoteConflicts(files)
    if len(conflicts) == 0 {
        return nil
    }

    buffer := bytes.NewBufferString("")
    formatConflicts(conflicts, buffer)
    return fmt.Errorf(buffer.String())
}

func (self *Drive) checkRemoteFreeSpace(missingFiles []*LocalFile, changedFiles []*changedFile) (bool, string) {
    about, err := self.service.About.Get().Fields("storageQuota").Do()
    if err != nil {
        return false, fmt.Sprintf("Failed to determine free space: %s", err)
    }

    quota := about.StorageQuota
    if quota.Limit == 0 {
        return true, ""
    }

    freeSpace := quota.Limit - quota.Usage

    var totalSize int64

    for _, lf := range missingFiles {
        totalSize += lf.Size()
    }

    for _, cf := range changedFiles {
        totalSize += cf.local.Size()
    }

    if totalSize > freeSpace {
        return false, fmt.Sprintf("Not enough free space, have %s need %s", formatSize(freeSpace, false), formatSize(totalSize, false))
    }

    return true, ""
}
<|endoftext|>


<commit_before>package driver

import (
	"bufio"
	"context"
	"fmt"
	"io"
	"strings"
	"time"

	"github.com/estesp/bucketbench/utils"
	log "github.com/sirupsen/logrus"
)

const defaultDockerBinary = "docker"

var dockerProcNames = []string{
	"dockerd",
	"docker-containerd",
	"docker-containerd-shim",
	"docker-proxy",
}

// DockerCLIDriver is an implementation of the driver interface for the Docker engine.
// IMPORTANT: This implementation does not protect instance metadata for thread safely.
// At this time there is no understood use case for multi-threaded use of this implementation.
type DockerCLIDriver struct {
	dockerBinary string
	dockerInfo   string
	logDriver    string
	logOpts      map[string]string
	streamStats  bool
}

// DockerContainer is an implementation of the container metadata needed for docker
type DockerContainer struct {
	name        string
	imageName   string
	cmdOverride string
	detached    bool
	trace       bool
}

// NewDockerCLIDriver creates an instance of the docker driver, providing a path to the docker client binary
func NewDockerCLIDriver(ctx context.Context, config *Config) (Driver, error) {
	binaryPath := config.Path
	if binaryPath == "" {
		binaryPath = defaultDockerBinary
	}

	resolvedBinPath, err := utils.ResolveBinary(binaryPath)
	if err != nil {
		return &DockerCLIDriver{}, err
	}

	driver := &DockerCLIDriver{
		dockerBinary: resolvedBinPath,
		logDriver:    config.LogDriver,
		logOpts:      config.LogOpts,
		streamStats:  config.StreamStats,
	}

	info, err := driver.Info(ctx)
	if err != nil {
		return nil, err
	}

	log.Debugf("running docker CLI driver: '%s', log driver: '%s'", info, config.LogDriver)
	return driver, nil
}

// newDockerContainer creates the metadata object of a docker-specific container with
// image name, container runtime name, and any required additional information
func newDockerContainer(name, image, cmd string, detached bool, trace bool) Container {
	return &DockerContainer{
		name:        name,
		imageName:   image,
		cmdOverride: cmd,
		detached:    detached,
		trace:       trace,
	}
}

// Name returns the name of the container
func (c *DockerContainer) Name() string {
	return c.name
}

// Detached returns whether the container should be started in detached mode
func (c *DockerContainer) Detached() bool {
	return c.detached
}

// Trace returns whether the container should be started with tracing enabled
func (c *DockerContainer) Trace() bool {
	return c.trace
}

// Image returns the image name that Docker will use
func (c *DockerContainer) Image() string {
	return c.imageName
}

// Command returns the optional overriding command that Docker will use
// when executing a container based on this container's image
func (c *DockerContainer) Command() string {
	return c.cmdOverride
}

//GetPodID return pod-id associated with container.
//only used by CRI-based drivers
func (c *DockerContainer) GetPodID() string {
	return ""
}

// Type returns a driver.Type to indentify the driver implementation
func (d *DockerCLIDriver) Type() Type {
	return DockerCLI
}

// Path returns the binary path of the docker binary in use
func (d *DockerCLIDriver) Path() string {
	return d.dockerBinary
}

// Close allows the driver to handle any resource free/connection closing
// as necessary. Docker CLI has no need to perform any actions on close.
func (d *DockerCLIDriver) Close() error {
	return nil
}

// PID returns a process ID of Docker daemon
func (d *DockerCLIDriver) PID() (int, error) {
	return getDockerPID("")
}

// Wait will block until container stop
func (d *DockerCLIDriver) Wait(ctx context.Context, ctr Container) (string, time.Duration, error) {
	return utils.ExecTimedCmd(ctx, d.dockerBinary, "wait "+ctr.Name())
}

// Info returns
func (d *DockerCLIDriver) Info(ctx context.Context) (string, error) {
	if d.dockerInfo != "" {
		return d.dockerInfo, nil
	}

	infoStart := "docker driver (binary: " + d.dockerBinary + ")\n"
	version, err := utils.ExecCmd(ctx, d.dockerBinary, "version")
	info, err := utils.ExecCmd(ctx, d.dockerBinary, "info")
	if err != nil {
		return "", fmt.Errorf("error trying to retrieve docker daemon info: %v", err)
	}

	d.dockerInfo = infoStart + parseDaemonInfo(version, info)
	return d.dockerInfo, nil
}

// Create will create a container instance matching the specific needs
// of a driver
func (d *DockerCLIDriver) Create(ctx context.Context, name, image, cmdOverride string, detached bool, trace bool) (Container, error) {
	return newDockerContainer(name, image, cmdOverride, detached, trace), nil
}

// Clean will clean the environment; removing any exited containers
func (d *DockerCLIDriver) Clean(ctx context.Context) error {
	// clean up any containers from a prior run
	log.Info("Docker: Stopping any running containers created during bucketbench runs")
	cmd := fmt.Sprintf("docker stop `docker ps -qf name=%s`", ContainerNamePrefix)
	out, err := utils.ExecShellCmd(ctx, cmd)
	if err != nil {
		// first make sure the error isn't simply that there were no
		// containers to stop:
		if !strings.Contains(out, "requires at least 1 argument") {
			log.Warnf("Docker: Failed to stop running bb-ctr-* containers: %v (output: %s)", err, out)
		}
	}
	log.Info("Docker: Removing exited containers from bucketbench runs")
	cmd = fmt.Sprintf("docker rm -f `docker ps -aqf name=%s`", ContainerNamePrefix)
	out, err = utils.ExecShellCmd(ctx, cmd)
	if err != nil {
		// first make sure the error isn't simply that there were no
		// exited containers to remove:
		if !strings.Contains(out, "requires at least 1 argument") {
			log.Warnf("Docker: Failed to remove exited bb-ctr-* containers: %v (output: %s)", err, out)
		}
	}
	return nil
}

// Run will execute a container using the driver
func (d *DockerCLIDriver) Run(ctx context.Context, ctr Container) (string, time.Duration, error) {
	args := []string{"run"}

	if ctr.Detached() {
		args = append(args, "-d")
	}

	if d.logDriver != "" {
		args = append(args, "--log-driver", d.logDriver)

		for name, value := range d.logOpts {
			args = append(args, "--log-opt", fmt.Sprintf("%s=%s", name, value))
		}
	}

	args = append(args, "--name", ctr.Name(), ctr.Image())

	if ctr.Command() != "" {
		args = append(args, ctr.Command())
	}

	return utils.ExecTimedCmd(ctx, d.dockerBinary, strings.Join(args, " "))
}

// Stop will stop/kill a container
func (d *DockerCLIDriver) Stop(ctx context.Context, ctr Container) (string, time.Duration, error) {
	return utils.ExecTimedCmd(ctx, d.dockerBinary, "kill "+ctr.Name())
}

// Remove will remove a container
func (d *DockerCLIDriver) Remove(ctx context.Context, ctr Container) (string, time.Duration, error) {
	return utils.ExecTimedCmd(ctx, d.dockerBinary, "rm "+ctr.Name())
}

// Pause will pause a container
func (d *DockerCLIDriver) Pause(ctx context.Context, ctr Container) (string, time.Duration, error) {
	return utils.ExecTimedCmd(ctx, d.dockerBinary, "pause "+ctr.Name())
}

// Unpause will unpause/resume a container
func (d *DockerCLIDriver) Unpause(ctx context.Context, ctr Container) (string, time.Duration, error) {
	return utils.ExecTimedCmd(ctx, d.dockerBinary, "unpause "+ctr.Name())
}

// Stats returns stats data from daemon for container
func (d *DockerCLIDriver) Stats(ctx context.Context, ctr Container) (io.ReadCloser, error) {
	var args string
	if d.streamStats {
		args = "stats " + ctr.Name()
	} else {
		args = "stats --no-stream " + ctr.Name()
	}

	return utils.ExecCmdStream(ctx, d.dockerBinary, args)
}

// ProcNames returns the list of process names contributing to mem/cpu usage during overhead benchmark
func (d *DockerCLIDriver) ProcNames() []string {
	return dockerProcNames
}

// return a condensed string of version and daemon information
func parseDaemonInfo(version, info string) string {
	var (
		clientVer string
		clientAPI string
		serverVer string
	)
	vReader := strings.NewReader(version)
	vScan := bufio.NewScanner(vReader)

	for vScan.Scan() {
		line := vScan.Text()
		parts := strings.Split(line, ":")
		switch strings.TrimSpace(parts[0]) {
		case "Version":
			if clientVer == "" {
				// first time is client
				clientVer = strings.TrimSpace(parts[1])
			} else {
				serverVer = strings.TrimSpace(parts[1])
			}
		case "API version":
			if clientAPI == "" {
				// first instance is client
				clientAPI = parts[1]
				clientVer = clientVer + "|API:" + strings.TrimSpace(parts[1])
			} else {
				serverVer = serverVer + "|API:" + strings.TrimSpace(parts[1])
			}
		default:
		}

	}
	iReader := strings.NewReader(info)
	iScan := bufio.NewScanner(iReader)

	for iScan.Scan() {
		line := iScan.Text()
		parts := strings.Split(line, ":")
		switch strings.TrimSpace(parts[0]) {
		case "Kernel Version":
			serverVer = serverVer + "|Kernel:" + strings.TrimSpace(parts[1])
		case "Storage Driver":
			serverVer = serverVer + "|Storage:" + strings.TrimSpace(parts[1])
		case "Backing Filesystem":
			serverVer = serverVer + "|BackingFS:" + strings.TrimSpace(parts[1])
		default:
		}

	}
	return fmt.Sprintf("[CLIENT:%s][SERVER:%s]", clientVer, serverVer)
}
<commit_msg>driver: use stop instead of kill in DockerCLI<commit_after>package driver

import (
	"bufio"
	"context"
	"fmt"
	"io"
	"strings"
	"time"

	"github.com/estesp/bucketbench/utils"
	log "github.com/sirupsen/logrus"
)

const defaultDockerBinary = "docker"

var dockerProcNames = []string{
	"dockerd",
	"docker-containerd",
	"docker-containerd-shim",
	"docker-proxy",
}

// DockerCLIDriver is an implementation of the driver interface for the Docker engine.
// IMPORTANT: This implementation does not protect instance metadata for thread safely.
// At this time there is no understood use case for multi-threaded use of this implementation.
type DockerCLIDriver struct {
	dockerBinary string
	dockerInfo   string
	logDriver    string
	logOpts      map[string]string
	streamStats  bool
}

// DockerContainer is an implementation of the container metadata needed for docker
type DockerContainer struct {
	name        string
	imageName   string
	cmdOverride string
	detached    bool
	trace       bool
}

// NewDockerCLIDriver creates an instance of the docker driver, providing a path to the docker client binary
func NewDockerCLIDriver(ctx context.Context, config *Config) (Driver, error) {
	binaryPath := config.Path
	if binaryPath == "" {
		binaryPath = defaultDockerBinary
	}

	resolvedBinPath, err := utils.ResolveBinary(binaryPath)
	if err != nil {
		return &DockerCLIDriver{}, err
	}

	driver := &DockerCLIDriver{
		dockerBinary: resolvedBinPath,
		logDriver:    config.LogDriver,
		logOpts:      config.LogOpts,
		streamStats:  config.StreamStats,
	}

	info, err := driver.Info(ctx)
	if err != nil {
		return nil, err
	}

	log.Debugf("running docker CLI driver: '%s', log driver: '%s'", info, config.LogDriver)
	return driver, nil
}

// newDockerContainer creates the metadata object of a docker-specific container with
// image name, container runtime name, and any required additional information
func newDockerContainer(name, image, cmd string, detached bool, trace bool) Container {
	return &DockerContainer{
		name:        name,
		imageName:   image,
		cmdOverride: cmd,
		detached:    detached,
		trace:       trace,
	}
}

// Name returns the name of the container
func (c *DockerContainer) Name() string {
	return c.name
}

// Detached returns whether the container should be started in detached mode
func (c *DockerContainer) Detached() bool {
	return c.detached
}

// Trace returns whether the container should be started with tracing enabled
func (c *DockerContainer) Trace() bool {
	return c.trace
}

// Image returns the image name that Docker will use
func (c *DockerContainer) Image() string {
	return c.imageName
}

// Command returns the optional overriding command that Docker will use
// when executing a container based on this container's image
func (c *DockerContainer) Command() string {
	return c.cmdOverride
}

//GetPodID return pod-id associated with container.
//only used by CRI-based drivers
func (c *DockerContainer) GetPodID() string {
	return ""
}

// Type returns a driver.Type to indentify the driver implementation
func (d *DockerCLIDriver) Type() Type {
	return DockerCLI
}

// Path returns the binary path of the docker binary in use
func (d *DockerCLIDriver) Path() string {
	return d.dockerBinary
}

// Close allows the driver to handle any resource free/connection closing
// as necessary. Docker CLI has no need to perform any actions on close.
func (d *DockerCLIDriver) Close() error {
	return nil
}

// PID returns a process ID of Docker daemon
func (d *DockerCLIDriver) PID() (int, error) {
	return getDockerPID("")
}

// Wait will block until container stop
func (d *DockerCLIDriver) Wait(ctx context.Context, ctr Container) (string, time.Duration, error) {
	return utils.ExecTimedCmd(ctx, d.dockerBinary, "wait "+ctr.Name())
}

// Info returns
func (d *DockerCLIDriver) Info(ctx context.Context) (string, error) {
	if d.dockerInfo != "" {
		return d.dockerInfo, nil
	}

	infoStart := "docker driver (binary: " + d.dockerBinary + ")\n"
	version, err := utils.ExecCmd(ctx, d.dockerBinary, "version")
	info, err := utils.ExecCmd(ctx, d.dockerBinary, "info")
	if err != nil {
		return "", fmt.Errorf("error trying to retrieve docker daemon info: %v", err)
	}

	d.dockerInfo = infoStart + parseDaemonInfo(version, info)
	return d.dockerInfo, nil
}

// Create will create a container instance matching the specific needs
// of a driver
func (d *DockerCLIDriver) Create(ctx context.Context, name, image, cmdOverride string, detached bool, trace bool) (Container, error) {
	return newDockerContainer(name, image, cmdOverride, detached, trace), nil
}

// Clean will clean the environment; removing any exited containers
func (d *DockerCLIDriver) Clean(ctx context.Context) error {
	// clean up any containers from a prior run
	log.Info("Docker: Stopping any running containers created during bucketbench runs")
	cmd := fmt.Sprintf("docker stop `docker ps -qf name=%s`", ContainerNamePrefix)
	out, err := utils.ExecShellCmd(ctx, cmd)
	if err != nil {
		// first make sure the error isn't simply that there were no
		// containers to stop:
		if !strings.Contains(out, "requires at least 1 argument") {
			log.Warnf("Docker: Failed to stop running bb-ctr-* containers: %v (output: %s)", err, out)
		}
	}
	log.Info("Docker: Removing exited containers from bucketbench runs")
	cmd = fmt.Sprintf("docker rm -f `docker ps -aqf name=%s`", ContainerNamePrefix)
	out, err = utils.ExecShellCmd(ctx, cmd)
	if err != nil {
		// first make sure the error isn't simply that there were no
		// exited containers to remove:
		if !strings.Contains(out, "requires at least 1 argument") {
			log.Warnf("Docker: Failed to remove exited bb-ctr-* containers: %v (output: %s)", err, out)
		}
	}
	return nil
}

// Run will execute a container using the driver
func (d *DockerCLIDriver) Run(ctx context.Context, ctr Container) (string, time.Duration, error) {
	args := []string{"run"}

	if ctr.Detached() {
		args = append(args, "-d")
	}

	if d.logDriver != "" {
		args = append(args, "--log-driver", d.logDriver)

		for name, value := range d.logOpts {
			args = append(args, "--log-opt", fmt.Sprintf("%s=%s", name, value))
		}
	}

	args = append(args, "--name", ctr.Name(), ctr.Image())

	if ctr.Command() != "" {
		args = append(args, ctr.Command())
	}

	return utils.ExecTimedCmd(ctx, d.dockerBinary, strings.Join(args, " "))
}

// Stop will stop a container
func (d *DockerCLIDriver) Stop(ctx context.Context, ctr Container) (string, time.Duration, error) {
	return utils.ExecTimedCmd(ctx, d.dockerBinary, "stop "+ctr.Name())
}

// Remove will remove a container
func (d *DockerCLIDriver) Remove(ctx context.Context, ctr Container) (string, time.Duration, error) {
	return utils.ExecTimedCmd(ctx, d.dockerBinary, "rm "+ctr.Name())
}

// Pause will pause a container
func (d *DockerCLIDriver) Pause(ctx context.Context, ctr Container) (string, time.Duration, error) {
	return utils.ExecTimedCmd(ctx, d.dockerBinary, "pause "+ctr.Name())
}

// Unpause will unpause/resume a container
func (d *DockerCLIDriver) Unpause(ctx context.Context, ctr Container) (string, time.Duration, error) {
	return utils.ExecTimedCmd(ctx, d.dockerBinary, "unpause "+ctr.Name())
}

// Stats returns stats data from daemon for container
func (d *DockerCLIDriver) Stats(ctx context.Context, ctr Container) (io.ReadCloser, error) {
	var args string
	if d.streamStats {
		args = "stats " + ctr.Name()
	} else {
		args = "stats --no-stream " + ctr.Name()
	}

	return utils.ExecCmdStream(ctx, d.dockerBinary, args)
}

// ProcNames returns the list of process names contributing to mem/cpu usage during overhead benchmark
func (d *DockerCLIDriver) ProcNames() []string {
	return dockerProcNames
}

// return a condensed string of version and daemon information
func parseDaemonInfo(version, info string) string {
	var (
		clientVer string
		clientAPI string
		serverVer string
	)
	vReader := strings.NewReader(version)
	vScan := bufio.NewScanner(vReader)

	for vScan.Scan() {
		line := vScan.Text()
		parts := strings.Split(line, ":")
		switch strings.TrimSpace(parts[0]) {
		case "Version":
			if clientVer == "" {
				// first time is client
				clientVer = strings.TrimSpace(parts[1])
			} else {
				serverVer = strings.TrimSpace(parts[1])
			}
		case "API version":
			if clientAPI == "" {
				// first instance is client
				clientAPI = parts[1]
				clientVer = clientVer + "|API:" + strings.TrimSpace(parts[1])
			} else {
				serverVer = serverVer + "|API:" + strings.TrimSpace(parts[1])
			}
		default:
		}

	}
	iReader := strings.NewReader(info)
	iScan := bufio.NewScanner(iReader)

	for iScan.Scan() {
		line := iScan.Text()
		parts := strings.Split(line, ":")
		switch strings.TrimSpace(parts[0]) {
		case "Kernel Version":
			serverVer = serverVer + "|Kernel:" + strings.TrimSpace(parts[1])
		case "Storage Driver":
			serverVer = serverVer + "|Storage:" + strings.TrimSpace(parts[1])
		case "Backing Filesystem":
			serverVer = serverVer + "|BackingFS:" + strings.TrimSpace(parts[1])
		default:
		}

	}
	return fmt.Sprintf("[CLIENT:%s][SERVER:%s]", clientVer, serverVer)
}
<|endoftext|>


<commit_before>package handler

import (
	"fmt"
	"net/http"
	"net/url"
	"sort"

	"github.com/skygeario/skygear-server/pkg/auth/dependency/oauth/protocol"
	coreurl "github.com/skygeario/skygear-server/pkg/core/url"
)

type AuthorizationResult interface {
	WriteResponse(rw http.ResponseWriter, r *http.Request)
	IsInternalError() bool
}

type (
	authorizationResultRedirect struct {
		RedirectURI *url.URL
		Response    protocol.AuthorizationResponse
	}
	authorizationResultError struct {
		RedirectURI   *url.URL
		InternalError bool
		Response      protocol.ErrorResponse
	}
	authorizationResultRequireAuthn struct {
		AuthenticateURI *url.URL
		AuthorizeURI    *url.URL
		Request         protocol.AuthorizationRequest
	}
)

func (a authorizationResultRedirect) WriteResponse(rw http.ResponseWriter, r *http.Request) {
	redirectURI := coreurl.WithQueryParamsAdded(a.RedirectURI, a.Response)
	http.Redirect(rw, r, redirectURI.String(), http.StatusFound)
}

func (a authorizationResultRedirect) IsInternalError() bool {
	return false
}

func (a authorizationResultError) WriteResponse(rw http.ResponseWriter, r *http.Request) {
	if a.RedirectURI != nil {
		redirectURI := coreurl.WithQueryParamsAdded(a.RedirectURI, a.Response)
		http.Redirect(rw, r, redirectURI.String(), http.StatusFound)
	} else {
		err := "Invalid OAuth authorization request:\n"
		keys := make([]string, 0, len(a.Response))
		for k := range a.Response {
			keys = append(keys, k)
		}
		sort.Strings(keys)
		for i, k := range keys {
			if i != 0 {
				err += "\n"
			}
			err += fmt.Sprintf("%s: %s", k, a.Response[k])
		}
		http.Error(rw, err, http.StatusBadRequest)
	}
}

func (a authorizationResultError) IsInternalError() bool {
	return a.InternalError
}

func (a authorizationResultRequireAuthn) WriteResponse(rw http.ResponseWriter, r *http.Request) {
	authorizeURI := coreurl.WithQueryParamsAdded(a.AuthorizeURI, a.Request)

	q := map[string]string{
		"redirect_uri": authorizeURI.String(),
		"client_id":    a.Request.ClientID(),
	}
	if uiLocales, ok := a.Request["ui_locales"]; ok {
		q["ui_locales"] = uiLocales
	}
	authenticateURI := coreurl.WithQueryParamsAdded(a.AuthenticateURI, q)

	http.Redirect(rw, r, authenticateURI.String(), http.StatusFound)
}

func (a authorizationResultRequireAuthn) IsInternalError() bool {
	return false
}
<commit_msg>Use html meta tag / js to redirect result after authorization<commit_after>package handler

import (
	"fmt"
	"html/template"
	"net/http"
	"net/url"
	"sort"

	"github.com/skygeario/skygear-server/pkg/auth/dependency/oauth/protocol"
	coreurl "github.com/skygeario/skygear-server/pkg/core/url"
)

const AuthorizationResultHTML = `<!DOCTYPE html>
<head>
<meta http-equiv="refresh" content="0;url={{ .redirect_uri }}" />
</head>
<script>
window.location.href = "{{ .redirect_uri }}"
</script>`

type AuthorizationResult interface {
	WriteResponse(rw http.ResponseWriter, r *http.Request)
	IsInternalError() bool
}

type (
	authorizationResultRedirect struct {
		RedirectURI *url.URL
		Response    protocol.AuthorizationResponse
	}
	authorizationResultError struct {
		RedirectURI   *url.URL
		InternalError bool
		Response      protocol.ErrorResponse
	}
	authorizationResultRequireAuthn struct {
		AuthenticateURI *url.URL
		AuthorizeURI    *url.URL
		Request         protocol.AuthorizationRequest
	}
)

func (a authorizationResultRedirect) WriteResponse(rw http.ResponseWriter, r *http.Request) {
	redirectURI := coreurl.WithQueryParamsAdded(a.RedirectURI, a.Response)
	redirect(rw, redirectURI.String())
}

func (a authorizationResultRedirect) IsInternalError() bool {
	return false
}

func (a authorizationResultError) WriteResponse(rw http.ResponseWriter, r *http.Request) {
	if a.RedirectURI != nil {
		redirectURI := coreurl.WithQueryParamsAdded(a.RedirectURI, a.Response)
		redirect(rw, redirectURI.String())
	} else {
		err := "Invalid OAuth authorization request:\n"
		keys := make([]string, 0, len(a.Response))
		for k := range a.Response {
			keys = append(keys, k)
		}
		sort.Strings(keys)
		for i, k := range keys {
			if i != 0 {
				err += "\n"
			}
			err += fmt.Sprintf("%s: %s", k, a.Response[k])
		}
		http.Error(rw, err, http.StatusBadRequest)
	}
}

func (a authorizationResultError) IsInternalError() bool {
	return a.InternalError
}

func (a authorizationResultRequireAuthn) WriteResponse(rw http.ResponseWriter, r *http.Request) {
	authorizeURI := coreurl.WithQueryParamsAdded(a.AuthorizeURI, a.Request)

	q := map[string]string{
		"redirect_uri": authorizeURI.String(),
		"client_id":    a.Request.ClientID(),
	}
	if uiLocales, ok := a.Request["ui_locales"]; ok {
		q["ui_locales"] = uiLocales
	}
	authenticateURI := coreurl.WithQueryParamsAdded(a.AuthenticateURI, q)

	http.Redirect(rw, r, authenticateURI.String(), http.StatusFound)
}

func (a authorizationResultRequireAuthn) IsInternalError() bool {
	return false
}

func redirect(rw http.ResponseWriter, redirectURI string) {
	rw.Header().Set("Location", redirectURI)
	rw.Header().Set("Content-Type", "text/html; charset=utf-8")
	rw.WriteHeader(http.StatusFound)

	tmpl, err := template.New("authorization_result").Parse(AuthorizationResultHTML)
	if err != nil {
		panic("oauth: invalid authorization result page template")
	}
	err = tmpl.Execute(rw, map[string]string{
		"redirect_uri": redirectURI,
	})
	if err != nil {
		panic("oauth: failed to load authorization result page")
	}
}
<|endoftext|>