<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
	"net/http"
	"strconv"
	"regexp"
	"errors"
	"sync"
	"text/template"
	"bytes"
	"io/ioutil"
	"time"
)

type VMInformation struct {
	mux sync.Mutex
	// This map is id => state (should be an enum I guess)
	Vms map[int]string
}

func (v *VMInformation) addVM(vmId int, state string) (error) {
	// VMs < 50 are reserved for administrative use
	if (vmId < 50 || vmId > 254) {
		return errors.New("invalid")
	}

	// Lock for our further checks
	v.mux.Lock()
	defer v.mux.Unlock()

	// Get the list of VMs and confirm whether this is one that's already running
	if _, exists := v.Vms[vmId]; exists {
		return errors.New("in use")
	}

	// It's both valid and not in use!
	v.Vms[vmId] = state

	return nil // no errors
}

func (v *VMInformation) updateVM(vmId int, state string) (error) {
	v.mux.Lock()
	defer v.mux.Unlock()

	// If the VM doesn't already exist in the map, return an error
	if _, ok := v.Vms[vmId]; !ok {
		return errors.New("Invalid vmId specified")
	}

	v.Vms[vmId] = state

	return nil
}

func (v *VMInformation) sync() (error) {
	// Shell out to get a list of screen sessions, which are VMs
	out, err := exec.Command("screen", "-ls").Output()
	if err != nil {
		return errors.New("Error running screen")
	}

	// Our new map
	newvms := make(map[int]string)

	// Regex out the running VM IDs
	re := regexp.MustCompile(`\b\.vm([0-9]+)\b`)

	matches := re.FindAllStringSubmatch(string(out), -1)

	// We only need to start locking now, to ensure we read a valid state
	v.mux.Lock()
	defer v.mux.Unlock()

	for i := 0; i < len(matches); i++ {
		// Ignore invalid VMs
		id, err := strconv.Atoi(matches[i][1])
		if err == nil {
			// Only get the state of it already exists
			if val, ok := v.Vms[id]; ok {
				newvms[id] = val
			} else {
				newvms[id] = "running"
			}
		}
	}

	// Now replace the map
	v.Vms = newvms

	return nil // no error!
}

func main() {
	os.Exit(run())
}

func run() (int) {
	// Get our VM struct working
	v := VMInformation{Vms: make(map[int]string)}
	v.sync()

	// Get the current numbers and data about VMs
	http.HandleFunc("/sync", func(w http.ResponseWriter, r *http.Request) {
		syncHandler(w, r, v)
	})

	// View information about a given VM
	http.HandleFunc("/view/", func(w http.ResponseWriter, r *http.Request) {
		viewHandler(w, r, v)
	})

	// Create a new VM of a given ID
	http.HandleFunc("/create/", func(w http.ResponseWriter, r *http.Request) {
		createHandler(w, r, v)
	})

	// Only bind to one interface -- IMPORTANT
	http.ListenAndServe("10.0.5.20:80", nil)

	return 0
}

func viewHandler(w http.ResponseWriter, r *http.Request, v VMInformation) {
	vmIdStr := r.URL.Path[len("/view/"):]
	vmId, err := strconv.Atoi(vmIdStr)
	// Check whether the ID is valid
	if err != nil {
		fmt.Fprintf(w, "invalid")
		return
	}

	// VMs < 50 are reserved for administrative use
	if (vmId < 50 || vmId > 254) {
		fmt.Fprintf(w, "invalid")
		return
	}

	// Confirm there is a vm running
	if _, exists := v.Vms[vmId]; !exists {
		fmt.Fprintf(w, "invalid")
		return
	}

	// valid and running
	fmt.Fprintf(w, v.Vms[vmId])
}

func syncHandler(w http.ResponseWriter, r *http.Request, v VMInformation) {
	// TODO: implement
}

func createHandler(w http.ResponseWriter, r *http.Request, v VMInformation) {
	vmIdStr := r.URL.Path[len("/create/"):]
	vmId, err := strconv.Atoi(vmIdStr)
	// Check whether the ID is valid
	if err != nil {
		fmt.Fprintf(w, "invalid")
		return
	}

	err = v.addVM(vmId, "creating")
	if err != nil {
		fmt.Fprintf(w, fmt.Sprintf("%v", err))
		return
	}

	// No error means we're ready to start the VM
	// Fork off a new thread to do the creation then let the user know we've started
	go createVM(vmId, v)
	fmt.Fprintf(w, "creating")
}

func createVM(vmId int, v VMInformation) {
	// This function assumes it's already been put into VMInformation
	// TODO: Write a validator for above asumption ^

	// Create a new directory for the VM disk image
	err := os.Mkdir(fmt.Sprintf("/root/vm-images/vm%v", vmId), 0755)
	if err != nil {
		v.updateVM(vmId, "broken")
		fmt.Fprintf(os.Stderr, "error creating disk image directory for new VM: %v", err)
		return
	}

	// Create new disk image
	err = exec.Command("qemu-img", "create", "-f", "qcow2", "-o", "backing_file=/root/vm-images/base-gentoo-vanilla-v2.img", fmt.Sprintf("/root/vm-images/vm%v/vm%v-gentoo-vanilla-v2.img", vmId, vmId)).Run()
	if err != nil {
		v.updateVM(vmId, "broken")
		fmt.Fprintf(os.Stderr, "error creating disk image for new VM: %v", err)
		return
	}

	// Create our network bridge and configuration
	// TODO: Do we need to lock the datastructure here? We might write network information for a VM that isn't made yet
	t, err := template.ParseFiles("assets/net")
	if err != nil {
		v.updateVM(vmId, "broken")
		fmt.Fprintf(os.Stderr, "error creating template for new VM: %v", err)
		return
	}
	var net bytes.Buffer
	err = t.Execute(&net, v)
	if err != nil {
		v.updateVM(vmId, "broken")
		fmt.Fprintf(os.Stderr, "error executing template for new VM: %v", err)
		return
	}

	// Write the file
	err = ioutil.WriteFile("/etc/conf.d/net", net.Bytes(), 0644)
	if err != nil {
		v.updateVM(vmId, "broken")
		fmt.Fprintf(os.Stderr, "error writing net template for new VM: %v", err)
		return
	}

	// Create the symlink for the bridge
	err = os.Symlink("/etc/init.d/net.lo", fmt.Sprintf("/etc/init.d/net.br%v", vmId))
	if err != nil {
		v.updateVM(vmId, "broken")
		fmt.Fprintf(os.Stderr, "error creating bridge symilnk for new VM: %v", err)
		return
	}

	// Restart network
	err = exec.Command("/etc/init.d/net.enp4s0", "restart").Run()
	if err != nil {
		v.updateVM(vmId, "broken")
		fmt.Fprintln(os.Stderr, "error executing enp4s0 restart for new VM: %v", err)
		return
	}
	err = exec.Command(fmt.Sprintf("/etc/init.d/net.br%v", vmId), "start").Run()
	if err != nil {
		v.updateVM(vmId, "broken")
		fmt.Fprintln(os.Stderr, "error executing bridge restart for new VM: %v", err)
		return
	}

	// Create a new screen/qemu instance
	cmd := exec.Command("screen", "-d", "-m", "-S", fmt.Sprintf("vm%v", vmId), "qemu-system-x86_64", "-nographic", "-enable-kvm", "-cpu", "host", "-curses", "-m", "512M", "-drive", fmt.Sprintf("file=/root/vm-images/vm%v/vm%v-gentoo-vanilla-v2.img,if=virtio", vmId, vmId), "-netdev", fmt.Sprintf("tap,helper=/usr/libexec/qemu-bridge-helper --br=br%v,id=hn0", vmId), "-device", "virtio-net-pci,netdev=hn0,id=nic1", "-append", fmt.Sprintf("root=/dev/vda4 ro vmid=%v", vmId), "-kernel", "/root/vm-images/kernels/vmlinuz-4.1.7-hardened-r1")
	// TODO: Find a way to check the error properly without using .Run() -- CHANGE TO .Start() else it won't work in futuer
	out, err := cmd.Output()
	outStr := string(out)
	if err != nil {
		v.updateVM(vmId, "broken")
		fmt.Fprintln(os.Stderr, "error starting screen session for new VM: %v \r\nadditional: %v", err, outStr)
		return
	}

/*
	// TODO Finish writing part that SSHs in and changes password
	// Lets wait until the VM has its networking set up (timeout if required)
	time.Sleep(60 * time.Second)

	worked := false
	// TODO Determine if best to use a ticker or sleep here
	for i := 0; i < 5; i++ {
		// SSH in and do the final configuraion
		time.Sleep(30 * time.Second)
	}
	if !worked {
		// TODO Write a better cleanup here, as this is clearly taking up space...
		v.updateVM(vmId, "broken")
		fmt.Fprintln(os.Stderr, "error could not connect to SSH within 3 minutes -- manual cleanup required")
		return
	}
	*/

	// Talk to the raspberry pi about getting a new Tor set up
	resp, err := http.Get(fmt.Sprintf("http://10.0.0.5/create/%v", vmId))
	if err != nil {
		v.updateVM(vmId, "broken")
		fmt.Fprintln(os.Stderr, "error talking to torcontrol: %v", err)
		return
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		v.updateVM(vmId, "broken")
		fmt.Fprintln(os.Stderr, "error getting response from torcontrol: %v", err)
		return
	}

	// Determine if the final part, tor stuff, worked
	status := string(body)

	if (status != "creating") {
		// TODO we should never get here, so handle this more strongly, it's probably an attack?
		fmt.Fprintln(os.Stderr, "error talking to torcontrol, response: %v", err)
		v.updateVM(vmId, "broken")
		return
	}

	// Wait a while for tor to generate it
	time.Sleep(30 * time.Second)

	// fetch the hostname
	resp, err = http.Get(fmt.Sprintf("http://10.0.0.5/view/%v", vmId))
	if err != nil {
		v.updateVM(vmId, "broken")
		fmt.Fprintln(os.Stderr, "error talking to torcontrol: %v", err)
		return
	}
	defer resp.Body.Close()
	body, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		v.updateVM(vmId, "broken")
		fmt.Fprintln(os.Stderr, "error getting response from torcontrol: %v", err)
		return
	}

	// Determine if the final part, tor stuff, worked
	status = string(body)

	if (status == "invalid" || status == "unknown") {
		// TODO we should never get here, so handle this more strongly, it's probably an attack?
		fmt.Fprintln(os.Stderr, "error talking to torcontrol, response: %v", err)
		v.updateVM(vmId, "broken")
		return
	}

	// Update VM status to be the onion address
	v.updateVM(vmId, status)
}
<commit_msg>Explicitly call resp.Body.Close() to prevent connection loss<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"net/http"
	"strconv"
	"regexp"
	"errors"
	"sync"
	"text/template"
	"bytes"
	"io/ioutil"
	"time"
)

type VMInformation struct {
	mux sync.Mutex
	// This map is id => state (should be an enum I guess)
	Vms map[int]string
}

func (v *VMInformation) addVM(vmId int, state string) (error) {
	// VMs < 50 are reserved for administrative use
	if (vmId < 50 || vmId > 254) {
		return errors.New("invalid")
	}

	// Lock for our further checks
	v.mux.Lock()
	defer v.mux.Unlock()

	// Get the list of VMs and confirm whether this is one that's already running
	if _, exists := v.Vms[vmId]; exists {
		return errors.New("in use")
	}

	// It's both valid and not in use!
	v.Vms[vmId] = state

	return nil // no errors
}

func (v *VMInformation) updateVM(vmId int, state string) (error) {
	v.mux.Lock()
	defer v.mux.Unlock()

	// If the VM doesn't already exist in the map, return an error
	if _, ok := v.Vms[vmId]; !ok {
		return errors.New("Invalid vmId specified")
	}

	v.Vms[vmId] = state

	return nil
}

func (v *VMInformation) sync() (error) {
	// Shell out to get a list of screen sessions, which are VMs
	out, err := exec.Command("screen", "-ls").Output()
	if err != nil {
		return errors.New("Error running screen")
	}

	// Our new map
	newvms := make(map[int]string)

	// Regex out the running VM IDs
	re := regexp.MustCompile(`\b\.vm([0-9]+)\b`)

	matches := re.FindAllStringSubmatch(string(out), -1)

	// We only need to start locking now, to ensure we read a valid state
	v.mux.Lock()
	defer v.mux.Unlock()

	for i := 0; i < len(matches); i++ {
		// Ignore invalid VMs
		id, err := strconv.Atoi(matches[i][1])
		if err == nil {
			// Only get the state of it already exists
			if val, ok := v.Vms[id]; ok {
				newvms[id] = val
			} else {
				newvms[id] = "running"
			}
		}
	}

	// Now replace the map
	v.Vms = newvms

	return nil // no error!
}

func main() {
	os.Exit(run())
}

func run() (int) {
	// Get our VM struct working
	v := VMInformation{Vms: make(map[int]string)}
	v.sync()

	// Get the current numbers and data about VMs
	http.HandleFunc("/sync", func(w http.ResponseWriter, r *http.Request) {
		syncHandler(w, r, v)
	})

	// View information about a given VM
	http.HandleFunc("/view/", func(w http.ResponseWriter, r *http.Request) {
		viewHandler(w, r, v)
	})

	// Create a new VM of a given ID
	http.HandleFunc("/create/", func(w http.ResponseWriter, r *http.Request) {
		createHandler(w, r, v)
	})

	// Only bind to one interface -- IMPORTANT
	http.ListenAndServe("10.0.5.20:80", nil)

	return 0
}

func viewHandler(w http.ResponseWriter, r *http.Request, v VMInformation) {
	vmIdStr := r.URL.Path[len("/view/"):]
	vmId, err := strconv.Atoi(vmIdStr)
	// Check whether the ID is valid
	if err != nil {
		fmt.Fprintf(w, "invalid")
		return
	}

	// VMs < 50 are reserved for administrative use
	if (vmId < 50 || vmId > 254) {
		fmt.Fprintf(w, "invalid")
		return
	}

	// Confirm there is a vm running
	if _, exists := v.Vms[vmId]; !exists {
		fmt.Fprintf(w, "invalid")
		return
	}

	// valid and running
	fmt.Fprintf(w, v.Vms[vmId])
}

func syncHandler(w http.ResponseWriter, r *http.Request, v VMInformation) {
	// TODO: implement
}

func createHandler(w http.ResponseWriter, r *http.Request, v VMInformation) {
	vmIdStr := r.URL.Path[len("/create/"):]
	vmId, err := strconv.Atoi(vmIdStr)
	// Check whether the ID is valid
	if err != nil {
		fmt.Fprintf(w, "invalid")
		return
	}

	err = v.addVM(vmId, "creating")
	if err != nil {
		fmt.Fprintf(w, fmt.Sprintf("%v", err))
		return
	}

	// No error means we're ready to start the VM
	// Fork off a new thread to do the creation then let the user know we've started
	go createVM(vmId, v)
	fmt.Fprintf(w, "creating")
}

func createVM(vmId int, v VMInformation) {
	// This function assumes it's already been put into VMInformation
	// TODO: Write a validator for above asumption ^

	// Create a new directory for the VM disk image
	err := os.Mkdir(fmt.Sprintf("/root/vm-images/vm%v", vmId), 0755)
	if err != nil {
		v.updateVM(vmId, "broken")
		fmt.Fprintf(os.Stderr, "error creating disk image directory for new VM: %v", err)
		return
	}

	// Create new disk image
	err = exec.Command("qemu-img", "create", "-f", "qcow2", "-o", "backing_file=/root/vm-images/base-gentoo-vanilla-v2.img", fmt.Sprintf("/root/vm-images/vm%v/vm%v-gentoo-vanilla-v2.img", vmId, vmId)).Run()
	if err != nil {
		v.updateVM(vmId, "broken")
		fmt.Fprintf(os.Stderr, "error creating disk image for new VM: %v", err)
		return
	}

	// Create our network bridge and configuration
	// TODO: Do we need to lock the datastructure here? We might write network information for a VM that isn't made yet
	t, err := template.ParseFiles("assets/net")
	if err != nil {
		v.updateVM(vmId, "broken")
		fmt.Fprintf(os.Stderr, "error creating template for new VM: %v", err)
		return
	}
	var net bytes.Buffer
	err = t.Execute(&net, v)
	if err != nil {
		v.updateVM(vmId, "broken")
		fmt.Fprintf(os.Stderr, "error executing template for new VM: %v", err)
		return
	}

	// Write the file
	err = ioutil.WriteFile("/etc/conf.d/net", net.Bytes(), 0644)
	if err != nil {
		v.updateVM(vmId, "broken")
		fmt.Fprintf(os.Stderr, "error writing net template for new VM: %v", err)
		return
	}

	// Create the symlink for the bridge
	err = os.Symlink("/etc/init.d/net.lo", fmt.Sprintf("/etc/init.d/net.br%v", vmId))
	if err != nil {
		v.updateVM(vmId, "broken")
		fmt.Fprintf(os.Stderr, "error creating bridge symilnk for new VM: %v", err)
		return
	}

	// Restart network
	err = exec.Command("/etc/init.d/net.enp4s0", "restart").Run()
	if err != nil {
		v.updateVM(vmId, "broken")
		fmt.Fprintln(os.Stderr, "error executing enp4s0 restart for new VM: %v", err)
		return
	}
	err = exec.Command(fmt.Sprintf("/etc/init.d/net.br%v", vmId), "start").Run()
	if err != nil {
		v.updateVM(vmId, "broken")
		fmt.Fprintln(os.Stderr, "error executing bridge restart for new VM: %v", err)
		return
	}

	// Create a new screen/qemu instance
	cmd := exec.Command("screen", "-d", "-m", "-S", fmt.Sprintf("vm%v", vmId), "qemu-system-x86_64", "-nographic", "-enable-kvm", "-cpu", "host", "-curses", "-m", "512M", "-drive", fmt.Sprintf("file=/root/vm-images/vm%v/vm%v-gentoo-vanilla-v2.img,if=virtio", vmId, vmId), "-netdev", fmt.Sprintf("tap,helper=/usr/libexec/qemu-bridge-helper --br=br%v,id=hn0", vmId), "-device", "virtio-net-pci,netdev=hn0,id=nic1", "-append", fmt.Sprintf("root=/dev/vda4 ro vmid=%v", vmId), "-kernel", "/root/vm-images/kernels/vmlinuz-4.1.7-hardened-r1")
	// TODO: Find a way to check the error properly without using .Run() -- CHANGE TO .Start() else it won't work in futuer
	out, err := cmd.Output()
	outStr := string(out)
	if err != nil {
		v.updateVM(vmId, "broken")
		fmt.Fprintln(os.Stderr, "error starting screen session for new VM: %v \r\nadditional: %v", err, outStr)
		return
	}

/*
	// TODO Finish writing part that SSHs in and changes password
	// Lets wait until the VM has its networking set up (timeout if required)
	time.Sleep(60 * time.Second)

	worked := false
	// TODO Determine if best to use a ticker or sleep here
	for i := 0; i < 5; i++ {
		// SSH in and do the final configuraion
		time.Sleep(30 * time.Second)
	}
	if !worked {
		// TODO Write a better cleanup here, as this is clearly taking up space...
		v.updateVM(vmId, "broken")
		fmt.Fprintln(os.Stderr, "error could not connect to SSH within 3 minutes -- manual cleanup required")
		return
	}
	*/

	// Talk to the raspberry pi about getting a new Tor set up
	resp, err := http.Get(fmt.Sprintf("http://10.0.0.5/create/%v", vmId))
	if err != nil {
		v.updateVM(vmId, "broken")
		fmt.Fprintln(os.Stderr, "error talking to torcontrol: %v", err)
		return
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		v.updateVM(vmId, "broken")
		fmt.Fprintln(os.Stderr, "error getting response from torcontrol: %v", err)
		return
	}

	// Determine if the final part, tor stuff, worked
	status := string(body)

	// Close the response body
	resp.Body.Close()

	if (status != "creating") {
		// TODO we should never get here, so handle this more strongly, it's probably an attack?
		fmt.Fprintln(os.Stderr, "error talking to torcontrol, response: %v", err)
		v.updateVM(vmId, "broken")
		return
	}

	// Wait a while for tor to generate it
	time.Sleep(30 * time.Second)

	// fetch the hostname
	resp, err = http.Get(fmt.Sprintf("http://10.0.0.5/view/%v", vmId))
	if err != nil {
		v.updateVM(vmId, "broken")
		fmt.Fprintln(os.Stderr, "error talking to torcontrol: %v", err)
		return
	}
	body, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		v.updateVM(vmId, "broken")
		fmt.Fprintln(os.Stderr, "error getting response from torcontrol: %v", err)
		return
	}

	// Determine if the final part, tor stuff, worked
	status = string(body)

	// Close the response body
	resp.Body.Close()

	if (status == "invalid" || status == "unknown") {
		// TODO we should never get here, so handle this more strongly, it's probably an attack?
		fmt.Fprintln(os.Stderr, "error talking to torcontrol, response: %v", err)
		v.updateVM(vmId, "broken")
		return
	}

	// Update VM status to be the onion address
	v.updateVM(vmId, status)
}
<|endoftext|>


<commit_before><commit_msg>implements gcm client which communicate with http2<commit_after><|endoftext|>


<commit_before>// Package gcnotifier provides a way to receive notifications after every time
// garbage collection (GC) runs. This can be useful to instruct your code to
// free additional memory resources that you may be using. To minimize the load
// on the GC the code that runs after receiving the notification should try to
// avoid allocations as much as possible.
package gcnotifier

import "runtime"

type sentinel struct {
	gcCh chan struct{}
}

// AfterGC returns a channel that will receive a notification after every GC
// run. If a notification is not consumed before another GC runs only one of the
// two notifications is sent. To stop the notifications you can safely close the
// channel.
func AfterGC() <-chan struct{} {
	s := &sentinel{gcCh: make(chan struct{})}
	runtime.SetFinalizer(s, finalizer)
	return s.gcCh
}

func finalizer(obj interface{}) {
	defer recover() // writing to a closed channel will panic
	s := obj.(*sentinel)
	select {
	case s.gcCh <- struct{}{}:
	default:
	}
	runtime.SetFinalizer(s, finalizer)
}
<commit_msg>Add AfterGCUntilCollected<commit_after>// Package gcnotifier provides a way to receive notifications after every time
// garbage collection (GC) runs. This can be useful to instruct your code to
// free additional memory resources that you may be using. To minimize the load
// on the GC the code that runs after receiving the notification should try to
// avoid allocations as much as possible.
package gcnotifier

import "runtime"

type sentinel struct {
	gcCh chan struct{}
}

// AfterGC returns a channel that will receive a notification after every GC
// run. If a notification is not consumed before another GC runs only one of the
// two notifications is sent. To stop the notifications you can safely close the
// channel.
func AfterGC() <-chan struct{} {
	s := &sentinel{gcCh: make(chan struct{})}
	runtime.SetFinalizer(s, finalizer)
	return s.gcCh
}

func finalizer(obj interface{}) {
	defer recover() // writing to a closed channel will panic
	s := obj.(*sentinel)
	select {
	case s.gcCh <- struct{}{}:
	default:
	}
	// we get here only if the channel was not closed
	runtime.SetFinalizer(s, finalizer)
}

// AfterGCUntilReachable is like AfterGC, but the channel will be closed when
// the object supplied as argument is garbage collected. No finalizer should be
// set on the object before or after calling this method. Pay attention to not
// inadvertently keep the object alive (e.g. by referencing it in a callback or
// goroutine) or the object may never be collected.
func AfterGCUntilCollected(obj interface{}) <-chan struct {
  gcCh := AfterGC()
	runtime.SetFinalizer(obj, func() { close(goCh) })
	return goCh
}
<|endoftext|>


<commit_before>package services_test

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	datatypes "github.com/maximilien/softlayer-go/data_types"
	softlayer "github.com/maximilien/softlayer-go/softlayer"
	testhelpers "github.com/maximilien/softlayer-go/test_helpers"
)

var (
	TIMEOUT          time.Duration
	POLLING_INTERVAL time.Duration
)

var _ = Describe("SoftLayer Services", func() {
	var (
		err error

		accountService      softlayer.SoftLayer_Account_Service
		virtualGuestService softlayer.SoftLayer_Virtual_Guest_Service
	)

	BeforeEach(func() {
		accountService, err = testhelpers.CreateAccountService()
		Expect(err).ToNot(HaveOccurred())

		virtualGuestService, err = testhelpers.CreateVirtualGuestService()
		Expect(err).ToNot(HaveOccurred())

		TIMEOUT = 5 * time.Minute
		POLLING_INTERVAL = 10 * time.Second
	})

	Context("uses SoftLayer_Account to list current virtual: disk images, guests, ssh keys, and network storage", func() {
		It("returns an array of SoftLayer_Virtual_Guest disk images", func() {
			virtualDiskImages, err := accountService.GetVirtualDiskImages()
			Expect(err).ToNot(HaveOccurred())
			Expect(len(virtualDiskImages)).To(BeNumerically(">=", 0))
		})

		It("returns an array of SoftLayer_Virtual_Guest objects", func() {
			virtualGuests, err := accountService.GetVirtualGuests()
			Expect(err).ToNot(HaveOccurred())
			Expect(len(virtualGuests)).To(BeNumerically(">=", 0))
		})

		It("returns an array of SoftLayer_Virtual_Guest network storage", func() {
			networkStorageArray, err := accountService.GetNetworkStorage()
			Expect(err).ToNot(HaveOccurred())
			Expect(len(networkStorageArray)).To(BeNumerically(">=", 0))
		})

		It("returns an array of SoftLayer_Ssh_Keys objects", func() {
			sshKeys, err := accountService.GetSshKeys()
			Expect(err).ToNot(HaveOccurred())
			Expect(len(sshKeys)).To(BeNumerically(">=", 0))
		})
	})

	Context("uses SoftLayer_Account to create and then delete a an ssh key", func() {
		It("creates the ssh key and verify it is present and then deletes it", func() {
			sshKeyPath := os.Getenv("SOFTLAYER_GO_TEST_SSH_KEY_PATH1")
			Expect(sshKeyPath).ToNot(Equal(""), "SOFTLAYER_GO_TEST_SSH_KEY_PATH1 env variable is not set")

			createdSshKey := createTestSshKey(sshKeyPath)
			waitForCreatedSshKeyToBePresent(createdSshKey.Id)

			sshKeyService, err := testhelpers.CreateSecuritySshKeyService()
			Expect(err).ToNot(HaveOccurred())

			deleted, err := sshKeyService.DeleteObject(createdSshKey.Id)
			Expect(err).ToNot(HaveOccurred())
			Expect(deleted).To(BeTrue())

			waitForDeletedSshKeyToNoLongerBePresent(createdSshKey.Id)
		})
	})

	Context("uses SoftLayer_Account to create and then delete a virtual guest instance", func() {
		It("creates the virtual guest instance and waits for it to be active then delete it", func() {
			virtualGuest := createVirtualGuestAndMarkItTest([]datatypes.SoftLayer_Security_Ssh_Key{})

			waitForVirtualGuestToBeRunning(virtualGuest.Id)
			waitForVirtualGuestToHaveNoActiveTransactions(virtualGuest.Id)

			deleteVirtualGuest(virtualGuest.Id)
		})
	})

	Context("uses SoftLayer_Account to create ssh key and new virtual guest with ssh key assigned", func() {
		It("creates key, creates virtual guest and adds key to list of VG", func() {
			sshKeyPath := os.Getenv("SOFTLAYER_GO_TEST_SSH_KEY_PATH2")
			Expect(sshKeyPath).ToNot(Equal(""), "SOFTLAYER_GO_TEST_SSH_KEY_PATH2 env variable is not set")

			createdSshKey := createTestSshKey(sshKeyPath)
			waitForCreatedSshKeyToBePresent(createdSshKey.Id)

			virtualGuest := createVirtualGuestAndMarkItTest([]datatypes.SoftLayer_Security_Ssh_Key{createdSshKey})

			waitForVirtualGuestToBeRunning(virtualGuest.Id)
			waitForVirtualGuestToHaveNoActiveTransactions(virtualGuest.Id)

			deleteVirtualGuest(virtualGuest.Id)
			deleteSshKey(createdSshKey.Id)
		})
	})

	XContext("uses SoftLayer_Account to create a new instance and network storage and attach them", func() {
		It("creates the virtual guest instance and waits for it to be active", func() {
			Expect(false).To(BeTrue())
		})

		It("creates the disk storage and attaches it to the instance", func() {
			Expect(false).To(BeTrue())
		})

		It("deletes the virtual guest instance if it is running", func() {
			Expect(false).To(BeTrue())
		})

		It("detaches and deletes the network storage if available", func() {
			Expect(false).To(BeTrue())
		})
	})
})

func createTestSshKey(sshKeyPath string) datatypes.SoftLayer_Security_Ssh_Key {
	testSshKeyValue, err := ioutil.ReadFile(sshKeyPath)
	Expect(err).ToNot(HaveOccurred())

	sshKey := datatypes.SoftLayer_Security_Ssh_Key{
		Key:         strings.Trim(string(testSshKeyValue), "\n"),
		Fingerprint: "f6:c2:9d:57:2f:74:be:a1:db:71:f2:e5:8e:0f:84:7e",
		Label:       testhelpers.TEST_LABEL_PREFIX,
		Notes:       testhelpers.TEST_NOTES_PREFIX,
	}

	sshKeyService, err := testhelpers.CreateSecuritySshKeyService()
	Expect(err).ToNot(HaveOccurred())

	fmt.Printf("----> creating ssh key\n")
	createdSshKey, err := sshKeyService.CreateObject(sshKey)
	Expect(err).ToNot(HaveOccurred())
	Expect(createdSshKey.Key).To(Equal(sshKey.Key), "key")
	Expect(createdSshKey.Label).To(Equal(sshKey.Label), "label")
	Expect(createdSshKey.Notes).To(Equal(sshKey.Notes), "notes")
	Expect(createdSshKey.CreateDate).ToNot(BeNil(), "createDate")
	Expect(createdSshKey.Fingerprint).ToNot(Equal(""), "fingerprint")
	Expect(createdSshKey.Id).To(BeNumerically(">", 0), "id")
	Expect(createdSshKey.ModifyDate).To(BeNil(), "modifyDate")
	fmt.Printf("----> created ssh key: %d\n", createdSshKey.Id)

	return createdSshKey
}

func createVirtualGuestAndMarkItTest(securitySshKeys []datatypes.SoftLayer_Security_Ssh_Key) datatypes.SoftLayer_Virtual_Guest {
	sshKeys := make([]datatypes.SshKey, len(securitySshKeys))
	for i, securitySshKey := range securitySshKeys {
		sshKeys[i] = datatypes.SshKey{Id: securitySshKey.Id}
	}

	virtualGuestTemplate := datatypes.SoftLayer_Virtual_Guest_Template{
		Hostname:  "test",
		Domain:    "softlayergo.com",
		StartCpus: 1,
		MaxMemory: 1024,
		Datacenter: datatypes.Datacenter{
			Name: "ams01",
		},
		SshKeys:                      sshKeys,
		HourlyBillingFlag:            true,
		LocalDiskFlag:                true,
		OperatingSystemReferenceCode: "UBUNTU_LATEST",
	}

	virtualGuestService, err := testhelpers.CreateVirtualGuestService()
	Expect(err).ToNot(HaveOccurred())

	fmt.Printf("----> creating new virtual guest\n")
	virtualGuest, err := virtualGuestService.CreateObject(virtualGuestTemplate)
	Expect(err).ToNot(HaveOccurred())
	fmt.Printf("----> created virtual guest: %d\n", virtualGuest.Id)

	waitForVirtualGuestToBeRunning(virtualGuest.Id)
	waitForVirtualGuestToHaveNoActiveTransactions(virtualGuest.Id)

	fmt.Printf("----> marking virtual guest with TEST:softlayer-go\n")
	err = testhelpers.MarkVirtualGuestAsTest(virtualGuest)
	Expect(err).ToNot(HaveOccurred(), "Could not mark virtual guest as test")
	fmt.Printf("----> marked virtual guest with TEST:softlayer-go\n")

	return virtualGuest
}

func deleteVirtualGuest(virtualGuestId int) {
	virtualGuestService, err := testhelpers.CreateVirtualGuestService()
	Expect(err).ToNot(HaveOccurred())

	fmt.Printf("----> deleting virtual guest: %d\n", virtualGuestId)
	deleted, err := virtualGuestService.DeleteObject(virtualGuestId)
	Expect(err).ToNot(HaveOccurred())
	Expect(deleted).To(BeTrue(), "could not delete virtual guest")

	waitForVirtualGuestToHaveNoActiveTransactions(virtualGuestId)
}

func deleteSshKey(sshKeyId int) {
	sshKeyService, err := testhelpers.CreateSecuritySshKeyService()
	Expect(err).ToNot(HaveOccurred())

	fmt.Printf("----> deleting ssh key: %d\n", sshKeyId)
	deleted, err := sshKeyService.DeleteObject(sshKeyId)
	Expect(err).ToNot(HaveOccurred())
	Expect(deleted).To(BeTrue(), "could not delete ssh key")

	waitForDeletedSshKeyToNoLongerBePresent(sshKeyId)
}

func waitForVirtualGuestToBeRunning(virtualGuestId int) {
	virtualGuestService, err := testhelpers.CreateVirtualGuestService()
	Expect(err).ToNot(HaveOccurred())

	fmt.Printf("----> waiting for virtual guest: %d, until RUNNING\n", virtualGuestId)
	Eventually(func() string {
		vgPowerState, err := virtualGuestService.GetPowerState(virtualGuestId)
		Expect(err).ToNot(HaveOccurred())
		fmt.Printf("----> virtual guest: %d, has power state: %s\n", virtualGuestId, vgPowerState.KeyName)
		return vgPowerState.KeyName
	}, TIMEOUT, POLLING_INTERVAL).Should(Equal("RUNNING"), "failed waiting for virtual guest to be RUNNING")
}

func waitForVirtualGuestToHaveNoActiveTransactions(virtualGuestId int) {
	virtualGuestService, err := testhelpers.CreateVirtualGuestService()
	Expect(err).ToNot(HaveOccurred())

	fmt.Printf("----> waiting for virtual guest to have no active transactions pending\n")
	Eventually(func() int {
		activeTransactions, err := virtualGuestService.GetActiveTransactions(virtualGuestId)
		Expect(err).ToNot(HaveOccurred())
		fmt.Printf("----> virtual guest: %d, has %d active transactions\n", virtualGuestId, len(activeTransactions))
		return len(activeTransactions)
	}, TIMEOUT, POLLING_INTERVAL).Should(Equal(0), "failed waiting for virtual guest to have no active transactions")
}

func waitForDeletedSshKeyToNoLongerBePresent(sshKeyId int) {
	accountService, err := testhelpers.CreateAccountService()
	Expect(err).ToNot(HaveOccurred())

	fmt.Printf("----> waiting for deleted ssh key to no longer be present\n")
	Eventually(func() bool {
		sshKeys, err := accountService.GetSshKeys()
		Expect(err).ToNot(HaveOccurred())

		deleted := true
		for _, sshKey := range sshKeys {
			if sshKey.Id == sshKeyId {
				deleted = false
			}
		}
		return deleted
	}, TIMEOUT, POLLING_INTERVAL).Should(BeTrue(), "failed waiting for deleted ssh key to be removed from list of ssh keys")
}

func waitForCreatedSshKeyToBePresent(sshKeyId int) {
	accountService, err := testhelpers.CreateAccountService()
	Expect(err).ToNot(HaveOccurred())

	fmt.Printf("----> waiting for created ssh key to be present\n")
	Eventually(func() bool {
		sshKeys, err := accountService.GetSshKeys()
		Expect(err).ToNot(HaveOccurred())

		keyPresent := false
		for _, sshKey := range sshKeys {
			if sshKey.Id == sshKeyId {
				keyPresent = true
			}
		}
		return keyPresent
	}, TIMEOUT, POLLING_INTERVAL).Should(BeTrue(), "created ssh key but not in the list of ssh keys")
}
<commit_msg>increased timeout and polling interval x2<commit_after>package services_test

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	datatypes "github.com/maximilien/softlayer-go/data_types"
	softlayer "github.com/maximilien/softlayer-go/softlayer"
	testhelpers "github.com/maximilien/softlayer-go/test_helpers"
)

var (
	TIMEOUT          time.Duration
	POLLING_INTERVAL time.Duration
)

var _ = Describe("SoftLayer Services", func() {
	var (
		err error

		accountService      softlayer.SoftLayer_Account_Service
		virtualGuestService softlayer.SoftLayer_Virtual_Guest_Service
	)

	BeforeEach(func() {
		accountService, err = testhelpers.CreateAccountService()
		Expect(err).ToNot(HaveOccurred())

		virtualGuestService, err = testhelpers.CreateVirtualGuestService()
		Expect(err).ToNot(HaveOccurred())

		TIMEOUT = 15 * time.Minute
		POLLING_INTERVAL = 15 * time.Second
	})

	Context("uses SoftLayer_Account to list current virtual: disk images, guests, ssh keys, and network storage", func() {
		It("returns an array of SoftLayer_Virtual_Guest disk images", func() {
			virtualDiskImages, err := accountService.GetVirtualDiskImages()
			Expect(err).ToNot(HaveOccurred())
			Expect(len(virtualDiskImages)).To(BeNumerically(">=", 0))
		})

		It("returns an array of SoftLayer_Virtual_Guest objects", func() {
			virtualGuests, err := accountService.GetVirtualGuests()
			Expect(err).ToNot(HaveOccurred())
			Expect(len(virtualGuests)).To(BeNumerically(">=", 0))
		})

		It("returns an array of SoftLayer_Virtual_Guest network storage", func() {
			networkStorageArray, err := accountService.GetNetworkStorage()
			Expect(err).ToNot(HaveOccurred())
			Expect(len(networkStorageArray)).To(BeNumerically(">=", 0))
		})

		It("returns an array of SoftLayer_Ssh_Keys objects", func() {
			sshKeys, err := accountService.GetSshKeys()
			Expect(err).ToNot(HaveOccurred())
			Expect(len(sshKeys)).To(BeNumerically(">=", 0))
		})
	})

	Context("uses SoftLayer_Account to create and then delete a an ssh key", func() {
		It("creates the ssh key and verify it is present and then deletes it", func() {
			sshKeyPath := os.Getenv("SOFTLAYER_GO_TEST_SSH_KEY_PATH1")
			Expect(sshKeyPath).ToNot(Equal(""), "SOFTLAYER_GO_TEST_SSH_KEY_PATH1 env variable is not set")

			createdSshKey := createTestSshKey(sshKeyPath)
			waitForCreatedSshKeyToBePresent(createdSshKey.Id)

			sshKeyService, err := testhelpers.CreateSecuritySshKeyService()
			Expect(err).ToNot(HaveOccurred())

			deleted, err := sshKeyService.DeleteObject(createdSshKey.Id)
			Expect(err).ToNot(HaveOccurred())
			Expect(deleted).To(BeTrue())

			waitForDeletedSshKeyToNoLongerBePresent(createdSshKey.Id)
		})
	})

	Context("uses SoftLayer_Account to create and then delete a virtual guest instance", func() {
		It("creates the virtual guest instance and waits for it to be active then delete it", func() {
			virtualGuest := createVirtualGuestAndMarkItTest([]datatypes.SoftLayer_Security_Ssh_Key{})

			waitForVirtualGuestToBeRunning(virtualGuest.Id)
			waitForVirtualGuestToHaveNoActiveTransactions(virtualGuest.Id)

			deleteVirtualGuest(virtualGuest.Id)
		})
	})

	Context("uses SoftLayer_Account to create ssh key and new virtual guest with ssh key assigned", func() {
		It("creates key, creates virtual guest and adds key to list of VG", func() {
			sshKeyPath := os.Getenv("SOFTLAYER_GO_TEST_SSH_KEY_PATH2")
			Expect(sshKeyPath).ToNot(Equal(""), "SOFTLAYER_GO_TEST_SSH_KEY_PATH2 env variable is not set")

			createdSshKey := createTestSshKey(sshKeyPath)
			waitForCreatedSshKeyToBePresent(createdSshKey.Id)

			virtualGuest := createVirtualGuestAndMarkItTest([]datatypes.SoftLayer_Security_Ssh_Key{createdSshKey})

			waitForVirtualGuestToBeRunning(virtualGuest.Id)
			waitForVirtualGuestToHaveNoActiveTransactions(virtualGuest.Id)

			deleteVirtualGuest(virtualGuest.Id)
			deleteSshKey(createdSshKey.Id)
		})
	})

	XContext("uses SoftLayer_Account to create a new instance and network storage and attach them", func() {
		It("creates the virtual guest instance and waits for it to be active", func() {
			Expect(false).To(BeTrue())
		})

		It("creates the disk storage and attaches it to the instance", func() {
			Expect(false).To(BeTrue())
		})

		It("deletes the virtual guest instance if it is running", func() {
			Expect(false).To(BeTrue())
		})

		It("detaches and deletes the network storage if available", func() {
			Expect(false).To(BeTrue())
		})
	})
})

func createTestSshKey(sshKeyPath string) datatypes.SoftLayer_Security_Ssh_Key {
	testSshKeyValue, err := ioutil.ReadFile(sshKeyPath)
	Expect(err).ToNot(HaveOccurred())

	sshKey := datatypes.SoftLayer_Security_Ssh_Key{
		Key:         strings.Trim(string(testSshKeyValue), "\n"),
		Fingerprint: "f6:c2:9d:57:2f:74:be:a1:db:71:f2:e5:8e:0f:84:7e",
		Label:       testhelpers.TEST_LABEL_PREFIX,
		Notes:       testhelpers.TEST_NOTES_PREFIX,
	}

	sshKeyService, err := testhelpers.CreateSecuritySshKeyService()
	Expect(err).ToNot(HaveOccurred())

	fmt.Printf("----> creating ssh key\n")
	createdSshKey, err := sshKeyService.CreateObject(sshKey)
	Expect(err).ToNot(HaveOccurred())
	Expect(createdSshKey.Key).To(Equal(sshKey.Key), "key")
	Expect(createdSshKey.Label).To(Equal(sshKey.Label), "label")
	Expect(createdSshKey.Notes).To(Equal(sshKey.Notes), "notes")
	Expect(createdSshKey.CreateDate).ToNot(BeNil(), "createDate")
	Expect(createdSshKey.Fingerprint).ToNot(Equal(""), "fingerprint")
	Expect(createdSshKey.Id).To(BeNumerically(">", 0), "id")
	Expect(createdSshKey.ModifyDate).To(BeNil(), "modifyDate")
	fmt.Printf("----> created ssh key: %d\n", createdSshKey.Id)

	return createdSshKey
}

func createVirtualGuestAndMarkItTest(securitySshKeys []datatypes.SoftLayer_Security_Ssh_Key) datatypes.SoftLayer_Virtual_Guest {
	sshKeys := make([]datatypes.SshKey, len(securitySshKeys))
	for i, securitySshKey := range securitySshKeys {
		sshKeys[i] = datatypes.SshKey{Id: securitySshKey.Id}
	}

	virtualGuestTemplate := datatypes.SoftLayer_Virtual_Guest_Template{
		Hostname:  "test",
		Domain:    "softlayergo.com",
		StartCpus: 1,
		MaxMemory: 1024,
		Datacenter: datatypes.Datacenter{
			Name: "ams01",
		},
		SshKeys:                      sshKeys,
		HourlyBillingFlag:            true,
		LocalDiskFlag:                true,
		OperatingSystemReferenceCode: "UBUNTU_LATEST",
	}

	virtualGuestService, err := testhelpers.CreateVirtualGuestService()
	Expect(err).ToNot(HaveOccurred())

	fmt.Printf("----> creating new virtual guest\n")
	virtualGuest, err := virtualGuestService.CreateObject(virtualGuestTemplate)
	Expect(err).ToNot(HaveOccurred())
	fmt.Printf("----> created virtual guest: %d\n", virtualGuest.Id)

	waitForVirtualGuestToBeRunning(virtualGuest.Id)
	waitForVirtualGuestToHaveNoActiveTransactions(virtualGuest.Id)

	fmt.Printf("----> marking virtual guest with TEST:softlayer-go\n")
	err = testhelpers.MarkVirtualGuestAsTest(virtualGuest)
	Expect(err).ToNot(HaveOccurred(), "Could not mark virtual guest as test")
	fmt.Printf("----> marked virtual guest with TEST:softlayer-go\n")

	return virtualGuest
}

func deleteVirtualGuest(virtualGuestId int) {
	virtualGuestService, err := testhelpers.CreateVirtualGuestService()
	Expect(err).ToNot(HaveOccurred())

	fmt.Printf("----> deleting virtual guest: %d\n", virtualGuestId)
	deleted, err := virtualGuestService.DeleteObject(virtualGuestId)
	Expect(err).ToNot(HaveOccurred())
	Expect(deleted).To(BeTrue(), "could not delete virtual guest")

	waitForVirtualGuestToHaveNoActiveTransactions(virtualGuestId)
}

func deleteSshKey(sshKeyId int) {
	sshKeyService, err := testhelpers.CreateSecuritySshKeyService()
	Expect(err).ToNot(HaveOccurred())

	fmt.Printf("----> deleting ssh key: %d\n", sshKeyId)
	deleted, err := sshKeyService.DeleteObject(sshKeyId)
	Expect(err).ToNot(HaveOccurred())
	Expect(deleted).To(BeTrue(), "could not delete ssh key")

	waitForDeletedSshKeyToNoLongerBePresent(sshKeyId)
}

func waitForVirtualGuestToBeRunning(virtualGuestId int) {
	virtualGuestService, err := testhelpers.CreateVirtualGuestService()
	Expect(err).ToNot(HaveOccurred())

	fmt.Printf("----> waiting for virtual guest: %d, until RUNNING\n", virtualGuestId)
	Eventually(func() string {
		vgPowerState, err := virtualGuestService.GetPowerState(virtualGuestId)
		Expect(err).ToNot(HaveOccurred())
		fmt.Printf("----> virtual guest: %d, has power state: %s\n", virtualGuestId, vgPowerState.KeyName)
		return vgPowerState.KeyName
	}, TIMEOUT, POLLING_INTERVAL).Should(Equal("RUNNING"), "failed waiting for virtual guest to be RUNNING")
}

func waitForVirtualGuestToHaveNoActiveTransactions(virtualGuestId int) {
	virtualGuestService, err := testhelpers.CreateVirtualGuestService()
	Expect(err).ToNot(HaveOccurred())

	fmt.Printf("----> waiting for virtual guest to have no active transactions pending\n")
	Eventually(func() int {
		activeTransactions, err := virtualGuestService.GetActiveTransactions(virtualGuestId)
		Expect(err).ToNot(HaveOccurred())
		fmt.Printf("----> virtual guest: %d, has %d active transactions\n", virtualGuestId, len(activeTransactions))
		return len(activeTransactions)
	}, TIMEOUT, POLLING_INTERVAL).Should(Equal(0), "failed waiting for virtual guest to have no active transactions")
}

func waitForDeletedSshKeyToNoLongerBePresent(sshKeyId int) {
	accountService, err := testhelpers.CreateAccountService()
	Expect(err).ToNot(HaveOccurred())

	fmt.Printf("----> waiting for deleted ssh key to no longer be present\n")
	Eventually(func() bool {
		sshKeys, err := accountService.GetSshKeys()
		Expect(err).ToNot(HaveOccurred())

		deleted := true
		for _, sshKey := range sshKeys {
			if sshKey.Id == sshKeyId {
				deleted = false
			}
		}
		return deleted
	}, TIMEOUT, POLLING_INTERVAL).Should(BeTrue(), "failed waiting for deleted ssh key to be removed from list of ssh keys")
}

func waitForCreatedSshKeyToBePresent(sshKeyId int) {
	accountService, err := testhelpers.CreateAccountService()
	Expect(err).ToNot(HaveOccurred())

	fmt.Printf("----> waiting for created ssh key to be present\n")
	Eventually(func() bool {
		sshKeys, err := accountService.GetSshKeys()
		Expect(err).ToNot(HaveOccurred())

		keyPresent := false
		for _, sshKey := range sshKeys {
			if sshKey.Id == sshKeyId {
				keyPresent = true
			}
		}
		return keyPresent
	}, TIMEOUT, POLLING_INTERVAL).Should(BeTrue(), "created ssh key but not in the list of ssh keys")
}
<|endoftext|>


<commit_before>package get

import (
	"compress/gzip"
	"crypto"
	"encoding/xml"
	"io"
	"log"

	"github.com/moio/minima/util"
)

// common

// XMLLocation maps a <location> tag in repodata/repomd.xml or repodata/<ID>-primary.xml.gz
type XMLLocation struct {
	Href string `xml:"href,attr"`
}

// repodata/repomd.xml

// XMLRepomd maps a <repomd> tag in repodata/repomd.xml
type XMLRepomd struct {
	Data []XMLData `xml:"data"`
}

// XMLData maps a <data> tag in repodata/repomd.xml
type XMLData struct {
	Type     string      `xml:"type,attr"`
	Location XMLLocation `xml:"location"`
	Checksum XMLChecksum `xml:"checksum"`
}

// repodata/<ID>-primary.xml.gz

// XMLMetaData maps a <metadata> tag in repodata/<ID>-primary.xml.gz
type XMLMetaData struct {
	Packages []XMLPackage `xml:"package"`
}

// XMLPackage maps a <package> tag in repodata/<ID>-primary.xml.gz
type XMLPackage struct {
	Arch     string      `xml:"arch"`
	Location XMLLocation `xml:"location"`
	Checksum XMLChecksum `xml:"checksum"`
}

// XMLChecksum maps a <checksum> tag in repodata/<ID>-primary.xml.gz
type XMLChecksum struct {
	Type     string `xml:"type,attr"`
	Checksum string `xml:",cdata"`
}

var hashMap = map[string]crypto.Hash{
	"sha":    crypto.SHA1,
	"sha1":   crypto.SHA1,
	"sha256": crypto.SHA256,
}

const repomdPath = "repodata/repomd.xml"

// Syncer syncs repos from an HTTP source to a Storage
type Syncer struct {
	// URL of the repo this syncer syncs
	Url     string
	archs   map[string]bool
	storage Storage
}

// NewSyncer creates a new Syncer
func NewSyncer(url string, archs map[string]bool, storage Storage) *Syncer {
	return &Syncer{url, archs, storage}
}

// StoreRepo stores an HTTP repo in a Storage, automatically retrying in case of recoverable errors
func (r *Syncer) StoreRepo() (err error) {
	checksumMap := r.readChecksumMap()
	for i := 0; i < 20; i++ {
		err = r.storeRepo(checksumMap)
		if err == nil {
			return
		}

		uerr, unexpectedStatusCode := err.(*UnexpectedStatusCodeError)
		if unexpectedStatusCode {
			if uerr.StatusCode == 404 {
				log.Printf("Got 404, presumably temporarily, retrying...\n")
			} else {
				return err
			}
		}

		_, checksumError := err.(*util.ChecksumError)
		if checksumError {
			log.Printf("Checksum did not match, presumably the repo was published while syncing, retrying...\n")
		} else {
			return err
		}
	}

	log.Printf("Too many temporary errors, aborting...\n")
	return err
}

// StoreRepo stores an HTTP repo in a Storage
func (r *Syncer) storeRepo(checksumMap map[string]XMLChecksum) (err error) {
	packagesToDownload, packagesToRecycle, err := r.processMetadata(checksumMap)
	if err != nil {
		return
	}

	downloadCount := len(packagesToDownload)
	log.Printf("Downloading %v packages...\n", downloadCount)
	for _, pack := range packagesToDownload {
		err = r.downloadStoreApply(pack.Location.Href, pack.Checksum.Checksum, hashMap[pack.Checksum.Type], util.Nop)
		if err != nil {
			return err
		}
	}

	recycleCount := len(packagesToRecycle)
	log.Printf("Recycling %v packages...\n", recycleCount)
	for _, pack := range packagesToRecycle {
		err = r.storage.Recycle(pack.Location.Href)
		if err != nil {
			return
		}
	}

	log.Printf("Committing changes...\n")
	err = r.storage.Commit()
	if err != nil {
		return
	}
	return
}

// downloadStore downloads a repo-relative path into a file
func (r *Syncer) downloadStore(path string) error {
	return r.downloadStoreApply(path, "", 0, util.Nop)
}

// downloadStoreApply downloads a repo-relative path into a file, while applying a ReaderConsumer
func (r *Syncer) downloadStoreApply(path string, checksum string, hash crypto.Hash, f util.ReaderConsumer) error {
	log.Printf("Downloading %v...", path)
	body, err := ReadURL(r.Url + "/" + path)
	if err != nil {
		return err
	}

	return util.Compose(r.storage.StoringMapper(path, checksum, hash), f)(body)
}

// processMetadata stores the repo metadata and returns a list of package file
// paths to download
func (r *Syncer) processMetadata(checksumMap map[string]XMLChecksum) (packagesToDownload []XMLPackage, packagesToRecycle []XMLPackage, err error) {
	err = r.downloadStoreApply(repomdPath, "", 0, func(reader io.ReadCloser) (err error) {
		decoder := xml.NewDecoder(reader)
		var repomd XMLRepomd
		err = decoder.Decode(&repomd)
		if err != nil {
			return
		}

		data := repomd.Data
		for i := 0; i < len(data); i++ {
			metadataLocation := data[i].Location.Href
			metadataChecksum := data[i].Checksum
			decision := r.decide(metadataLocation, metadataChecksum, checksumMap)
			switch decision {
			case Download:
				err = r.downloadStore(metadataLocation)
				if err != nil {
					return
				}
			case Recycle:
				r.storage.Recycle(metadataLocation)
			}

			if data[i].Type == "primary" {
				packagesToDownload, packagesToRecycle, err = r.processPrimary(metadataLocation, checksumMap)
			}
		}
		return
	})
	if err != nil {
		return
	}

	err = r.downloadStore(repomdPath + ".asc")
	if err != nil {
		uerr, unexpectedStatusCode := err.(*UnexpectedStatusCodeError)
		if unexpectedStatusCode && uerr.StatusCode == 404 {
			log.Printf("Got 404, ignoring...")
		} else {
			return
		}
	}

	err = r.downloadStore(repomdPath + ".key")
	if err != nil {
		uerr, unexpectedStatusCode := err.(*UnexpectedStatusCodeError)
		if unexpectedStatusCode && uerr.StatusCode == 404 {
			log.Printf("Got 404, ignoring...")
			err = nil
		} else {
			return
		}
	}

	return
}

func (r *Syncer) readMetaData(reader io.Reader) (primary XMLMetaData, err error) {
	gzReader, err := gzip.NewReader(reader)
	if err != nil {
		return
	}
	defer gzReader.Close()

	decoder := xml.NewDecoder(gzReader)
	err = decoder.Decode(&primary)

	return
}

func (r *Syncer) readChecksumMap() (checksumMap map[string]XMLChecksum) {
	checksumMap = make(map[string]XMLChecksum)
	repomdReader, err := r.storage.NewReader(repomdPath, Permanent)
	if err != nil {
		if err == ErrFileNotFound {
			log.Println("First-time sync started")
		} else {
			log.Println(err.Error())
			log.Println("Error while reading previously-downloaded metadata. Starting sync from scratch")
		}
		return
	}
	defer repomdReader.Close()

	decoder := xml.NewDecoder(repomdReader)
	var repomd XMLRepomd
	err = decoder.Decode(&repomd)
	if err != nil {
		log.Println(err.Error())
		log.Println("Error while parsing previously-downloaded metadata. Starting sync from scratch")
		return
	}

	data := repomd.Data
	for i := 0; i < len(data); i++ {
		dataHref := data[i].Location.Href
		dataChecksum := data[i].Checksum
		checksumMap[dataHref] = dataChecksum
		if data[i].Type == "primary" {
			primaryReader, err := r.storage.NewReader(dataHref, Permanent)
			if err != nil {
				return
			}
			primary, err := r.readMetaData(primaryReader)
			if err != nil {
				return
			}
			for _, pack := range primary.Packages {
				checksumMap[pack.Location.Href] = pack.Checksum
			}
		}
	}
	return
}

// processPrimary stores the primary XML metadata file and returns a list of
// package file paths to download
func (r *Syncer) processPrimary(path string, checksumMap map[string]XMLChecksum) (packagesToDownload []XMLPackage, packagesToRecycle []XMLPackage, err error) {
	reader, err := r.storage.NewReader(path, Temporary)
	if err != nil {
		return
	}
	primary, err := r.readMetaData(reader)
	if err != nil {
		return
	}

	allArchs := len(r.archs) == 0
	for _, pack := range primary.Packages {
		if allArchs || pack.Arch == "noarch" || r.archs[pack.Arch] {
			decision := r.decide(pack.Location.Href, pack.Checksum, checksumMap)
			switch decision {
			case Download:
				packagesToDownload = append(packagesToDownload, pack)
			case Recycle:
				packagesToRecycle = append(packagesToRecycle, pack)
			}
		}
	}
	return
}

// Decision encodes what to do with a file
type Decision int

const (
	// Download means the Syncer will download a file
	Download Decision = iota
	// Recycle means the Syncer will copy an existing file without downloading
	Recycle
	// Skip means the Syncer detected an already-existing file and has nothing to do
	Skip
)

func (r *Syncer) decide(location string, checksum XMLChecksum, checksumMap map[string]XMLChecksum) Decision {
	previousChecksum, foundInPermanentLocation := checksumMap[location]
	if !foundInPermanentLocation || previousChecksum.Type != checksum.Type || previousChecksum.Checksum != checksum.Checksum {
		reader, err := r.storage.NewReader(location, Temporary)
		if err != nil {
			log.Printf("...'%v' not found or not recyclable, will be downloaded\n", location)
			return Download
		}
		defer reader.Close()
		readChecksum, err := util.Checksum(reader, hashMap[checksum.Type])
		if err != nil || readChecksum != checksum.Checksum {
			log.Printf("...'%v' found in partially-downloaded repo, not recyclable, will be re-downloaded\n", location)
			return Download
		}
		log.Printf("...'%v' found in partially-downloaded repo, recyclable, will be skipped\n", location)
		return Skip
	}
	log.Printf("...'%v' found in already-downloaded repo, recyclable, will be recycled\n", location)
	return Recycle
}
<commit_msg>Logging: be less verbose<commit_after>package get

import (
	"compress/gzip"
	"crypto"
	"encoding/xml"
	"fmt"
	"io"
	"log"
	"path"

	"github.com/moio/minima/util"
)

// common

// XMLLocation maps a <location> tag in repodata/repomd.xml or repodata/<ID>-primary.xml.gz
type XMLLocation struct {
	Href string `xml:"href,attr"`
}

// repodata/repomd.xml

// XMLRepomd maps a <repomd> tag in repodata/repomd.xml
type XMLRepomd struct {
	Data []XMLData `xml:"data"`
}

// XMLData maps a <data> tag in repodata/repomd.xml
type XMLData struct {
	Type     string      `xml:"type,attr"`
	Location XMLLocation `xml:"location"`
	Checksum XMLChecksum `xml:"checksum"`
}

// repodata/<ID>-primary.xml.gz

// XMLMetaData maps a <metadata> tag in repodata/<ID>-primary.xml.gz
type XMLMetaData struct {
	Packages []XMLPackage `xml:"package"`
}

// XMLPackage maps a <package> tag in repodata/<ID>-primary.xml.gz
type XMLPackage struct {
	Arch     string      `xml:"arch"`
	Location XMLLocation `xml:"location"`
	Checksum XMLChecksum `xml:"checksum"`
}

// XMLChecksum maps a <checksum> tag in repodata/<ID>-primary.xml.gz
type XMLChecksum struct {
	Type     string `xml:"type,attr"`
	Checksum string `xml:",cdata"`
}

var hashMap = map[string]crypto.Hash{
	"sha":    crypto.SHA1,
	"sha1":   crypto.SHA1,
	"sha256": crypto.SHA256,
}

const repomdPath = "repodata/repomd.xml"

// Syncer syncs repos from an HTTP source to a Storage
type Syncer struct {
	// URL of the repo this syncer syncs
	Url     string
	archs   map[string]bool
	storage Storage
}

// NewSyncer creates a new Syncer
func NewSyncer(url string, archs map[string]bool, storage Storage) *Syncer {
	return &Syncer{url, archs, storage}
}

// StoreRepo stores an HTTP repo in a Storage, automatically retrying in case of recoverable errors
func (r *Syncer) StoreRepo() (err error) {
	checksumMap := r.readChecksumMap()
	for i := 0; i < 20; i++ {
		err = r.storeRepo(checksumMap)
		if err == nil {
			return
		}

		uerr, unexpectedStatusCode := err.(*UnexpectedStatusCodeError)
		if unexpectedStatusCode {
			if uerr.StatusCode == 404 {
				log.Printf("Got 404, presumably temporarily, retrying...\n")
			} else {
				return err
			}
		}

		_, checksumError := err.(*util.ChecksumError)
		if checksumError {
			log.Printf(err.Error())
			log.Printf("Checksum did not match, presumably the repo was published while syncing, retrying...\n")
		} else {
			return err
		}
	}

	log.Printf("Too many temporary errors, aborting...\n")
	return err
}

// StoreRepo stores an HTTP repo in a Storage
func (r *Syncer) storeRepo(checksumMap map[string]XMLChecksum) (err error) {
	packagesToDownload, packagesToRecycle, err := r.processMetadata(checksumMap)
	if err != nil {
		return
	}

	downloadCount := len(packagesToDownload)
	log.Printf("Downloading %v packages...\n", downloadCount)
	for i, pack := range packagesToDownload {
		description := fmt.Sprintf("(%v/%v) %v", i+1, downloadCount, path.Base(pack.Location.Href))
		err = r.downloadStoreApply(pack.Location.Href, pack.Checksum.Checksum, description, hashMap[pack.Checksum.Type], util.Nop)
		if err != nil {
			return err
		}
	}

	recycleCount := len(packagesToRecycle)
	log.Printf("Recycling %v packages...\n", recycleCount)
	for _, pack := range packagesToRecycle {
		err = r.storage.Recycle(pack.Location.Href)
		if err != nil {
			return
		}
	}

	log.Printf("Committing changes...\n")
	err = r.storage.Commit()
	if err != nil {
		return
	}
	return
}

// downloadStore downloads a repo-relative path into a file
func (r *Syncer) downloadStore(path string, description string) error {
	return r.downloadStoreApply(path, "", description, 0, util.Nop)
}

// downloadStoreApply downloads a repo-relative path into a file, while applying a ReaderConsumer
func (r *Syncer) downloadStoreApply(path string, checksum string, description string, hash crypto.Hash, f util.ReaderConsumer) error {
	log.Printf("Downloading %v...", description)
	body, err := ReadURL(r.Url + "/" + path)
	if err != nil {
		return err
	}

	return util.Compose(r.storage.StoringMapper(path, checksum, hash), f)(body)
}

// processMetadata stores the repo metadata and returns a list of package file
// paths to download
func (r *Syncer) processMetadata(checksumMap map[string]XMLChecksum) (packagesToDownload []XMLPackage, packagesToRecycle []XMLPackage, err error) {
	err = r.downloadStoreApply(repomdPath, "", path.Base(repomdPath), 0, func(reader io.ReadCloser) (err error) {
		decoder := xml.NewDecoder(reader)
		var repomd XMLRepomd
		err = decoder.Decode(&repomd)
		if err != nil {
			return
		}

		data := repomd.Data
		for i := 0; i < len(data); i++ {
			metadataLocation := data[i].Location.Href
			metadataChecksum := data[i].Checksum
			decision := r.decide(metadataLocation, metadataChecksum, checksumMap)
			switch decision {
			case Download:
				err = r.downloadStore(metadataLocation, path.Base(metadataLocation))
				if err != nil {
					return
				}
			case Recycle:
				r.storage.Recycle(metadataLocation)
			}

			if data[i].Type == "primary" {
				packagesToDownload, packagesToRecycle, err = r.processPrimary(metadataLocation, checksumMap)
			}
		}
		return
	})
	if err != nil {
		return
	}

	ascPath := repomdPath + ".asc"
	err = r.downloadStore(ascPath, path.Base(ascPath))
	if err != nil {
		uerr, unexpectedStatusCode := err.(*UnexpectedStatusCodeError)
		if unexpectedStatusCode && uerr.StatusCode == 404 {
			log.Printf("Got 404, ignoring...")
		} else {
			return
		}
	}

	keyPath := repomdPath + ".key"
	err = r.downloadStore(keyPath, path.Base(keyPath))
	if err != nil {
		uerr, unexpectedStatusCode := err.(*UnexpectedStatusCodeError)
		if unexpectedStatusCode && uerr.StatusCode == 404 {
			log.Printf("Got 404, ignoring...")
			err = nil
		} else {
			return
		}
	}

	return
}

func (r *Syncer) readMetaData(reader io.Reader) (primary XMLMetaData, err error) {
	gzReader, err := gzip.NewReader(reader)
	if err != nil {
		return
	}
	defer gzReader.Close()

	decoder := xml.NewDecoder(gzReader)
	err = decoder.Decode(&primary)

	return
}

func (r *Syncer) readChecksumMap() (checksumMap map[string]XMLChecksum) {
	checksumMap = make(map[string]XMLChecksum)
	repomdReader, err := r.storage.NewReader(repomdPath, Permanent)
	if err != nil {
		if err == ErrFileNotFound {
			log.Println("First-time sync started")
		} else {
			log.Println(err.Error())
			log.Println("Error while reading previously-downloaded metadata. Starting sync from scratch")
		}
		return
	}
	defer repomdReader.Close()

	decoder := xml.NewDecoder(repomdReader)
	var repomd XMLRepomd
	err = decoder.Decode(&repomd)
	if err != nil {
		log.Println(err.Error())
		log.Println("Error while parsing previously-downloaded metadata. Starting sync from scratch")
		return
	}

	data := repomd.Data
	for i := 0; i < len(data); i++ {
		dataHref := data[i].Location.Href
		dataChecksum := data[i].Checksum
		checksumMap[dataHref] = dataChecksum
		if data[i].Type == "primary" {
			primaryReader, err := r.storage.NewReader(dataHref, Permanent)
			if err != nil {
				return
			}
			primary, err := r.readMetaData(primaryReader)
			if err != nil {
				return
			}
			for _, pack := range primary.Packages {
				checksumMap[pack.Location.Href] = pack.Checksum
			}
		}
	}
	return
}

// processPrimary stores the primary XML metadata file and returns a list of
// package file paths to download
func (r *Syncer) processPrimary(path string, checksumMap map[string]XMLChecksum) (packagesToDownload []XMLPackage, packagesToRecycle []XMLPackage, err error) {
	reader, err := r.storage.NewReader(path, Temporary)
	if err != nil {
		return
	}
	primary, err := r.readMetaData(reader)
	if err != nil {
		return
	}

	allArchs := len(r.archs) == 0
	for _, pack := range primary.Packages {
		if allArchs || pack.Arch == "noarch" || r.archs[pack.Arch] {
			decision := r.decide(pack.Location.Href, pack.Checksum, checksumMap)
			switch decision {
			case Download:
				packagesToDownload = append(packagesToDownload, pack)
			case Recycle:
				packagesToRecycle = append(packagesToRecycle, pack)
			}
		}
	}
	return
}

// Decision encodes what to do with a file
type Decision int

const (
	// Download means the Syncer will download a file
	Download Decision = iota
	// Recycle means the Syncer will copy an existing file without downloading
	Recycle
	// Skip means the Syncer detected an already-existing file and has nothing to do
	Skip
)

func (r *Syncer) decide(location string, checksum XMLChecksum, checksumMap map[string]XMLChecksum) Decision {
	previousChecksum, foundInPermanentLocation := checksumMap[location]
	if !foundInPermanentLocation || previousChecksum.Type != checksum.Type || previousChecksum.Checksum != checksum.Checksum {
		reader, err := r.storage.NewReader(location, Temporary)
		if err != nil {
			return Download
		}
		defer reader.Close()
		readChecksum, err := util.Checksum(reader, hashMap[checksum.Type])
		if err != nil || readChecksum != checksum.Checksum {
			return Download
		}
		return Skip
	}
	return Recycle
}
<|endoftext|>


<commit_before>package adhier

import (
	"fmt"
	"math"

	"github.com/ready-steady/numeric/basis/linhat"
	"github.com/ready-steady/numeric/grid/newcot"
)

// Interpolation in one dimension.
func ExampleInterpolator_step() {
	const (
		inputs    = 1
		outputs   = 1
		tolerance = 1e-4
	)

	grid, basis := newcot.NewClosed(inputs), linhat.NewClosed(inputs)
	interpolator := New(grid, basis, NewConfig())

	target := NewGenericTarget(inputs, outputs)
	target.ComputeFunc = func(x, y []float64) {
		if x[0] <= 0.5 {
			y[0] = 1
		} else {
			y[0] = 0
		}
	}
	target.RefineFunc = func(ε []float64) bool {
		return math.Abs(ε[0]) > tolerance
	}

	surrogate := interpolator.Compute(target)

	fmt.Println(surrogate)

	// Output:
	// Surrogate{inputs: 1, outputs: 1, level: 9, nodes: 18}
}

// Interpolation in two dimensions.
func ExampleInterpolator_cube() {
	const (
		inputs    = 2
		outputs   = 1
		tolerance = 1e-4
	)

	grid, basis := newcot.NewClosed(inputs), linhat.NewClosed(inputs)
	interpolator := New(grid, basis, NewConfig())

	target := NewAbsErrorTarget(inputs, outputs, tolerance)
	target.ComputeFunc = func(x, y []float64) {
		if math.Abs(2*x[0]-1) < 0.45 && math.Abs(2*x[1]-1) < 0.45 {
			y[0] = 1
		} else {
			y[0] = 0
		}
	}

	surrogate := interpolator.Compute(target)

	fmt.Println(surrogate)

	// Output:
	// Surrogate{inputs: 2, outputs: 1, level: 9, nodes: 377}
}
<commit_msg>adhier: a cosmetic adjustment<commit_after>package adhier

import (
	"fmt"
	"math"

	"github.com/ready-steady/numeric/basis/linhat"
	"github.com/ready-steady/numeric/grid/newcot"
)

// Interpolation in one dimension.
func ExampleInterpolator_step() {
	const (
		inputs    = 1
		outputs   = 1
		tolerance = 1e-4
	)

	grid, basis := newcot.NewClosed(inputs), linhat.NewClosed(inputs)
	interpolator := New(grid, basis, NewConfig())

	target := NewGenericTarget(inputs, outputs)
	target.ComputeFunc = func(x, y []float64) {
		if x[0] <= 0.5 {
			y[0] = 1
		} else {
			y[0] = 0
		}
	}
	target.RefineFunc = func(Δ []float64) bool {
		return math.Abs(Δ[0]) > tolerance
	}

	surrogate := interpolator.Compute(target)

	fmt.Println(surrogate)

	// Output:
	// Surrogate{inputs: 1, outputs: 1, level: 9, nodes: 18}
}

// Interpolation in two dimensions.
func ExampleInterpolator_cube() {
	const (
		inputs    = 2
		outputs   = 1
		tolerance = 1e-4
	)

	grid, basis := newcot.NewClosed(inputs), linhat.NewClosed(inputs)
	interpolator := New(grid, basis, NewConfig())

	target := NewAbsErrorTarget(inputs, outputs, tolerance)
	target.ComputeFunc = func(x, y []float64) {
		if math.Abs(2*x[0]-1) < 0.45 && math.Abs(2*x[1]-1) < 0.45 {
			y[0] = 1
		} else {
			y[0] = 0
		}
	}

	surrogate := interpolator.Compute(target)

	fmt.Println(surrogate)

	// Output:
	// Surrogate{inputs: 2, outputs: 1, level: 9, nodes: 377}
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"regexp"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/cloud9"
	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccAWSCloud9EnvironmentEc2_basic(t *testing.T) {
	var conf cloud9.Environment

	rString := acctest.RandString(8)
	envName := fmt.Sprintf("tf_acc_env_basic_%s", rString)
	uEnvName := fmt.Sprintf("tf_acc_env_basic_updated_%s", rString)

	resourceName := "aws_cloud9_environment_ec2.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSCloud9EnvironmentEc2Destroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSCloud9EnvironmentEc2Config(envName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCloud9EnvironmentEc2Exists(resourceName, &conf),
					resource.TestCheckResourceAttr(resourceName, "instance_type", "t2.micro"),
					resource.TestCheckResourceAttr(resourceName, "name", envName),
					resource.TestMatchResourceAttr(resourceName, "arn", regexp.MustCompile(`^arn:[^:]+:cloud9:[^:]+:[^:]+:environment:.+$`)),
					resource.TestMatchResourceAttr(resourceName, "owner_arn", regexp.MustCompile(`^arn:`)),
				),
			},
			{
				Config: testAccAWSCloud9EnvironmentEc2Config(uEnvName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCloud9EnvironmentEc2Exists(resourceName, &conf),
					resource.TestCheckResourceAttr(resourceName, "instance_type", "t2.micro"),
					resource.TestCheckResourceAttr(resourceName, "name", uEnvName),
					resource.TestMatchResourceAttr(resourceName, "arn", regexp.MustCompile(`^arn:[^:]+:cloud9:[^:]+:[^:]+:environment:.+$`)),
					resource.TestMatchResourceAttr(resourceName, "owner_arn", regexp.MustCompile(`^arn:`)),
				),
			},
		},
	})
}

func TestAccAWSCloud9EnvironmentEc2_allFields(t *testing.T) {
	var conf cloud9.Environment

	rString := acctest.RandString(8)
	envName := fmt.Sprintf("tf_acc_env_basic_%s", rString)
	uEnvName := fmt.Sprintf("tf_acc_env_basic_updated_%s", rString)
	description := fmt.Sprintf("Tf Acc Test %s", rString)
	uDescription := fmt.Sprintf("Tf Acc Test Updated %s", rString)
	userName := fmt.Sprintf("tf_acc_cloud9_env_%s", rString)

	resourceName := "aws_cloud9_environment_ec2.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSCloud9EnvironmentEc2Destroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSCloud9EnvironmentEc2AllFieldsConfig(envName, description, userName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCloud9EnvironmentEc2Exists(resourceName, &conf),
					resource.TestCheckResourceAttr(resourceName, "instance_type", "t2.micro"),
					resource.TestCheckResourceAttr(resourceName, "name", envName),
					resource.TestMatchResourceAttr(resourceName, "arn", regexp.MustCompile(`^arn:[^:]+:cloud9:[^:]+:[^:]+:environment:.+$`)),
					resource.TestMatchResourceAttr(resourceName, "owner_arn", regexp.MustCompile(`^arn:`)),
					resource.TestCheckResourceAttr(resourceName, "type", "ec2"),
				),
			},
			{
				Config: testAccAWSCloud9EnvironmentEc2AllFieldsConfig(uEnvName, uDescription, userName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCloud9EnvironmentEc2Exists(resourceName, &conf),
					resource.TestCheckResourceAttr(resourceName, "instance_type", "t2.micro"),
					resource.TestCheckResourceAttr(resourceName, "name", uEnvName),
					resource.TestMatchResourceAttr(resourceName, "arn", regexp.MustCompile(`^arn:[^:]+:cloud9:[^:]+:[^:]+:environment:.+$`)),
					resource.TestMatchResourceAttr(resourceName, "owner_arn", regexp.MustCompile(`^arn:`)),
					resource.TestCheckResourceAttr(resourceName, "type", "ec2"),
				),
			},
		},
	})
}

func TestAccAWSCloud9EnvironmentEc2_importBasic(t *testing.T) {
	rString := acctest.RandString(8)
	name := fmt.Sprintf("tf_acc_api_doc_part_import_%s", rString)

	resourceName := "aws_cloud9_environment_ec2.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSCloud9EnvironmentEc2Destroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSCloud9EnvironmentEc2Config(name),
			},
			{
				ResourceName:            resourceName,
				ImportState:             true,
				ImportStateVerify:       true,
				ImportStateVerifyIgnore: []string{"instance_type"},
			},
		},
	})
}

func testAccCheckAWSCloud9EnvironmentEc2Exists(n string, res *cloud9.Environment) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No Cloud9 Environment EC2 ID is set")
		}

		conn := testAccProvider.Meta().(*AWSClient).cloud9conn

		out, err := conn.DescribeEnvironments(&cloud9.DescribeEnvironmentsInput{
			EnvironmentIds: []*string{aws.String(rs.Primary.ID)},
		})
		if err != nil {
			if isAWSErr(err, cloud9.ErrCodeNotFoundException, "") {
				return fmt.Errorf("Cloud9 Environment EC2 (%q) not found", rs.Primary.ID)
			}
			return err
		}
		if len(out.Environments) == 0 {
			return fmt.Errorf("Cloud9 Environment EC2 (%q) not found", rs.Primary.ID)
		}
		env := out.Environments[0]

		*res = *env

		return nil
	}
}

func testAccCheckAWSCloud9EnvironmentEc2Destroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).cloud9conn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_cloud9_environment_ec2" {
			continue
		}

		out, err := conn.DescribeEnvironments(&cloud9.DescribeEnvironmentsInput{
			EnvironmentIds: []*string{aws.String(rs.Primary.ID)},
		})
		if err != nil {
			if isAWSErr(err, cloud9.ErrCodeNotFoundException, "") {
				return nil
			}
			// :'-(
			if isAWSErr(err, "AccessDeniedException", "is not authorized to access this resource") {
				return nil
			}
			return err
		}
		if len(out.Environments) == 0 {
			return nil
		}

		return fmt.Errorf("Cloud9 Environment EC2 %q still exists.", rs.Primary.ID)
	}
	return nil
}

func testAccAWSCloud9EnvironmentEc2Config(name string) string {
	return fmt.Sprintf(`
resource "aws_cloud9_environment_ec2" "test" {
  instance_type = "t2.micro"
  name = "%s"
}
`, name)
}

func testAccAWSCloud9EnvironmentEc2AllFieldsConfig(name, description, userName string) string {
	return fmt.Sprintf(`
resource "aws_cloud9_environment_ec2" "test" {
  instance_type = "t2.micro"
  name = "%s"
  description = "%s"
  automatic_stop_time_minutes = 60
  subnet_id = "${aws_subnet.test.id}"
  owner_arn = "${aws_iam_user.test.arn}"
  depends_on = ["aws_route_table_association.test"]
}

resource "aws_vpc" "test" {
  cidr_block = "10.10.0.0/16"
  tags = {
    Name = "terraform-testacc-cloud9-environment-ec2-all-fields"
  }
}

resource "aws_subnet" "test" {
  vpc_id = "${aws_vpc.test.id}"
  cidr_block = "10.10.0.0/19"
  tags = {
    Name = "tf-acc-cloud9-environment-ec2-all-fields"
  }
}

resource "aws_internet_gateway" "test" {
  vpc_id = "${aws_vpc.test.id}"
}

resource "aws_route_table" "test" {
  vpc_id = "${aws_vpc.test.id}"
  route {
    cidr_block = "0.0.0.0/0"
    gateway_id = "${aws_internet_gateway.test.id}"
  }
}

resource "aws_route_table_association" "test" {
  subnet_id      = "${aws_subnet.test.id}"
  route_table_id = "${aws_route_table.test.id}"
}

resource "aws_iam_user" "test" {
  name = "%s"
}
`, name, description, userName)
}
<commit_msg>tests/service/cloud9: Add PreCheck for service availability<commit_after>package aws

import (
	"fmt"
	"regexp"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/cloud9"
	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccAWSCloud9EnvironmentEc2_basic(t *testing.T) {
	var conf cloud9.Environment

	rString := acctest.RandString(8)
	envName := fmt.Sprintf("tf_acc_env_basic_%s", rString)
	uEnvName := fmt.Sprintf("tf_acc_env_basic_updated_%s", rString)

	resourceName := "aws_cloud9_environment_ec2.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t); testAccPreCheckAWSCloud9(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSCloud9EnvironmentEc2Destroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSCloud9EnvironmentEc2Config(envName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCloud9EnvironmentEc2Exists(resourceName, &conf),
					resource.TestCheckResourceAttr(resourceName, "instance_type", "t2.micro"),
					resource.TestCheckResourceAttr(resourceName, "name", envName),
					resource.TestMatchResourceAttr(resourceName, "arn", regexp.MustCompile(`^arn:[^:]+:cloud9:[^:]+:[^:]+:environment:.+$`)),
					resource.TestMatchResourceAttr(resourceName, "owner_arn", regexp.MustCompile(`^arn:`)),
				),
			},
			{
				Config: testAccAWSCloud9EnvironmentEc2Config(uEnvName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCloud9EnvironmentEc2Exists(resourceName, &conf),
					resource.TestCheckResourceAttr(resourceName, "instance_type", "t2.micro"),
					resource.TestCheckResourceAttr(resourceName, "name", uEnvName),
					resource.TestMatchResourceAttr(resourceName, "arn", regexp.MustCompile(`^arn:[^:]+:cloud9:[^:]+:[^:]+:environment:.+$`)),
					resource.TestMatchResourceAttr(resourceName, "owner_arn", regexp.MustCompile(`^arn:`)),
				),
			},
		},
	})
}

func TestAccAWSCloud9EnvironmentEc2_allFields(t *testing.T) {
	var conf cloud9.Environment

	rString := acctest.RandString(8)
	envName := fmt.Sprintf("tf_acc_env_basic_%s", rString)
	uEnvName := fmt.Sprintf("tf_acc_env_basic_updated_%s", rString)
	description := fmt.Sprintf("Tf Acc Test %s", rString)
	uDescription := fmt.Sprintf("Tf Acc Test Updated %s", rString)
	userName := fmt.Sprintf("tf_acc_cloud9_env_%s", rString)

	resourceName := "aws_cloud9_environment_ec2.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t); testAccPreCheckAWSCloud9(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSCloud9EnvironmentEc2Destroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSCloud9EnvironmentEc2AllFieldsConfig(envName, description, userName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCloud9EnvironmentEc2Exists(resourceName, &conf),
					resource.TestCheckResourceAttr(resourceName, "instance_type", "t2.micro"),
					resource.TestCheckResourceAttr(resourceName, "name", envName),
					resource.TestMatchResourceAttr(resourceName, "arn", regexp.MustCompile(`^arn:[^:]+:cloud9:[^:]+:[^:]+:environment:.+$`)),
					resource.TestMatchResourceAttr(resourceName, "owner_arn", regexp.MustCompile(`^arn:`)),
					resource.TestCheckResourceAttr(resourceName, "type", "ec2"),
				),
			},
			{
				Config: testAccAWSCloud9EnvironmentEc2AllFieldsConfig(uEnvName, uDescription, userName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCloud9EnvironmentEc2Exists(resourceName, &conf),
					resource.TestCheckResourceAttr(resourceName, "instance_type", "t2.micro"),
					resource.TestCheckResourceAttr(resourceName, "name", uEnvName),
					resource.TestMatchResourceAttr(resourceName, "arn", regexp.MustCompile(`^arn:[^:]+:cloud9:[^:]+:[^:]+:environment:.+$`)),
					resource.TestMatchResourceAttr(resourceName, "owner_arn", regexp.MustCompile(`^arn:`)),
					resource.TestCheckResourceAttr(resourceName, "type", "ec2"),
				),
			},
		},
	})
}

func TestAccAWSCloud9EnvironmentEc2_importBasic(t *testing.T) {
	rString := acctest.RandString(8)
	name := fmt.Sprintf("tf_acc_api_doc_part_import_%s", rString)

	resourceName := "aws_cloud9_environment_ec2.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t); testAccPreCheckAWSCloud9(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSCloud9EnvironmentEc2Destroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSCloud9EnvironmentEc2Config(name),
			},
			{
				ResourceName:            resourceName,
				ImportState:             true,
				ImportStateVerify:       true,
				ImportStateVerifyIgnore: []string{"instance_type"},
			},
		},
	})
}

func testAccCheckAWSCloud9EnvironmentEc2Exists(n string, res *cloud9.Environment) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No Cloud9 Environment EC2 ID is set")
		}

		conn := testAccProvider.Meta().(*AWSClient).cloud9conn

		out, err := conn.DescribeEnvironments(&cloud9.DescribeEnvironmentsInput{
			EnvironmentIds: []*string{aws.String(rs.Primary.ID)},
		})
		if err != nil {
			if isAWSErr(err, cloud9.ErrCodeNotFoundException, "") {
				return fmt.Errorf("Cloud9 Environment EC2 (%q) not found", rs.Primary.ID)
			}
			return err
		}
		if len(out.Environments) == 0 {
			return fmt.Errorf("Cloud9 Environment EC2 (%q) not found", rs.Primary.ID)
		}
		env := out.Environments[0]

		*res = *env

		return nil
	}
}

func testAccCheckAWSCloud9EnvironmentEc2Destroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).cloud9conn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_cloud9_environment_ec2" {
			continue
		}

		out, err := conn.DescribeEnvironments(&cloud9.DescribeEnvironmentsInput{
			EnvironmentIds: []*string{aws.String(rs.Primary.ID)},
		})
		if err != nil {
			if isAWSErr(err, cloud9.ErrCodeNotFoundException, "") {
				return nil
			}
			// :'-(
			if isAWSErr(err, "AccessDeniedException", "is not authorized to access this resource") {
				return nil
			}
			return err
		}
		if len(out.Environments) == 0 {
			return nil
		}

		return fmt.Errorf("Cloud9 Environment EC2 %q still exists.", rs.Primary.ID)
	}
	return nil
}

func testAccPreCheckAWSCloud9(t *testing.T) {
	conn := testAccProvider.Meta().(*AWSClient).cloud9conn

	input := &cloud9.ListEnvironmentsInput{}

	_, err := conn.ListEnvironments(input)

	if testAccPreCheckSkipError(err) {
		t.Skipf("skipping acceptance testing: %s", err)
	}

	if err != nil {
		t.Fatalf("unexpected PreCheck error: %s", err)
	}
}

func testAccAWSCloud9EnvironmentEc2Config(name string) string {
	return fmt.Sprintf(`
resource "aws_cloud9_environment_ec2" "test" {
  instance_type = "t2.micro"
  name = "%s"
}
`, name)
}

func testAccAWSCloud9EnvironmentEc2AllFieldsConfig(name, description, userName string) string {
	return fmt.Sprintf(`
resource "aws_cloud9_environment_ec2" "test" {
  instance_type = "t2.micro"
  name = "%s"
  description = "%s"
  automatic_stop_time_minutes = 60
  subnet_id = "${aws_subnet.test.id}"
  owner_arn = "${aws_iam_user.test.arn}"
  depends_on = ["aws_route_table_association.test"]
}

resource "aws_vpc" "test" {
  cidr_block = "10.10.0.0/16"
  tags = {
    Name = "terraform-testacc-cloud9-environment-ec2-all-fields"
  }
}

resource "aws_subnet" "test" {
  vpc_id = "${aws_vpc.test.id}"
  cidr_block = "10.10.0.0/19"
  tags = {
    Name = "tf-acc-cloud9-environment-ec2-all-fields"
  }
}

resource "aws_internet_gateway" "test" {
  vpc_id = "${aws_vpc.test.id}"
}

resource "aws_route_table" "test" {
  vpc_id = "${aws_vpc.test.id}"
  route {
    cidr_block = "0.0.0.0/0"
    gateway_id = "${aws_internet_gateway.test.id}"
  }
}

resource "aws_route_table_association" "test" {
  subnet_id      = "${aws_subnet.test.id}"
  route_table_id = "${aws_route_table.test.id}"
}

resource "aws_iam_user" "test" {
  name = "%s"
}
`, name, description, userName)
}
<|endoftext|>