<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package azure

import (
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	. "launchpad.net/gocheck"
	"launchpad.net/gwacl"
	"launchpad.net/juju-core/errors"
)

type StorageSuite struct {
	ProviderSuite
}

var _ = Suite(new(StorageSuite))

func makeResponse(content string, status int) *http.Response {
	return &http.Response{
		Status:     fmt.Sprintf("%d", status),
		StatusCode: status,
		Body:       ioutil.NopCloser(strings.NewReader(content)),
	}
}

// MockingTransport is used as an http.Client.Transport for testing.  It
// records the sequence of requests, and returns a predetermined sequence of
// Responses and errors.
type MockingTransport struct {
	Exchanges     []*MockingTransportExchange
	ExchangeCount int
}

// MockingTransport implements the http.RoundTripper interface.
var _ http.RoundTripper = &MockingTransport{}

func (t *MockingTransport) AddExchange(response *http.Response, err error) {
	exchange := MockingTransportExchange{Response: response, Error: err}
	t.Exchanges = append(t.Exchanges, &exchange)
}

func (t *MockingTransport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
	exchange := t.Exchanges[t.ExchangeCount]
	t.ExchangeCount += 1
	exchange.Request = req
	return exchange.Response, exchange.Error
}

// MockingTransportExchange is a recording of a request and a response over
// HTTP.
type MockingTransportExchange struct {
	Request  *http.Request
	Response *http.Response
	Error    error
}

// testStorageContext is a struct implementing the storageContext interface
// used in test.  It will return, via getContainer() and getStorageContext()
// the objects used at creation time.
type testStorageContext struct {
	container      string
	storageContext *gwacl.StorageContext
}

func (context *testStorageContext) getContainer() string {
	return context.container
}

func (context *testStorageContext) getStorageContext() (*gwacl.StorageContext, error) {
	return context.storageContext, nil
}

// makeFakeStorage creates a test azureStorage object that will talk to a
// fake HTTP server set up to always return preconfigured http.Response objects.
// The MockingTransport object can be used to check that the expected query has
// been issued to the test server.
func makeFakeStorage(container, account string) (azureStorage, *MockingTransport) {
	transport := &MockingTransport{}
	client := &http.Client{Transport: transport}
	storageContext := gwacl.NewTestStorageContext(client)
	storageContext.Account = "account"
	context := &testStorageContext{container: container, storageContext: storageContext}
	azStorage := azureStorage{context}
	return azStorage, transport
}

var blobListResponse = `
  <?xml version="1.0" encoding="utf-8"?>
  <EnumerationResults ContainerName="http://myaccount.blob.core.windows.net/mycontainer">
    <Prefix>prefix</Prefix>
    <Marker>marker</Marker>
    <MaxResults>maxresults</MaxResults>
    <Delimiter>delimiter</Delimiter>
    <Blobs>
      <Blob>
        <Name>prefix-1</Name>
        <Url>blob-url1</Url>
      </Blob>
      <Blob>
        <Name>prefix-2</Name>
        <Url>blob-url2</Url>
      </Blob>
    </Blobs>
    <NextMarker />
  </EnumerationResults>`

func (*StorageSuite) TestList(c *C) {
	container := "container"
	response := makeResponse(blobListResponse, http.StatusOK)
	azStorage, transport := makeFakeStorage(container, "account")
	transport.AddExchange(response, nil)

	prefix := "prefix"
	names, err := azStorage.List(prefix)
	c.Assert(err, IsNil)
	c.Assert(transport.ExchangeCount, Equals, 1)
	// The prefix has been passed down as a query parameter.
	c.Check(transport.Exchanges[0].Request.URL.Query()["prefix"], DeepEquals, []string{prefix})
	// The container name is used in the requested URL.
	c.Check(transport.Exchanges[0].Request.URL.String(), Matches, ".*"+container+".*")
	c.Check(names, DeepEquals, []string{"prefix-1", "prefix-2"})
}

func (*StorageSuite) TestGet(c *C) {
	blobContent := "test blob"
	container := "container"
	filename := "blobname"
	response := makeResponse(blobContent, http.StatusOK)
	azStorage, transport := makeFakeStorage(container, "account")
	transport.AddExchange(response, nil)

	reader, err := azStorage.Get(filename)
	c.Assert(err, IsNil)
	c.Assert(reader, NotNil)
	defer reader.Close()

	context, err := azStorage.getStorageContext()
	c.Assert(err, IsNil)
	c.Assert(transport.ExchangeCount, Equals, 1)
	c.Check(transport.Exchanges[0].Request.URL.String(), Matches, context.GetFileURL(container, filename)+"?.*")
	data, err := ioutil.ReadAll(reader)
	c.Assert(err, IsNil)
	c.Check(string(data), Equals, blobContent)
}

func (*StorageSuite) TestGetReturnsNotFoundIf404(c *C) {
	container := "container"
	filename := "blobname"
	response := makeResponse("not found", http.StatusNotFound)
	azStorage, transport := makeFakeStorage(container, "account")
	transport.AddExchange(response, nil)
	_, err := azStorage.Get(filename)
	c.Assert(err, NotNil)
	c.Check(errors.IsNotFoundError(err), Equals, true)
}

func (*StorageSuite) TestPut(c *C) {
	blobContent := "test blob"
	container := "container"
	filename := "blobname"
	response := makeResponse("", http.StatusCreated)
	azStorage, transport := makeFakeStorage(container, "account")
	transport.AddExchange(response, nil)
	transport.AddExchange(response, nil)
	err := azStorage.Put(filename, strings.NewReader(blobContent), int64(len(blobContent)))
	c.Assert(err, IsNil)

	context, err := azStorage.getStorageContext()
	c.Assert(err, IsNil)
	c.Assert(transport.ExchangeCount, Equals, 2)
	c.Check(transport.Exchanges[0].Request.URL.String(), Matches, context.GetFileURL(container, filename)+"?.*")
}

func (*StorageSuite) TestRemove(c *C) {
	container := "container"
	filename := "blobname"
	response := makeResponse("", http.StatusAccepted)
	azStorage, transport := makeFakeStorage(container, "account")
	transport.AddExchange(response, nil)
	err := azStorage.Remove(filename)
	c.Assert(err, IsNil)

	context, err := azStorage.getStorageContext()
	c.Assert(err, IsNil)
	c.Assert(transport.ExchangeCount, Equals, 1)
	c.Check(transport.Exchanges[0].Request.URL.String(), Matches, context.GetFileURL(container, filename)+"?.*")
	c.Check(transport.Exchanges[0].Request.Method, Equals, "DELETE")
}

func (*StorageSuite) TestRemoveErrors(c *C) {
	container := "container"
	filename := "blobname"
	response := makeResponse("", http.StatusForbidden)
	azStorage, transport := makeFakeStorage(container, "account")
	transport.AddExchange(response, nil)
	err := azStorage.Remove(filename)
	c.Assert(err, NotNil)
}

func (*StorageSuite) TestRemoveAll(c *C) {
	// When we ask gwacl to remove all blobs, it calls DeleteContainer.
	response := makeResponse("", http.StatusAccepted)
	storage, transport := makeFakeStorage("cntnr", "account")
	transport.AddExchange(response, nil)

	err := storage.RemoveAll()
	c.Assert(err, IsNil)

	_, err = storage.getStorageContext()
	c.Assert(err, IsNil)
	// Without going too far into gwacl's innards, this is roughly what
	// it needs to do in order to delete a container.
	c.Assert(transport.ExchangeCount, Equals, 1)
	c.Check(transport.Exchanges[0].Request.URL.String(), Matches, "http.*/cntnr?.*restype=container.*")
	c.Check(transport.Exchanges[0].Request.Method, Equals, "DELETE")
}

func (*StorageSuite) TestRemoveNonExistentBlobSucceeds(c *C) {
	container := "container"
	filename := "blobname"
	response := makeResponse("", http.StatusNotFound)
	azStorage, transport := makeFakeStorage(container, "account")
	transport.AddExchange(response, nil)
	err := azStorage.Remove(filename)
	c.Assert(err, IsNil)
}

func (*StorageSuite) TestURL(c *C) {
	container := "container"
	filename := "blobname"
	account := "account"
	azStorage, _ := makeFakeStorage(container, account)
	URL, err := azStorage.URL(filename)
	c.Assert(err, IsNil)
	parsedURL, err := url.Parse(URL)
	c.Assert(err, IsNil)
	c.Check(parsedURL.Host, Matches, fmt.Sprintf("%s.blob.core.windows.net", account))
	c.Check(parsedURL.Path, Matches, fmt.Sprintf("/%s/%s", container, filename))
	values, err := url.ParseQuery(parsedURL.RawQuery)
	c.Assert(err, IsNil)
	signature := values.Get("sig")
	// The query string contains a non-empty signature.
	c.Check(signature, Not(HasLen), 0)
	// The signature is base64-encoded.
	_, err = base64.StdEncoding.DecodeString(signature)
	c.Assert(err, IsNil)
}

func (*StorageSuite) TestCreateContainerWhenNotAlreadyExists(c *C) {
	azStorage, transport := makeFakeStorage("", "account")
	transport.AddExchange(makeResponse("", http.StatusNotFound), nil)
	transport.AddExchange(makeResponse("", http.StatusCreated), nil)

	err := azStorage.CreateContainer("cntnr")

	c.Assert(err, IsNil)
	c.Assert(transport.ExchangeCount, Equals, 2)
	// Without going too far into gwacl's innards, this is roughly what
	// it needs to do in order to call GetContainerProperties.
	c.Check(transport.Exchanges[0].Request.URL.String(), Matches, "http.*/cntnr?.*restype=container.*")
	c.Check(transport.Exchanges[0].Request.Method, Equals, "GET")

	// ... and for CreateContainer.
	c.Check(transport.Exchanges[1].Request.URL.String(), Matches, "http.*/cntnr?.*restype=container.*")
	c.Check(transport.Exchanges[1].Request.Method, Equals, "PUT")
}

func (*StorageSuite) TestCreateContainerWhenAlreadyExists(c *C) {
	container := ""
	azStorage, transport := makeFakeStorage(container, "account")
	header := make(http.Header)
	header.Add("Last-Modified", "last-modified")
	header.Add("ETag", "etag")
	header.Add("X-Ms-Lease-Status", "status")
	header.Add("X-Ms-Lease-State", "state")
	header.Add("X-Ms-Lease-Duration", "duration")
	response := makeResponse("", http.StatusOK)
	response.Header = header
	transport.AddExchange(response, nil)

	err := azStorage.CreateContainer("cntnr")

	c.Assert(err, IsNil)
	c.Assert(transport.ExchangeCount, Equals, 1)
	// Without going too far into gwacl's innards, this is roughly what
	// it needs to do in order to call GetContainerProperties.
	c.Check(transport.Exchanges[0].Request.URL.String(), Matches, "http.*/cntnr?.*restype=container.*")
	c.Check(transport.Exchanges[0].Request.Method, Equals, "GET")
}

func (*StorageSuite) TestDeleteContainer(c *C) {
	azStorage, transport := makeFakeStorage("", "account")
	transport.AddExchange(makeResponse("", http.StatusAccepted), nil)

	err := azStorage.DeleteContainer("cntnr")

	c.Assert(err, IsNil)
	c.Assert(transport.ExchangeCount, Equals, 1)
	// Without going too far into gwacl's innards, this is roughly what
	// it needs to do in order to call GetContainerProperties.
	c.Check(transport.Exchanges[0].Request.URL.String(), Matches, "http.*/cntnr?.*restype=container.*")
	c.Check(transport.Exchanges[0].Request.Method, Equals, "DELETE")
}
<commit_msg>rvb's review comments<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package azure

import (
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	. "launchpad.net/gocheck"
	"launchpad.net/gwacl"
	"launchpad.net/juju-core/errors"
)

type StorageSuite struct {
	ProviderSuite
}

var _ = Suite(new(StorageSuite))

func makeResponse(content string, status int) *http.Response {
	return &http.Response{
		Status:     fmt.Sprintf("%d", status),
		StatusCode: status,
		Body:       ioutil.NopCloser(strings.NewReader(content)),
	}
}

// MockingTransport is used as an http.Client.Transport for testing.  It
// records the sequence of requests, and returns a predetermined sequence of
// Responses and errors.
type MockingTransport struct {
	Exchanges     []*MockingTransportExchange
	ExchangeCount int
}

// MockingTransport implements the http.RoundTripper interface.
var _ http.RoundTripper = &MockingTransport{}

func (t *MockingTransport) AddExchange(response *http.Response, err error) {
	exchange := MockingTransportExchange{Response: response, Error: err}
	t.Exchanges = append(t.Exchanges, &exchange)
}

func (t *MockingTransport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
	exchange := t.Exchanges[t.ExchangeCount]
	t.ExchangeCount += 1
	exchange.Request = req
	return exchange.Response, exchange.Error
}

// MockingTransportExchange is a recording of a request and a response over
// HTTP.
type MockingTransportExchange struct {
	Request  *http.Request
	Response *http.Response
	Error    error
}

// testStorageContext is a struct implementing the storageContext interface
// used in test.  It will return, via getContainer() and getStorageContext()
// the objects used at creation time.
type testStorageContext struct {
	container      string
	storageContext *gwacl.StorageContext
}

func (context *testStorageContext) getContainer() string {
	return context.container
}

func (context *testStorageContext) getStorageContext() (*gwacl.StorageContext, error) {
	return context.storageContext, nil
}

// makeFakeStorage creates a test azureStorage object that will talk to a
// fake HTTP server set up to always return preconfigured http.Response objects.
// The MockingTransport object can be used to check that the expected query has
// been issued to the test server.
func makeFakeStorage(container, account string) (azureStorage, *MockingTransport) {
	transport := &MockingTransport{}
	client := &http.Client{Transport: transport}
	storageContext := gwacl.NewTestStorageContext(client)
	storageContext.Account = account
	context := &testStorageContext{container: container, storageContext: storageContext}
	azStorage := azureStorage{context}
	return azStorage, transport
}

var blobListResponse = `
  <?xml version="1.0" encoding="utf-8"?>
  <EnumerationResults ContainerName="http://myaccount.blob.core.windows.net/mycontainer">
    <Prefix>prefix</Prefix>
    <Marker>marker</Marker>
    <MaxResults>maxresults</MaxResults>
    <Delimiter>delimiter</Delimiter>
    <Blobs>
      <Blob>
        <Name>prefix-1</Name>
        <Url>blob-url1</Url>
      </Blob>
      <Blob>
        <Name>prefix-2</Name>
        <Url>blob-url2</Url>
      </Blob>
    </Blobs>
    <NextMarker />
  </EnumerationResults>`

func (*StorageSuite) TestList(c *C) {
	container := "container"
	response := makeResponse(blobListResponse, http.StatusOK)
	azStorage, transport := makeFakeStorage(container, "account")
	transport.AddExchange(response, nil)

	prefix := "prefix"
	names, err := azStorage.List(prefix)
	c.Assert(err, IsNil)
	c.Assert(transport.ExchangeCount, Equals, 1)
	// The prefix has been passed down as a query parameter.
	c.Check(transport.Exchanges[0].Request.URL.Query()["prefix"], DeepEquals, []string{prefix})
	// The container name is used in the requested URL.
	c.Check(transport.Exchanges[0].Request.URL.String(), Matches, ".*"+container+".*")
	c.Check(names, DeepEquals, []string{"prefix-1", "prefix-2"})
}

func (*StorageSuite) TestGet(c *C) {
	blobContent := "test blob"
	container := "container"
	filename := "blobname"
	response := makeResponse(blobContent, http.StatusOK)
	azStorage, transport := makeFakeStorage(container, "account")
	transport.AddExchange(response, nil)

	reader, err := azStorage.Get(filename)
	c.Assert(err, IsNil)
	c.Assert(reader, NotNil)
	defer reader.Close()

	context, err := azStorage.getStorageContext()
	c.Assert(err, IsNil)
	c.Assert(transport.ExchangeCount, Equals, 1)
	c.Check(transport.Exchanges[0].Request.URL.String(), Matches, context.GetFileURL(container, filename)+"?.*")
	data, err := ioutil.ReadAll(reader)
	c.Assert(err, IsNil)
	c.Check(string(data), Equals, blobContent)
}

func (*StorageSuite) TestGetReturnsNotFoundIf404(c *C) {
	container := "container"
	filename := "blobname"
	response := makeResponse("not found", http.StatusNotFound)
	azStorage, transport := makeFakeStorage(container, "account")
	transport.AddExchange(response, nil)
	_, err := azStorage.Get(filename)
	c.Assert(err, NotNil)
	c.Check(errors.IsNotFoundError(err), Equals, true)
}

func (*StorageSuite) TestPut(c *C) {
	blobContent := "test blob"
	container := "container"
	filename := "blobname"
	response := makeResponse("", http.StatusCreated)
	azStorage, transport := makeFakeStorage(container, "account")
	transport.AddExchange(response, nil)
	transport.AddExchange(response, nil)
	err := azStorage.Put(filename, strings.NewReader(blobContent), int64(len(blobContent)))
	c.Assert(err, IsNil)

	context, err := azStorage.getStorageContext()
	c.Assert(err, IsNil)
	c.Assert(transport.ExchangeCount, Equals, 2)
	c.Check(transport.Exchanges[0].Request.URL.String(), Matches, context.GetFileURL(container, filename)+"?.*")
}

func (*StorageSuite) TestRemove(c *C) {
	container := "container"
	filename := "blobname"
	response := makeResponse("", http.StatusAccepted)
	azStorage, transport := makeFakeStorage(container, "account")
	transport.AddExchange(response, nil)
	err := azStorage.Remove(filename)
	c.Assert(err, IsNil)

	context, err := azStorage.getStorageContext()
	c.Assert(err, IsNil)
	c.Assert(transport.ExchangeCount, Equals, 1)
	c.Check(transport.Exchanges[0].Request.URL.String(), Matches, context.GetFileURL(container, filename)+"?.*")
	c.Check(transport.Exchanges[0].Request.Method, Equals, "DELETE")
}

func (*StorageSuite) TestRemoveErrors(c *C) {
	container := "container"
	filename := "blobname"
	response := makeResponse("", http.StatusForbidden)
	azStorage, transport := makeFakeStorage(container, "account")
	transport.AddExchange(response, nil)
	err := azStorage.Remove(filename)
	c.Assert(err, NotNil)
}

func (*StorageSuite) TestRemoveAll(c *C) {
	// When we ask gwacl to remove all blobs, it calls DeleteContainer.
	response := makeResponse("", http.StatusAccepted)
	storage, transport := makeFakeStorage("cntnr", "account")
	transport.AddExchange(response, nil)

	err := storage.RemoveAll()
	c.Assert(err, IsNil)

	_, err = storage.getStorageContext()
	c.Assert(err, IsNil)
	// Without going too far into gwacl's innards, this is roughly what
	// it needs to do in order to delete a container.
	c.Assert(transport.ExchangeCount, Equals, 1)
	c.Check(transport.Exchanges[0].Request.URL.String(), Matches, "http.*/cntnr?.*restype=container.*")
	c.Check(transport.Exchanges[0].Request.Method, Equals, "DELETE")
}

func (*StorageSuite) TestRemoveNonExistentBlobSucceeds(c *C) {
	container := "container"
	filename := "blobname"
	response := makeResponse("", http.StatusNotFound)
	azStorage, transport := makeFakeStorage(container, "account")
	transport.AddExchange(response, nil)
	err := azStorage.Remove(filename)
	c.Assert(err, IsNil)
}

func (*StorageSuite) TestURL(c *C) {
	container := "container"
	filename := "blobname"
	account := "account"
	azStorage, _ := makeFakeStorage(container, account)
	URL, err := azStorage.URL(filename)
	c.Assert(err, IsNil)
	parsedURL, err := url.Parse(URL)
	c.Assert(err, IsNil)
	c.Check(parsedURL.Host, Matches, fmt.Sprintf("%s.blob.core.windows.net", account))
	c.Check(parsedURL.Path, Matches, fmt.Sprintf("/%s/%s", container, filename))
	values, err := url.ParseQuery(parsedURL.RawQuery)
	c.Assert(err, IsNil)
	signature := values.Get("sig")
	// The query string contains a non-empty signature.
	c.Check(signature, Not(HasLen), 0)
	// The signature is base64-encoded.
	_, err = base64.StdEncoding.DecodeString(signature)
	c.Assert(err, IsNil)
}

func (*StorageSuite) TestCreateContainerWhenNotAlreadyExists(c *C) {
	azStorage, transport := makeFakeStorage("", "account")
	transport.AddExchange(makeResponse("", http.StatusNotFound), nil)
	transport.AddExchange(makeResponse("", http.StatusCreated), nil)

	err := azStorage.CreateContainer("cntnr")

	c.Assert(err, IsNil)
	c.Assert(transport.ExchangeCount, Equals, 2)
	// Without going too far into gwacl's innards, this is roughly what
	// it needs to do in order to call GetContainerProperties.
	c.Check(transport.Exchanges[0].Request.URL.String(), Matches, "http.*/cntnr?.*restype=container.*")
	c.Check(transport.Exchanges[0].Request.Method, Equals, "GET")

	// ... and for CreateContainer.
	c.Check(transport.Exchanges[1].Request.URL.String(), Matches, "http.*/cntnr?.*restype=container.*")
	c.Check(transport.Exchanges[1].Request.Method, Equals, "PUT")
}

func (*StorageSuite) TestCreateContainerWhenAlreadyExists(c *C) {
	container := ""
	azStorage, transport := makeFakeStorage(container, "account")
	header := make(http.Header)
	header.Add("Last-Modified", "last-modified")
	header.Add("ETag", "etag")
	header.Add("X-Ms-Lease-Status", "status")
	header.Add("X-Ms-Lease-State", "state")
	header.Add("X-Ms-Lease-Duration", "duration")
	response := makeResponse("", http.StatusOK)
	response.Header = header
	transport.AddExchange(response, nil)

	err := azStorage.CreateContainer("cntnr")

	c.Assert(err, IsNil)
	c.Assert(transport.ExchangeCount, Equals, 1)
	// Without going too far into gwacl's innards, this is roughly what
	// it needs to do in order to call GetContainerProperties.
	c.Check(transport.Exchanges[0].Request.URL.String(), Matches, "http.*/cntnr?.*restype=container.*")
	c.Check(transport.Exchanges[0].Request.Method, Equals, "GET")
}

func (*StorageSuite) TestDeleteContainer(c *C) {
	azStorage, transport := makeFakeStorage("", "account")
	transport.AddExchange(makeResponse("", http.StatusAccepted), nil)

	err := azStorage.DeleteContainer("cntnr")

	c.Assert(err, IsNil)
	c.Assert(transport.ExchangeCount, Equals, 1)
	// Without going too far into gwacl's innards, this is roughly what
	// it needs to do in order to call GetContainerProperties.
	c.Check(transport.Exchanges[0].Request.URL.String(), Matches, "http.*/cntnr?.*restype=container.*")
	c.Check(transport.Exchanges[0].Request.Method, Equals, "DELETE")
}
<|endoftext|>


<commit_before>package cmd

import (
	"io/ioutil"
	"os"
	"os/signal"

	log "github.com/sirupsen/logrus"
	"github.com/spf13/cobra"

	"github.com/cvmfs/ducc/lib"
)

func init() {
	loopCmd.Flags().BoolVarP(&overwriteLayer, "overwrite-layers", "f", false, "overwrite the layer if they are already inside the CVMFS repository")
	loopCmd.Flags().BoolVarP(&convertAgain, "convert-again", "g", false, "convert again images that are already successfull converted")
	loopCmd.Flags().BoolVarP(&skipFlat, "skip-flat", "s", false, "do not create a flat images (compatible with singularity)")
	loopCmd.Flags().BoolVarP(&skipLayers, "skip-layers", "d", false, "do not unpack the layers into the repository, implies --skip-thin-image")
	loopCmd.Flags().BoolVarP(&skipThinImage, "skip-thin-image", "i", false, "do not create and push the docker thin image")
	rootCmd.AddCommand(loopCmd)
}

var loopCmd = &cobra.Command{
	Use:   "loop wish-list.yaml",
	Short: "An infinite loop that keep converting all the images",
	Args:  cobra.ExactArgs(1),
	Run: func(cmd *cobra.Command, args []string) {
		AliveMessage()
		defer lib.ExecCommand("docker", "system", "prune", "--force", "--all")
		showWeReceivedSignal := make(chan os.Signal, 1)
		signal.Notify(showWeReceivedSignal, os.Interrupt)

		stopWishLoopSignal := make(chan os.Signal, 1)
		signal.Notify(stopWishLoopSignal, os.Interrupt)

		go func() {
			<-showWeReceivedSignal
			lib.Log().Info("Received SIGINT (Ctrl-C) waiting the last layer to upload then exiting.")
		}()

		checkQuitSignal := func() {
			select {
			case <-stopWishLoopSignal:
				lib.Log().Info("Received SIGINT (Ctrl-C) Quitting")
				os.Exit(1)
			default:
			}
		}

		for {
			data, err := ioutil.ReadFile(args[0])
			if err != nil {
				lib.LogE(err).Fatal("Impossible to read the recipe file")
				os.Exit(1)
			}
			recipe, err := lib.ParseYamlRecipeV1(data)
			if err != nil {
				lib.LogE(err).Fatal("Impossible to parse the recipe file")
				os.Exit(1)
			}
			if !lib.RepositoryExists(recipe.Repo) {
				lib.LogE(err).Error("The repository does not seems to exists.")
				os.Exit(RepoNotExistsError)
			}
			for wish := range recipe.Wishes {
				fields := log.Fields{"input image": wish.InputName,
					"repository":   wish.CvmfsRepo,
					"output image": wish.OutputName}
				lib.Log().WithFields(fields).Info("Start conversion of wish")
				if !skipLayers {
					err = lib.ConvertWishDocker(wish, convertAgain, overwriteLayer, !skipThinImage)
					if err != nil {
						lib.LogE(err).WithFields(fields).Error("Error in converting wish (docker), going on")
					}
				}
				if !skipFlat {
					err = lib.ConvertWishSingularity(wish)
					if err != nil {
						lib.LogE(err).WithFields(fields).Error("Error in converting wish (singularity), going on")
					}
				}
				checkQuitSignal()
			}
			checkQuitSignal()
		}
	},
}
<commit_msg>fix weird wording<commit_after>package cmd

import (
	"io/ioutil"
	"os"
	"os/signal"

	log "github.com/sirupsen/logrus"
	"github.com/spf13/cobra"

	"github.com/cvmfs/ducc/lib"
)

func init() {
	loopCmd.Flags().BoolVarP(&overwriteLayer, "overwrite-layers", "f", false, "overwrite the layer if they are already inside the CVMFS repository")
	loopCmd.Flags().BoolVarP(&convertAgain, "convert-again", "g", false, "convert again images that are already successfull converted")
	loopCmd.Flags().BoolVarP(&skipFlat, "skip-flat", "s", false, "do not create a flat images (compatible with singularity)")
	loopCmd.Flags().BoolVarP(&skipLayers, "skip-layers", "d", false, "do not unpack the layers into the repository, implies --skip-thin-image")
	loopCmd.Flags().BoolVarP(&skipThinImage, "skip-thin-image", "i", false, "do not create and push the docker thin image")
	rootCmd.AddCommand(loopCmd)
}

var loopCmd = &cobra.Command{
	Use:   "loop wish-list.yaml",
	Short: "An infinite loop that keep converting all the images",
	Args:  cobra.ExactArgs(1),
	Run: func(cmd *cobra.Command, args []string) {
		AliveMessage()
		defer lib.ExecCommand("docker", "system", "prune", "--force", "--all")
		showWeReceivedSignal := make(chan os.Signal, 1)
		signal.Notify(showWeReceivedSignal, os.Interrupt)

		stopWishLoopSignal := make(chan os.Signal, 1)
		signal.Notify(stopWishLoopSignal, os.Interrupt)

		go func() {
			<-showWeReceivedSignal
			lib.Log().Info("Received SIGINT (Ctrl-C) waiting the last layer to upload then exiting.")
		}()

		checkQuitSignal := func() {
			select {
			case <-stopWishLoopSignal:
				lib.Log().Info("Received SIGINT (Ctrl-C) Quitting")
				os.Exit(1)
			default:
			}
		}

		for {
			data, err := ioutil.ReadFile(args[0])
			if err != nil {
				lib.LogE(err).Fatal("Impossible to read the recipe file")
				os.Exit(1)
			}
			recipe, err := lib.ParseYamlRecipeV1(data)
			if err != nil {
				lib.LogE(err).Fatal("Impossible to parse the recipe file")
				os.Exit(1)
			}
			if !lib.RepositoryExists(recipe.Repo) {
				lib.LogE(err).Error("The repository does not exists.")
				os.Exit(RepoNotExistsError)
			}
			for wish := range recipe.Wishes {
				fields := log.Fields{"input image": wish.InputName,
					"repository":   wish.CvmfsRepo,
					"output image": wish.OutputName}
				lib.Log().WithFields(fields).Info("Start conversion of wish")
				if !skipLayers {
					err = lib.ConvertWishDocker(wish, convertAgain, overwriteLayer, !skipThinImage)
					if err != nil {
						lib.LogE(err).WithFields(fields).Error("Error in converting wish (docker), going on")
					}
				}
				if !skipFlat {
					err = lib.ConvertWishSingularity(wish)
					if err != nil {
						lib.LogE(err).WithFields(fields).Error("Error in converting wish (singularity), going on")
					}
				}
				checkQuitSignal()
			}
			checkQuitSignal()
		}
	},
}
<|endoftext|>


<commit_before>package dynaml

import (
	"github.com/mandelsoft/spiff/yaml"
	"os"
)

func func_lookup(directory bool, arguments []interface{}, binding Binding) (interface{}, EvaluationInfo, bool) {
	info := DefaultInfo()

	paths := []string{}

	switch len(arguments) {
	case 0, 1:
		return info.Error("lookup_file requires at least two arguments")
	default:
		for index, arg := range arguments[1:] {
			switch v := arg.(type) {
			case []yaml.Node:
				for _, p := range v {
					if p.Value() == nil {
						continue
					}
					switch v := p.Value().(type) {
					case string:
						paths = append(paths, v)
					default:
						return info.Error("lookup_file: argument %d must be a list of strings", index)
					}
				}
			case string:
				paths = append(paths, v)
			default:
				return info.Error("lookup_file: argument %d must be a string or a list of strings", index)
			}
		}
	}

	name, ok := arguments[0].(string)
	if !ok {
		return info.Error("lookup_file: first argument must be a string")
	}

	if name == "" {
		return info.Error("lookup_file: first argument is empty string")
	}

	result := []yaml.Node{}

	for _, d := range paths {
		if d != "" {
			p := d + "/" + name
			s, err := os.Stat(p)
			if os.IsNotExist(err) || err != nil {
				continue
			}
			if s.IsDir() == directory {
				result = append(result, node(p, binding))
			}
		}
	}
	return result, info, true
}
<commit_msg>lookup now checks for absolute names<commit_after>package dynaml

import (
	"github.com/mandelsoft/spiff/yaml"
	"os"
	"path/filepath"
)

func func_lookup(directory bool, arguments []interface{}, binding Binding) (interface{}, EvaluationInfo, bool) {
	info := DefaultInfo()

	paths := []string{}

	switch len(arguments) {
	case 0, 1:
		return info.Error("lookup_file requires at least two arguments")
	default:
		for index, arg := range arguments[1:] {
			switch v := arg.(type) {
			case []yaml.Node:
				for _, p := range v {
					if p.Value() == nil {
						continue
					}
					switch v := p.Value().(type) {
					case string:
						paths = append(paths, v)
					default:
						return info.Error("lookup_file: argument %d must be a list of strings", index)
					}
				}
			case string:
				paths = append(paths, v)
			default:
				return info.Error("lookup_file: argument %d must be a string or a list of strings", index)
			}
		}
	}

	name, ok := arguments[0].(string)
	if !ok {
		return info.Error("lookup_file: first argument must be a string")
	}

	if name == "" {
		return info.Error("lookup_file: first argument is empty string")
	}

	result := []yaml.Node{}
	if filepath.IsAbs(name) {
		if checkExistence(name, directory) {
			result = append(result, node(name, binding))
		}
		return result, info, true
	}

	for _, d := range paths {
		if d != "" {
			p := d + "/" + name
			if checkExistence(p, directory) {
				result = append(result, node(p, binding))
			}
		}
	}
	return result, info, true
}

func checkExistence(path string, directory bool) bool {
	s, err := os.Stat(path)
	if os.IsNotExist(err) || err != nil {
		return false
	}
	return s.IsDir() == directory
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"strings"
	"time"

	"github.com/ghodss/yaml"
	"github.com/labstack/echo"
	"github.com/nats-io/nats"
	"github.com/nu7hatch/gouuid"
)

// ServiceInput : service received by the endpoint
type ServiceInput struct {
	Datacenter string `json:"datacenter"`
	Name       string `json:"name"`
}

type ServicePayload struct {
	ID         string           `json:"id"`
	PrevID     string           `json:"previous_id"`
	Datacenter *json.RawMessage `json:"datacenter"`
	Group      *json.RawMessage `json:"client"`
	Service    *json.RawMessage `json:"service"`
}

// Maps input as a valid Serviceinput
func mapInputService(c echo.Context) (s ServiceInput, definition []byte, jsonbody []byte, err error) {
	req := c.Request()
	definition, err = ioutil.ReadAll(req.Body())

	// Normalize input body to json
	ctype := req.Header().Get("Content-Type")

	if ctype != "application/json" && ctype != "application/yaml" {
		return s, definition, jsonbody, errors.New(`"Invalid input format"`)
	}

	if ctype == "application/yaml" {
		jsonbody, err = yaml.YAMLToJSON(definition)
		if err != nil {
			return s, definition, jsonbody, errors.New(`"Invalid yaml input"`)
		}
	} else {
		jsonbody = definition
	}

	if err = json.Unmarshal(jsonbody, &s); err != nil {
		return s, definition, jsonbody, errors.New(`"Invalid input"`)
	}

	return s, definition, jsonbody, nil
}

// Generates a service ID based on an input service
func generateServiceID(salt string) string {
	sufix := generateStreamID(salt)
	prefix, _ := uuid.NewV4()

	return prefix.String() + "-" + string(sufix[:])
}

func generateStreamID(salt string) string {
	compose := []byte(salt)
	hasher := md5.New()
	hasher.Write(compose)
	return hex.EncodeToString(hasher.Sum(nil))
}

func getDatacenter(name string, group int) (datacenter []byte, err error) {
	var msg *nats.Msg

	query := fmt.Sprintf(`{"name": "%s", "group_id": %d}`, name, group)
	if msg, err = n.Request("datacenter.find", []byte(query), 1*time.Second); err != nil {
		return datacenter, ErrGatewayTimeout
	}
	if string(msg.Data) == `[]` {
		return datacenter, errors.New(`"Specified datacenter does not exist"`)
	}

	// Get only the first datcenter
	datacenters := make([]interface{}, 0)
	json.Unmarshal(msg.Data, &datacenters)
	res, err := json.Marshal(datacenters[0])
	if err != nil {
		return datacenter, errors.New("Internal error trying to get the datacenter")
	}

	return res, nil
}

func getDatacenterByID(id int) (user []byte, err error) {
	var msg *nats.Msg

	query := fmt.Sprintf(`{"id": %d}`, id)
	if msg, err = n.Request("datacenter.get", []byte(query), 1*time.Second); err != nil {
		return user, ErrGatewayTimeout
	}
	if strings.Contains(string(msg.Data), `"error"`) {
		return user, errors.New(`"Specified datacenter does not exist"`)
	}
	return msg.Data, nil
}

func getUser(id int) (user []byte, err error) {
	var msg *nats.Msg

	query := fmt.Sprintf(`{"id": %d}`, id)
	if msg, err = n.Request("user.get", []byte(query), 1*time.Second); err != nil {
		return user, ErrGatewayTimeout
	}
	if strings.Contains(string(msg.Data), `"error"`) {
		return user, errors.New(`"Specified user does not exist"`)
	}
	return msg.Data, nil
}

func getGroup(id int) (group []byte, err error) {
	var msg *nats.Msg

	query := fmt.Sprintf(`{"id": %d}`, id)
	if msg, err = n.Request("group.get", []byte(query), 1*time.Second); err != nil {
		return group, ErrGatewayTimeout
	}
	if strings.Contains(string(msg.Data), `"error"`) {
		return group, errors.New(`"Specified group does not exist"`)
	}
	return msg.Data, nil
}

func getService(name string, group int) (service *Service, err error) {
	var msg *nats.Msg

	query := fmt.Sprintf(`{"name":"%s","group_id":%d}`, name, group)
	if msg, err = n.Request("service.find", []byte(query), 1*time.Second); err != nil {
		return service, ErrGatewayTimeout
	}

	p := []Service{}
	json.Unmarshal(msg.Data, &p)
	if len(p) == 0 {
		return nil, nil
	}

	return &p[0], nil
}

func mapCreateDefinition(payload ServicePayload) (body []byte, err error) {
	var msg *nats.Msg

	if body, err = json.Marshal(payload); err != nil {
		return body, errors.New("Provided yaml is not valid")
	}

	if msg, err = n.Request("definition.map.creation", body, 1*time.Second); err != nil {
		return body, errors.New("Provided yaml is not valid")
	}

	var s struct {
		Error string `json:"error"`
	}

	json.Unmarshal(msg.Data, &s)
	if s.Error != "" {
		return body, errors.New(s.Error)
	}

	return msg.Data, nil
}

func getServiceRaw(name string, group int) (service []byte, err error) {
	var msg *nats.Msg

	query := fmt.Sprintf(`{"name":"%s","group_id":%d}`, name, group)
	if msg, err = n.Request("service.find", []byte(query), 1*time.Second); err != nil {
		return service, ErrGatewayTimeout
	}
	p := []*json.RawMessage{}

	if err = json.Unmarshal(msg.Data, &p); err != nil {
		return nil, errors.New(`"Internal error"`)
	}

	if len(p) == 0 {
		return nil, errors.New(`"Service not found"`)
	}

	if body, err := p[0].MarshalJSON(); err != nil {
		return nil, errors.New("Internal error")
	} else {
		return body, nil
	}
}

type OutputService struct {
	ID           string `json:"id"`
	DatacenterID int    `json:"datacenter_id"`
	Name         string `json:"name"`
	Version      string `json:"version"`
	Status       string `json:"status"`
	Options      string `json:"options"`
	Endpoint     string `json:"endpoint"`
	Definition   string `json:"definition"`
	Networks     []struct {
		Name   string `json:"name"`
		Subnet string `json:"network_aws_id"`
	} `json:"networks"`
	Instances []struct {
		Name          string `json:"name"`
		InstanceAWSID string `json:"instance_aws_id"`
		PublicIP      string `json:"public_ip"`
		IP            string `json:"ip"`
	} `json:"instances"`
	SecurityGroups []struct {
		Name            string `json:"name"`
		NatGatewayAWSID string `json:"nat_gateway_aws_id"`
	} `json:"nats"`
}

type ServiceMapping struct {
	Networks struct {
		Items []struct {
			Name   string `json:"name"`
			Subnet string `json:"network_aws_id"`
		} `json:"items"`
	} `json:"networks"`
	Instances struct {
		Items []struct {
			Name          string `json:"name"`
			InstanceAWSID string `json:"instance_aws_id"`
			PublicIP      string `json:"public_ip"`
			IP            string `json:"ip"`
		} `json:"items"`
	} `json:"instances"`
	SecurityGroups struct {
		Items []struct {
			Name            string `json:"name"`
			NatGatewayAWSID string `json:"nat_gateway_aws_id"`
		} `json:"items"`
	} `json:"nats"`
}

func getServicesOutput(filter map[string]interface{}) (list []OutputService, err error) {
	var msg *nats.Msg

	query, err := json.Marshal(filter)
	if err != nil {
		return list, err
	}

	if msg, err = n.Request("service.find", query, 1*time.Second); err != nil {
		return list, ErrGatewayTimeout
	}

	if err := json.Unmarshal(msg.Data, &list); err != nil {
		return list, errors.New("Internal error")
	}

	// Popolate service with detailedd info
	for i, v := range list {
		mapping := ServiceMapping{}
		if msg, err = n.Request("service.get.mapping", []byte(`{"id":"`+v.ID+`"}`), 1*time.Second); err != nil {
			return list, ErrGatewayTimeout
		}

		if err := json.Unmarshal(msg.Data, &mapping); err != nil {
			return list, errors.New("Internal error")
		}

		list[i].Networks = mapping.Networks.Items
		list[i].SecurityGroups = mapping.SecurityGroups.Items
		list[i].Instances = mapping.Instances.Items
	}

	return list, nil
}

func resetService(au User, name string) (status int, err error) {
	var list []OutputService
	filter := make(map[string]interface{})
	filter["group_id"] = au.GroupID
	filter["name"] = name

	if list, err = getServicesOutput(filter); err != nil {
		return 500, errors.New("Internal error")
	}
	if len(list) == 0 {
		return 404, errors.New(`No services found with for '` + name + `'`)
	}
	if list[0].Status != "in_progress" {
		return 200, errors.New("Reset only applies to 'in progress' serices, however service '" + name + "' is on status '" + list[0].Status)
	}

	query := `{"id":"` + list[0].ID + `","status":"errored"}`
	if _, err := n.Request("service.set", []byte(query), 1*time.Second); err != nil {
		return 500, errors.New("Could not update the service")
	}

	return 200, nil
}
func saveService(id string, name string, t string, v time.Time, s string, o string, d string, m string, group uint, datacenter uint) {
	var payload struct {
		Uuid         string    `json:"id"`
		GroupID      uint      `json:"group_id"`
		DatacenterID uint      `json:"datacenter_id"`
		Name         string    `json:"name"`
		Type         string    `json:"type"`
		Version      time.Time `json:"version"`
		Status       string    `json:"status"`
		Options      string    `json:"options"`
		Definition   string    `json:"definition"`
		Mapping      string    `json:"mapping"`
	}

	payload.Uuid = id
	payload.Name = name
	payload.Type = t
	payload.GroupID = group
	payload.DatacenterID = datacenter
	payload.Version = v
	payload.Status = s
	payload.Options = o
	payload.Definition = d
	payload.Mapping = m

	body, _ := json.Marshal(payload)

	n.Publish("service.set", body)
}
<commit_msg>Add service groups to service info<commit_after>package main

import (
	"crypto/md5"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"strings"
	"time"

	"github.com/ghodss/yaml"
	"github.com/labstack/echo"
	"github.com/nats-io/nats"
	"github.com/nu7hatch/gouuid"
)

// ServiceInput : service received by the endpoint
type ServiceInput struct {
	Datacenter string `json:"datacenter"`
	Name       string `json:"name"`
}

type ServicePayload struct {
	ID         string           `json:"id"`
	PrevID     string           `json:"previous_id"`
	Datacenter *json.RawMessage `json:"datacenter"`
	Group      *json.RawMessage `json:"client"`
	Service    *json.RawMessage `json:"service"`
}

// Maps input as a valid Serviceinput
func mapInputService(c echo.Context) (s ServiceInput, definition []byte, jsonbody []byte, err error) {
	req := c.Request()
	definition, err = ioutil.ReadAll(req.Body())

	// Normalize input body to json
	ctype := req.Header().Get("Content-Type")

	if ctype != "application/json" && ctype != "application/yaml" {
		return s, definition, jsonbody, errors.New(`"Invalid input format"`)
	}

	if ctype == "application/yaml" {
		jsonbody, err = yaml.YAMLToJSON(definition)
		if err != nil {
			return s, definition, jsonbody, errors.New(`"Invalid yaml input"`)
		}
	} else {
		jsonbody = definition
	}

	if err = json.Unmarshal(jsonbody, &s); err != nil {
		return s, definition, jsonbody, errors.New(`"Invalid input"`)
	}

	return s, definition, jsonbody, nil
}

// Generates a service ID based on an input service
func generateServiceID(salt string) string {
	sufix := generateStreamID(salt)
	prefix, _ := uuid.NewV4()

	return prefix.String() + "-" + string(sufix[:])
}

func generateStreamID(salt string) string {
	compose := []byte(salt)
	hasher := md5.New()
	hasher.Write(compose)
	return hex.EncodeToString(hasher.Sum(nil))
}

func getDatacenter(name string, group int) (datacenter []byte, err error) {
	var msg *nats.Msg

	query := fmt.Sprintf(`{"name": "%s", "group_id": %d}`, name, group)
	if msg, err = n.Request("datacenter.find", []byte(query), 1*time.Second); err != nil {
		return datacenter, ErrGatewayTimeout
	}
	if string(msg.Data) == `[]` {
		return datacenter, errors.New(`"Specified datacenter does not exist"`)
	}

	// Get only the first datcenter
	datacenters := make([]interface{}, 0)
	json.Unmarshal(msg.Data, &datacenters)
	res, err := json.Marshal(datacenters[0])
	if err != nil {
		return datacenter, errors.New("Internal error trying to get the datacenter")
	}

	return res, nil
}

func getDatacenterByID(id int) (user []byte, err error) {
	var msg *nats.Msg

	query := fmt.Sprintf(`{"id": %d}`, id)
	if msg, err = n.Request("datacenter.get", []byte(query), 1*time.Second); err != nil {
		return user, ErrGatewayTimeout
	}
	if strings.Contains(string(msg.Data), `"error"`) {
		return user, errors.New(`"Specified datacenter does not exist"`)
	}
	return msg.Data, nil
}

func getUser(id int) (user []byte, err error) {
	var msg *nats.Msg

	query := fmt.Sprintf(`{"id": %d}`, id)
	if msg, err = n.Request("user.get", []byte(query), 1*time.Second); err != nil {
		return user, ErrGatewayTimeout
	}
	if strings.Contains(string(msg.Data), `"error"`) {
		return user, errors.New(`"Specified user does not exist"`)
	}
	return msg.Data, nil
}

func getGroup(id int) (group []byte, err error) {
	var msg *nats.Msg

	query := fmt.Sprintf(`{"id": %d}`, id)
	if msg, err = n.Request("group.get", []byte(query), 1*time.Second); err != nil {
		return group, ErrGatewayTimeout
	}
	if strings.Contains(string(msg.Data), `"error"`) {
		return group, errors.New(`"Specified group does not exist"`)
	}
	return msg.Data, nil
}

func getService(name string, group int) (service *Service, err error) {
	var msg *nats.Msg

	query := fmt.Sprintf(`{"name":"%s","group_id":%d}`, name, group)
	if msg, err = n.Request("service.find", []byte(query), 1*time.Second); err != nil {
		return service, ErrGatewayTimeout
	}

	p := []Service{}
	json.Unmarshal(msg.Data, &p)
	if len(p) == 0 {
		return nil, nil
	}

	return &p[0], nil
}

func mapCreateDefinition(payload ServicePayload) (body []byte, err error) {
	var msg *nats.Msg

	if body, err = json.Marshal(payload); err != nil {
		return body, errors.New("Provided yaml is not valid")
	}

	if msg, err = n.Request("definition.map.creation", body, 1*time.Second); err != nil {
		return body, errors.New("Provided yaml is not valid")
	}

	var s struct {
		Error string `json:"error"`
	}

	json.Unmarshal(msg.Data, &s)
	if s.Error != "" {
		return body, errors.New(s.Error)
	}

	return msg.Data, nil
}

func getServiceRaw(name string, group int) (service []byte, err error) {
	var msg *nats.Msg

	query := fmt.Sprintf(`{"name":"%s","group_id":%d}`, name, group)
	if msg, err = n.Request("service.find", []byte(query), 1*time.Second); err != nil {
		return service, ErrGatewayTimeout
	}
	p := []*json.RawMessage{}

	if err = json.Unmarshal(msg.Data, &p); err != nil {
		return nil, errors.New(`"Internal error"`)
	}

	if len(p) == 0 {
		return nil, errors.New(`"Service not found"`)
	}

	if body, err := p[0].MarshalJSON(); err != nil {
		return nil, errors.New("Internal error")
	} else {
		return body, nil
	}
}

type OutputService struct {
	ID           string `json:"id"`
	DatacenterID int    `json:"datacenter_id"`
	Name         string `json:"name"`
	Version      string `json:"version"`
	Status       string `json:"status"`
	Options      string `json:"options"`
	Endpoint     string `json:"endpoint"`
	Definition   string `json:"definition"`
	Networks     []struct {
		Name   string `json:"name"`
		Subnet string `json:"network_aws_id"`
	} `json:"networks"`
	Instances []struct {
		Name          string `json:"name"`
		InstanceAWSID string `json:"instance_aws_id"`
		PublicIP      string `json:"public_ip"`
		IP            string `json:"ip"`
	} `json:"instances"`
	Nats []struct {
		Name            string `json:"name"`
		NatGatewayAWSID string `json:"nat_gateway_aws_id"`
	} `json:"nats"`
	SecurityGroups []struct {
		Name               string `json:"name"`
		SecurityGroupAWSID string `json:"security_group_aws_id"`
	} `json:"security_groups"`
}

type ServiceMapping struct {
	Networks struct {
		Items []struct {
			Name   string `json:"name"`
			Subnet string `json:"network_aws_id"`
		} `json:"items"`
	} `json:"networks"`
	Instances struct {
		Items []struct {
			Name          string `json:"name"`
			InstanceAWSID string `json:"instance_aws_id"`
			PublicIP      string `json:"public_ip"`
			IP            string `json:"ip"`
		} `json:"items"`
	} `json:"instances"`
	Nats struct {
		Items []struct {
			Name            string `json:"name"`
			NatGatewayAWSID string `json:"nat_gateway_aws_id"`
		} `json:"items"`
	} `json:"nats"`
	SecurityGroups struct {
		Items []struct {
			Name               string `json:"name"`
			SecurityGroupAWSID string `json:"security_group_aws_id"`
		} `json:"items"`
	} `json:"firewalls"`
}

func getServicesOutput(filter map[string]interface{}) (list []OutputService, err error) {
	var msg *nats.Msg

	query, err := json.Marshal(filter)
	if err != nil {
		return list, err
	}

	if msg, err = n.Request("service.find", query, 1*time.Second); err != nil {
		return list, ErrGatewayTimeout
	}

	if err := json.Unmarshal(msg.Data, &list); err != nil {
		return list, errors.New("Internal error")
	}

	// Popolate service with detailedd info
	for i, v := range list {
		mapping := ServiceMapping{}
		if msg, err = n.Request("service.get.mapping", []byte(`{"id":"`+v.ID+`"}`), 1*time.Second); err != nil {
			return list, ErrGatewayTimeout
		}

		if err := json.Unmarshal(msg.Data, &mapping); err != nil {
			return list, errors.New("Internal error")
		}

		list[i].Networks = mapping.Networks.Items
		list[i].SecurityGroups = mapping.SecurityGroups.Items
		list[i].Nats = mapping.Nats.Items
		list[i].Instances = mapping.Instances.Items
	}

	return list, nil
}

func resetService(au User, name string) (status int, err error) {
	var list []OutputService
	filter := make(map[string]interface{})
	filter["group_id"] = au.GroupID
	filter["name"] = name

	if list, err = getServicesOutput(filter); err != nil {
		return 500, errors.New("Internal error")
	}
	if len(list) == 0 {
		return 404, errors.New(`No services found with for '` + name + `'`)
	}
	if list[0].Status != "in_progress" {
		return 200, errors.New("Reset only applies to 'in progress' serices, however service '" + name + "' is on status '" + list[0].Status)
	}

	query := `{"id":"` + list[0].ID + `","status":"errored"}`
	if _, err := n.Request("service.set", []byte(query), 1*time.Second); err != nil {
		return 500, errors.New("Could not update the service")
	}

	return 200, nil
}
func saveService(id string, name string, t string, v time.Time, s string, o string, d string, m string, group uint, datacenter uint) {
	var payload struct {
		Uuid         string    `json:"id"`
		GroupID      uint      `json:"group_id"`
		DatacenterID uint      `json:"datacenter_id"`
		Name         string    `json:"name"`
		Type         string    `json:"type"`
		Version      time.Time `json:"version"`
		Status       string    `json:"status"`
		Options      string    `json:"options"`
		Definition   string    `json:"definition"`
		Mapping      string    `json:"mapping"`
	}

	payload.Uuid = id
	payload.Name = name
	payload.Type = t
	payload.GroupID = group
	payload.DatacenterID = datacenter
	payload.Version = v
	payload.Status = s
	payload.Options = o
	payload.Definition = d
	payload.Mapping = m

	body, _ := json.Marshal(payload)

	n.Publish("service.set", body)
}
<|endoftext|>


<commit_before>// Copyright 2015 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package csource

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
	"unsafe"

	"github.com/google/syzkaller/prog"
	"github.com/google/syzkaller/sys"
)

type Options struct {
	Threaded bool
	Collide  bool
}

func Write(p *prog.Prog, opts Options) []byte {
	exec := p.SerializeForExec()
	w := new(bytes.Buffer)

	fmt.Fprintf(w, `// autogenerated by syzkaller (http://github.com/google/syzkaller)
#include <unistd.h>
#include <sys/syscall.h>
#include <string.h>
#include <stdint.h>
#include <pthread.h>

`)

	handled := make(map[string]bool)
	for _, c := range p.Calls {
		name := c.Meta.CallName
		if handled[name] {
			continue
		}
		handled[name] = true
		fmt.Fprintf(w, "#ifndef SYS_%v\n", name)
		fmt.Fprintf(w, "#define SYS_%v %v\n", name, c.Meta.NR)
		fmt.Fprintf(w, "#endif\n")
	}
	fmt.Fprintf(w, "\n")

	calls, nvar := generateCalls(exec)
	fmt.Fprintf(w, "long r[%v];\n\n", nvar)

	if !opts.Threaded && !opts.Collide {
		fmt.Fprintf(w, "int main()\n{\n")
		fmt.Fprintf(w, "\tmemset(r, -1, sizeof(r));\n")
		for _, c := range calls {
			fmt.Fprintf(w, "%s", c)
		}
		fmt.Fprintf(w, "\treturn 0;\n}\n")
	} else {
		fmt.Fprintf(w, "void *thr(void *arg)\n{\n")
		fmt.Fprintf(w, "\tswitch ((long)arg) {\n")
		for i, c := range calls {
			fmt.Fprintf(w, "\tcase %v:\n", i)
			fmt.Fprintf(w, "%s", strings.Replace(c, "\t", "\t\t", -1))
			fmt.Fprintf(w, "\t\tbreak;\n")
		}
		fmt.Fprintf(w, "\t}\n")
		fmt.Fprintf(w, "\treturn 0;\n}\n\n")

		fmt.Fprintf(w, "int main()\n{\n")
		fmt.Fprintf(w, "\tlong i;\n")
		fmt.Fprintf(w, "\tpthread_t th[%v];\n", len(calls))
		fmt.Fprintf(w, "\n")
		fmt.Fprintf(w, "\tmemset(r, -1, sizeof(r));\n")
		fmt.Fprintf(w, "\tfor (i = 0; i < %v; i++) {\n", len(calls))
		fmt.Fprintf(w, "\t\tpthread_create(&th[i], 0, thr, (void*)i);\n")
		fmt.Fprintf(w, "\t\tusleep(10000);\n")
		fmt.Fprintf(w, "\t}\n")
		if opts.Collide {
			fmt.Fprintf(w, "\tfor (i = 0; i < %v; i++) {\n", len(calls))
			fmt.Fprintf(w, "\t\tpthread_create(&th[i], 0, thr, (void*)i);\n")
			fmt.Fprintf(w, "\t\tif (i%%2==0)\n")
			fmt.Fprintf(w, "\t\t\tusleep(10000);\n")
			fmt.Fprintf(w, "\t}\n")
		}
		fmt.Fprintf(w, "\tusleep(100000);\n")
		fmt.Fprintf(w, "\treturn 0;\n}\n")
	}
	return w.Bytes()
}

func generateCalls(exec []byte) ([]string, int) {
	read := func() uintptr {
		if len(exec) < 8 {
			panic("exec program overflow")
		}
		v := *(*uint64)(unsafe.Pointer(&exec[0]))
		exec = exec[8:]
		return uintptr(v)
	}
	resultRef := func() string {
		arg := read()
		res := fmt.Sprintf("r[%v]", arg)
		if opDiv := read(); opDiv != 0 {
			res = fmt.Sprintf("%v/%v", res, opDiv)
		}
		if opAdd := read(); opAdd != 0 {
			res = fmt.Sprintf("%v+%v", res, opAdd)
		}
		return res
	}
	lastCall := 0
	seenCall := false
	var calls []string
	w := new(bytes.Buffer)
	newCall := func() {
		if seenCall {
			seenCall = false
			calls = append(calls, w.String())
			w = new(bytes.Buffer)
		}
	}
	n := 0
loop:
	for ; ; n++ {
		switch instr := read(); instr {
		case prog.ExecInstrEOF:
			break loop
		case prog.ExecInstrCopyin:
			newCall()
			addr := read()
			typ := read()
			size := read()
			switch typ {
			case prog.ExecArgConst:
				arg := read()
				fmt.Fprintf(w, "\t*(uint%v_t*)0x%x = (uint%v_t)0x%x;\n", size*8, addr, size*8, arg)
			case prog.ExecArgResult:
				fmt.Fprintf(w, "\t*(uint%v_t*)0x%x = %v;\n", size*8, addr, resultRef())
			case prog.ExecArgData:
				data := exec[:size]
				exec = exec[(size+7)/8*8:]
				var esc []byte
				for _, v := range data {
					hex := func(v byte) byte {
						if v < 10 {
							return '0' + v
						}
						return 'a' + v - 10
					}
					esc = append(esc, '\\', 'x', hex(v>>4), hex(v<<4>>4))
				}
				fmt.Fprintf(w, "\tmemcpy((void*)0x%x, \"%s\", %v);\n", addr, esc, size)
			default:
				panic("bad argument type")
			}
		case prog.ExecInstrCopyout:
			addr := read()
			size := read()
			fmt.Fprintf(w, "\tif (r[%v] != -1)\n", lastCall)
			fmt.Fprintf(w, "\t\tr[%v] = *(uint%v_t*)0x%x;\n", n, size*8, addr)
		default:
			// Normal syscall.
			newCall()
			meta := sys.Calls[instr]
			fmt.Fprintf(w, "\tr[%v] = syscall(SYS_%v", n, meta.CallName)
			nargs := read()
			for i := uintptr(0); i < nargs; i++ {
				typ := read()
				size := read()
				_ = size
				switch typ {
				case prog.ExecArgConst:
					fmt.Fprintf(w, ", 0x%xul", read())
				case prog.ExecArgResult:
					fmt.Fprintf(w, ", %v", resultRef())
				default:
					panic("unknown arg type")
				}
			}
			for i := nargs; i < 6; i++ {
				fmt.Fprintf(w, ", 0")
			}
			fmt.Fprintf(w, ");\n")
			lastCall = n
			seenCall = true
		}
	}
	newCall()
	return calls, n
}

// Build builds a C/C++ program from source file src
// and returns name of the resulting binary.
func Build(src string) (string, error) {
	bin, err := ioutil.TempFile("", "syzkaller")
	if err != nil {
		return "", fmt.Errorf("failed to create temp file: %v", err)
	}
	bin.Close()
	out, err := exec.Command("gcc", "-x", "c++", "-std=gnu++11", src, "-o", bin.Name(), "-lpthread", "-static", "-O1", "-g").CombinedOutput()
	if err != nil {
		os.Remove(bin.Name())
		data, _ := ioutil.ReadFile(src)
		return "", fmt.Errorf("failed to build program::\n%s\n%s", data, out)
	}
	return bin.Name(), nil
}

// Format reformats C source using clang-format.
func Format(src []byte) ([]byte, error) {
	stdout, stderr := new(bytes.Buffer), new(bytes.Buffer)
	cmd := exec.Command("clang-format", "-assume-filename=/src.c", "-style", style)
	cmd.Stdin = bytes.NewReader(src)
	cmd.Stdout = stdout
	cmd.Stderr = stderr
	if err := cmd.Run(); err != nil {
		return nil, fmt.Errorf("failed to format source: %v\n%v", err, stderr.String())
	}
	return stdout.Bytes(), nil
}

// Something acceptable for kernel developers and email-friendly.
var style = `{
BasedOnStyle: LLVM,
IndentWidth: 2,
UseTab: Never,
BreakBeforeBraces: Linux,
IndentCaseLabels: false,
DerivePointerAlignment: false,
PointerAlignment: Left,
AlignTrailingComments: true,
AllowShortBlocksOnASingleLine: false,
AllowShortCaseLabelsOnASingleLine: false,
AllowShortFunctionsOnASingleLine: false,
AllowShortIfStatementsOnASingleLine: false,
AllowShortLoopsOnASingleLine: false,
ColumnLimit: 72,
}`
<commit_msg>csource: use dynamic libraries if static are not supported<commit_after>// Copyright 2015 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package csource

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
	"unsafe"

	"github.com/google/syzkaller/prog"
	"github.com/google/syzkaller/sys"
)

type Options struct {
	Threaded bool
	Collide  bool
}

func Write(p *prog.Prog, opts Options) []byte {
	exec := p.SerializeForExec()
	w := new(bytes.Buffer)

	fmt.Fprintf(w, `// autogenerated by syzkaller (http://github.com/google/syzkaller)
#include <unistd.h>
#include <sys/syscall.h>
#include <string.h>
#include <stdint.h>
#include <pthread.h>

`)

	handled := make(map[string]bool)
	for _, c := range p.Calls {
		name := c.Meta.CallName
		if handled[name] {
			continue
		}
		handled[name] = true
		fmt.Fprintf(w, "#ifndef SYS_%v\n", name)
		fmt.Fprintf(w, "#define SYS_%v %v\n", name, c.Meta.NR)
		fmt.Fprintf(w, "#endif\n")
	}
	fmt.Fprintf(w, "\n")

	calls, nvar := generateCalls(exec)
	fmt.Fprintf(w, "long r[%v];\n\n", nvar)

	if !opts.Threaded && !opts.Collide {
		fmt.Fprintf(w, "int main()\n{\n")
		fmt.Fprintf(w, "\tmemset(r, -1, sizeof(r));\n")
		for _, c := range calls {
			fmt.Fprintf(w, "%s", c)
		}
		fmt.Fprintf(w, "\treturn 0;\n}\n")
	} else {
		fmt.Fprintf(w, "void *thr(void *arg)\n{\n")
		fmt.Fprintf(w, "\tswitch ((long)arg) {\n")
		for i, c := range calls {
			fmt.Fprintf(w, "\tcase %v:\n", i)
			fmt.Fprintf(w, "%s", strings.Replace(c, "\t", "\t\t", -1))
			fmt.Fprintf(w, "\t\tbreak;\n")
		}
		fmt.Fprintf(w, "\t}\n")
		fmt.Fprintf(w, "\treturn 0;\n}\n\n")

		fmt.Fprintf(w, "int main()\n{\n")
		fmt.Fprintf(w, "\tlong i;\n")
		fmt.Fprintf(w, "\tpthread_t th[%v];\n", len(calls))
		fmt.Fprintf(w, "\n")
		fmt.Fprintf(w, "\tmemset(r, -1, sizeof(r));\n")
		fmt.Fprintf(w, "\tfor (i = 0; i < %v; i++) {\n", len(calls))
		fmt.Fprintf(w, "\t\tpthread_create(&th[i], 0, thr, (void*)i);\n")
		fmt.Fprintf(w, "\t\tusleep(10000);\n")
		fmt.Fprintf(w, "\t}\n")
		if opts.Collide {
			fmt.Fprintf(w, "\tfor (i = 0; i < %v; i++) {\n", len(calls))
			fmt.Fprintf(w, "\t\tpthread_create(&th[i], 0, thr, (void*)i);\n")
			fmt.Fprintf(w, "\t\tif (i%%2==0)\n")
			fmt.Fprintf(w, "\t\t\tusleep(10000);\n")
			fmt.Fprintf(w, "\t}\n")
		}
		fmt.Fprintf(w, "\tusleep(100000);\n")
		fmt.Fprintf(w, "\treturn 0;\n}\n")
	}
	return w.Bytes()
}

func generateCalls(exec []byte) ([]string, int) {
	read := func() uintptr {
		if len(exec) < 8 {
			panic("exec program overflow")
		}
		v := *(*uint64)(unsafe.Pointer(&exec[0]))
		exec = exec[8:]
		return uintptr(v)
	}
	resultRef := func() string {
		arg := read()
		res := fmt.Sprintf("r[%v]", arg)
		if opDiv := read(); opDiv != 0 {
			res = fmt.Sprintf("%v/%v", res, opDiv)
		}
		if opAdd := read(); opAdd != 0 {
			res = fmt.Sprintf("%v+%v", res, opAdd)
		}
		return res
	}
	lastCall := 0
	seenCall := false
	var calls []string
	w := new(bytes.Buffer)
	newCall := func() {
		if seenCall {
			seenCall = false
			calls = append(calls, w.String())
			w = new(bytes.Buffer)
		}
	}
	n := 0
loop:
	for ; ; n++ {
		switch instr := read(); instr {
		case prog.ExecInstrEOF:
			break loop
		case prog.ExecInstrCopyin:
			newCall()
			addr := read()
			typ := read()
			size := read()
			switch typ {
			case prog.ExecArgConst:
				arg := read()
				fmt.Fprintf(w, "\t*(uint%v_t*)0x%x = (uint%v_t)0x%x;\n", size*8, addr, size*8, arg)
			case prog.ExecArgResult:
				fmt.Fprintf(w, "\t*(uint%v_t*)0x%x = %v;\n", size*8, addr, resultRef())
			case prog.ExecArgData:
				data := exec[:size]
				exec = exec[(size+7)/8*8:]
				var esc []byte
				for _, v := range data {
					hex := func(v byte) byte {
						if v < 10 {
							return '0' + v
						}
						return 'a' + v - 10
					}
					esc = append(esc, '\\', 'x', hex(v>>4), hex(v<<4>>4))
				}
				fmt.Fprintf(w, "\tmemcpy((void*)0x%x, \"%s\", %v);\n", addr, esc, size)
			default:
				panic("bad argument type")
			}
		case prog.ExecInstrCopyout:
			addr := read()
			size := read()
			fmt.Fprintf(w, "\tif (r[%v] != -1)\n", lastCall)
			fmt.Fprintf(w, "\t\tr[%v] = *(uint%v_t*)0x%x;\n", n, size*8, addr)
		default:
			// Normal syscall.
			newCall()
			meta := sys.Calls[instr]
			fmt.Fprintf(w, "\tr[%v] = syscall(SYS_%v", n, meta.CallName)
			nargs := read()
			for i := uintptr(0); i < nargs; i++ {
				typ := read()
				size := read()
				_ = size
				switch typ {
				case prog.ExecArgConst:
					fmt.Fprintf(w, ", 0x%xul", read())
				case prog.ExecArgResult:
					fmt.Fprintf(w, ", %v", resultRef())
				default:
					panic("unknown arg type")
				}
			}
			for i := nargs; i < 6; i++ {
				fmt.Fprintf(w, ", 0")
			}
			fmt.Fprintf(w, ");\n")
			lastCall = n
			seenCall = true
		}
	}
	newCall()
	return calls, n
}

// Build builds a C/C++ program from source file src
// and returns name of the resulting binary.
func Build(src string) (string, error) {
	bin, err := ioutil.TempFile("", "syzkaller")
	if err != nil {
		return "", fmt.Errorf("failed to create temp file: %v", err)
	}
	bin.Close()
	out, err := exec.Command("gcc", "-x", "c++", "-std=gnu++11", src, "-o", bin.Name(), "-pthread", "-static", "-O1", "-g").CombinedOutput()
	if err != nil {
		// Some distributions don't have static libraries.
		out, err = exec.Command("gcc", "-x", "c++", "-std=gnu++11", src, "-o", bin.Name(), "-pthread", "-O1", "-g").CombinedOutput()
	}
	if err != nil {
		os.Remove(bin.Name())
		data, _ := ioutil.ReadFile(src)
		return "", fmt.Errorf("failed to build program::\n%s\n%s", data, out)
	}
	return bin.Name(), nil
}

// Format reformats C source using clang-format.
func Format(src []byte) ([]byte, error) {
	stdout, stderr := new(bytes.Buffer), new(bytes.Buffer)
	cmd := exec.Command("clang-format", "-assume-filename=/src.c", "-style", style)
	cmd.Stdin = bytes.NewReader(src)
	cmd.Stdout = stdout
	cmd.Stderr = stderr
	if err := cmd.Run(); err != nil {
		return nil, fmt.Errorf("failed to format source: %v\n%v", err, stderr.String())
	}
	return stdout.Bytes(), nil
}

// Something acceptable for kernel developers and email-friendly.
var style = `{
BasedOnStyle: LLVM,
IndentWidth: 2,
UseTab: Never,
BreakBeforeBraces: Linux,
IndentCaseLabels: false,
DerivePointerAlignment: false,
PointerAlignment: Left,
AlignTrailingComments: true,
AllowShortBlocksOnASingleLine: false,
AllowShortCaseLabelsOnASingleLine: false,
AllowShortFunctionsOnASingleLine: false,
AllowShortIfStatementsOnASingleLine: false,
AllowShortLoopsOnASingleLine: false,
ColumnLimit: 72,
}`
<|endoftext|>


<commit_before>package cloudstack

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/atsaki/golang-cloudstack-library"

	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourcePortForwardingRule() *schema.Resource {
	return &schema.Resource{
		Create: resourcePortForwardingRuleCreate,
		Read:   resourcePortForwardingRuleRead,
		Delete: resourcePortForwardingRuleDelete,

		Schema: map[string]*schema.Schema{
			"ip_address_id": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"protocol": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"private_port": &schema.Schema{
				Type:     schema.TypeInt,
				Required: true,
				ForceNew: true,
			},
			"public_port": &schema.Schema{
				Type:     schema.TypeInt,
				Required: true,
				ForceNew: true,
			},
			"virtual_machine_id": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"cidr_list": &schema.Schema{
				Type:     schema.TypeSet,
				Optional: true,
				Computed: true,
				ForceNew: true,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
				Set: func(v interface{}) int {
					return hashcode.String(v.(string))
				},
			},
			"private_end_port": &schema.Schema{
				Type:     schema.TypeInt,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},
			"public_end_port": &schema.Schema{
				Type:     schema.TypeInt,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},
			"open_firewall": &schema.Schema{
				Type:     schema.TypeBool,
				Default:  false,
				Optional: true,
			},
		},
	}
}

func resourcePortForwardingRuleCreate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	ipAddressId := d.Get("ip_address_id").(string)
	privatePort := d.Get("private_port").(int)
	protocol := d.Get("protocol").(string)
	publicPort := d.Get("public_port").(int)
	virtualMachineId := d.Get("virtual_machine_id").(string)
	param := cloudstack.NewCreatePortForwardingRuleParameter(
		ipAddressId, privatePort, protocol, publicPort, virtualMachineId)

	cl := d.Get("cidr_list").(*schema.Set)
	if cl.Len() > 0 {
		param.CidrList = make([]string, cl.Len())
		for i, cidr := range cl.List() {
			param.CidrList[i] = cidr.(string)
		}
	}

	if d.Get("public_end_port").(int) != 0 {
		param.PublicEndPort.Set(d.Get("public_end_port"))
	}
	if d.Get("private_end_port").(int) != 0 {
		param.PrivateEndPort.Set(d.Get("private_end_port"))
	}

	param.OpenFirewall.Set(d.Get("open_firewall").(bool))

	pfRule, err := config.client.CreatePortForwardingRule(param)
	if err != nil {
		return fmt.Errorf("Error create portforwarding rule: %s", err)
	}

	d.SetId(pfRule.Id.String())

	return resourcePortForwardingRuleRead(d, meta)
}

func resourcePortForwardingRuleRead(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	param := cloudstack.NewListPortForwardingRulesParameter()
	param.Id.Set(d.Id())
	pfRules, err := config.client.ListPortForwardingRules(param)

	if err != nil {
		param = cloudstack.NewListPortForwardingRulesParameter()
		pfRules, err = config.client.ListPortForwardingRules(param)
		if err != nil {
			return fmt.Errorf("Failed to list portforwarding rule: %s", err)
		}

		fn := func(pf interface{}) bool {
			return pf.(*cloudstack.PortForwardingRule).Id.String() == d.Id()
		}
		pfRules = filter(pfRules, fn).([]*cloudstack.PortForwardingRule)
	}

	if len(pfRules) == 0 {
		d.SetId("")
		return nil
	}

	pfRule := pfRules[0]

	var cidrList []interface{}
	for _, s := range strings.Split(pfRule.CidrList.String(), ",") {
		s = strings.TrimSpace(s)
		if s != "" {
			cidrList = append(cidrList, s)
		}
	}
	d.Set("cidr_list", cidrList)

	if !pfRule.PublicEndPort.IsNil() {
		publicEndPort, err := strconv.Atoi(pfRule.PublicEndPort.String())
		if err != nil {
			return fmt.Errorf("Error convert string to int: %s", err)
		}
		d.Set("public_end_port", publicEndPort)
	}

	if !pfRule.PrivateEndPort.IsNil() {
		privateEndPort, err := strconv.Atoi(pfRule.PrivateEndPort.String())
		if err != nil {
			return fmt.Errorf("Error convert string to int: %s", err)
		}
		d.Set("private_end_port", privateEndPort)
	}

	return nil
}

func resourcePortForwardingRuleDelete(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	if err := resourcePortForwardingRuleRead(d, meta); err != nil {
		return err
	}

	if d.Id() == "" {
		return nil
	}
	param := cloudstack.NewDeletePortForwardingRuleParameter(d.Id())
	_, err := config.client.DeletePortForwardingRule(param)
	if err != nil {
		return fmt.Errorf("Error delete portforwarding rule: %s", err)
	}

	return resourcePortForwardingRuleRead(d, meta)
}
<commit_msg>Add ForceNew to PortForwarding because it has no Update function<commit_after>package cloudstack

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/atsaki/golang-cloudstack-library"

	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourcePortForwardingRule() *schema.Resource {
	return &schema.Resource{
		Create: resourcePortForwardingRuleCreate,
		Read:   resourcePortForwardingRuleRead,
		Delete: resourcePortForwardingRuleDelete,

		Schema: map[string]*schema.Schema{
			"ip_address_id": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"protocol": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"private_port": &schema.Schema{
				Type:     schema.TypeInt,
				Required: true,
				ForceNew: true,
			},
			"public_port": &schema.Schema{
				Type:     schema.TypeInt,
				Required: true,
				ForceNew: true,
			},
			"virtual_machine_id": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"cidr_list": &schema.Schema{
				Type:     schema.TypeSet,
				Optional: true,
				Computed: true,
				ForceNew: true,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
				Set: func(v interface{}) int {
					return hashcode.String(v.(string))
				},
			},
			"private_end_port": &schema.Schema{
				Type:     schema.TypeInt,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},
			"public_end_port": &schema.Schema{
				Type:     schema.TypeInt,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},
			"open_firewall": &schema.Schema{
				Type:     schema.TypeBool,
				Default:  false,
				Optional: true,
				ForceNew: true,
			},
		},
	}
}

func resourcePortForwardingRuleCreate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	ipAddressId := d.Get("ip_address_id").(string)
	privatePort := d.Get("private_port").(int)
	protocol := d.Get("protocol").(string)
	publicPort := d.Get("public_port").(int)
	virtualMachineId := d.Get("virtual_machine_id").(string)
	param := cloudstack.NewCreatePortForwardingRuleParameter(
		ipAddressId, privatePort, protocol, publicPort, virtualMachineId)

	cl := d.Get("cidr_list").(*schema.Set)
	if cl.Len() > 0 {
		param.CidrList = make([]string, cl.Len())
		for i, cidr := range cl.List() {
			param.CidrList[i] = cidr.(string)
		}
	}

	if d.Get("public_end_port").(int) != 0 {
		param.PublicEndPort.Set(d.Get("public_end_port"))
	}
	if d.Get("private_end_port").(int) != 0 {
		param.PrivateEndPort.Set(d.Get("private_end_port"))
	}

	param.OpenFirewall.Set(d.Get("open_firewall").(bool))

	pfRule, err := config.client.CreatePortForwardingRule(param)
	if err != nil {
		return fmt.Errorf("Error create portforwarding rule: %s", err)
	}

	d.SetId(pfRule.Id.String())

	return resourcePortForwardingRuleRead(d, meta)
}

func resourcePortForwardingRuleRead(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	param := cloudstack.NewListPortForwardingRulesParameter()
	param.Id.Set(d.Id())
	pfRules, err := config.client.ListPortForwardingRules(param)

	if err != nil {
		param = cloudstack.NewListPortForwardingRulesParameter()
		pfRules, err = config.client.ListPortForwardingRules(param)
		if err != nil {
			return fmt.Errorf("Failed to list portforwarding rule: %s", err)
		}

		fn := func(pf interface{}) bool {
			return pf.(*cloudstack.PortForwardingRule).Id.String() == d.Id()
		}
		pfRules = filter(pfRules, fn).([]*cloudstack.PortForwardingRule)
	}

	if len(pfRules) == 0 {
		d.SetId("")
		return nil
	}

	pfRule := pfRules[0]

	var cidrList []interface{}
	for _, s := range strings.Split(pfRule.CidrList.String(), ",") {
		s = strings.TrimSpace(s)
		if s != "" {
			cidrList = append(cidrList, s)
		}
	}
	d.Set("cidr_list", cidrList)

	if !pfRule.PublicEndPort.IsNil() {
		publicEndPort, err := strconv.Atoi(pfRule.PublicEndPort.String())
		if err != nil {
			return fmt.Errorf("Error convert string to int: %s", err)
		}
		d.Set("public_end_port", publicEndPort)
	}

	if !pfRule.PrivateEndPort.IsNil() {
		privateEndPort, err := strconv.Atoi(pfRule.PrivateEndPort.String())
		if err != nil {
			return fmt.Errorf("Error convert string to int: %s", err)
		}
		d.Set("private_end_port", privateEndPort)
	}

	return nil
}

func resourcePortForwardingRuleDelete(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	if err := resourcePortForwardingRuleRead(d, meta); err != nil {
		return err
	}

	if d.Id() == "" {
		return nil
	}
	param := cloudstack.NewDeletePortForwardingRuleParameter(d.Id())
	_, err := config.client.DeletePortForwardingRule(param)
	if err != nil {
		return fmt.Errorf("Error delete portforwarding rule: %s", err)
	}

	return resourcePortForwardingRuleRead(d, meta)
}
<|endoftext|>


<commit_before>// Copyright © 2016 Wei Shen <shenwei356@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package cmd

import (
	"bufio"
	"fmt"
	"math"
	"os"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/bsipos/thist"
	"github.com/spf13/cobra"
)

// watchCmd represents the seq command
var watchCmd = &cobra.Command{
	Use:   "watch",
	Short: "sampling by proportion",
	Long: `sampling by proportion

`,
	Run: func(cmd *cobra.Command, args []string) {
		config := getConfigs(cmd)
		files := getFileList(args)
		runtime.GOMAXPROCS(config.NumCPUs)

		printField := getFlagString(cmd, "field")
		printPdf := getFlagString(cmd, "image")
		printFreq := getFlagInt(cmd, "print-freq")
		printDump := getFlagBool(cmd, "dump")
		printLog := getFlagBool(cmd, "log")
		outFile := config.OutFile
		printDelay := getFlagInt(cmd, "delay")
		printReset := getFlagBool(cmd, "reset")
		if printDelay < 0 {
			printDelay = 0
		}
		printBins := getFlagInt(cmd, "bins")
		printPass := getFlagBool(cmd, "pass")

		if printFreq > 0 {
			config.ChunkSize = printFreq
		}

		if config.Tabs {
			config.OutDelimiter = rune('\t')
		}

		outw := os.Stdout
		if outFile != "-" {
			tw, err := os.Create(outFile)
			checkError(err)
			outw = tw
		}
		outfh := bufio.NewWriter(outw)

		defer outfh.Flush()
		defer outw.Close()

		binMode := "termfit"
		if printBins > 0 {
			binMode = "fixed"
		}
		h := thist.NewHist([]float64{}, printField, binMode, printBins, true)

		transform := func(x float64) float64 { return x }
		if printLog {
			transform = func(x float64) float64 {
				return math.Log10(x + 1)
			}
		}

		field2col := make(map[string]int)
		var col int
		if config.NoHeaderRow {
			if len(printField) == 0 {
				fmt.Fprintf(os.Stderr, "No field specified!")
				os.Exit(1)
			}
			pcol, err := strconv.Atoi(printField)
			if err != nil {
				fmt.Fprintf(os.Stderr, "Illegal field number: %s\n", printField)
				os.Exit(1)
			}
			col = pcol - 1
			if col < 0 {
				fmt.Fprintf(os.Stderr, "Illegal field number: %d!\n", pcol)
				os.Exit(1)
			}
		}
		if len(printField) == 0 {
			fmt.Fprintf(os.Stderr, "No field specified!\n")
			os.Exit(1)
		}

		var count int
		for _, file := range files {
			csvReader, err := newCSVReaderByConfig(config, file)
			checkError(err)
			csvReader.Run()

			isHeaderLine := !config.NoHeaderRow
			for chunk := range csvReader.Ch {
				checkError(chunk.Err)

				for _, record := range chunk.Data {
					if isHeaderLine {
						for i, column := range record {
							field2col[column] = i
						}

						isHeaderLine = false
						if printPass {
							outfh.Write([]byte(strings.Join(record, string(config.OutDelimiter)) + "\n"))
						}
						continue
					} // header

					i := col
					if !config.NoHeaderRow {
						var ok bool
						i, ok = field2col[printField]
						if !ok {
							fmt.Fprintf(os.Stderr, "Invalid field specified: %s\n", printField)
							os.Exit(1)
						}
					}
					p, err := strconv.ParseFloat(record[i], 64)
					if err == nil {
						count++
						h.Update(transform(p))
						if printPass {
							outfh.Write([]byte(strings.Join(record, string(config.OutDelimiter)) + "\n"))
						}
					} else {
						continue
					}
					if printFreq > 0 && count%printFreq == 0 {
						if printDump {
							os.Stderr.Write([]byte(h.Dump()))
						} else {
							os.Stderr.Write([]byte(thist.ClearScreenString()))
							os.Stderr.Write([]byte(h.Draw()))
							if printPdf != "" {
								h.SaveImage(printPdf)
							}
						}
						outfh.Flush()
						if printReset {
							h = thist.NewHist([]float64{}, printField, binMode, printBins, true)
						}
						time.Sleep(time.Duration(printDelay) * time.Second)
						os.Stderr.Write([]byte(thist.ClearScreenString()))
					}

				} // record
			} //chunk

		} //file
		if printFreq < 0 || count%printFreq != 0 {
			if printDump {
				os.Stderr.Write([]byte(h.Dump()))
			} else {
				os.Stderr.Write([]byte(thist.ClearScreenString()))
				os.Stderr.Write([]byte(h.Draw()))
			}
			outfh.Flush()
			if printPdf != "" {
				h.SaveImage(printPdf)
			}
		}
	},
}

func init() {
	RootCmd.AddCommand(watchCmd)

	watchCmd.Flags().StringP("field", "f", "", "field to watch")
	watchCmd.Flags().IntP("print-freq", "p", -1, "print frequency (-1 for print after parsing)")
	watchCmd.Flags().StringP("image", "O", "", "print frequency (-1 for print after parsing)")
	watchCmd.Flags().IntP("delay", "w", 1, "sleep this many seconds after online plotting")
	watchCmd.Flags().IntP("bins", "B", -1, "number of histogram bins")
	watchCmd.Flags().BoolP("dump", "y", false, "dump histogram instead of plotting")
	watchCmd.Flags().BoolP("log", "L", false, "log10 transform values")
	watchCmd.Flags().BoolP("reset", "R", false, "reset histogram after every print")
	watchCmd.Flags().BoolP("pass", "x", false, "reset histogram after every print")
}
<commit_msg>Added quiet mode.<commit_after>// Copyright © 2016 Wei Shen <shenwei356@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package cmd

import (
	"bufio"
	"fmt"
	"math"
	"os"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/bsipos/thist"
	"github.com/spf13/cobra"
)

// watchCmd represents the seq command
var watchCmd = &cobra.Command{
	Use:   "watch",
	Short: "sampling by proportion",
	Long: `sampling by proportion

`,
	Run: func(cmd *cobra.Command, args []string) {
		config := getConfigs(cmd)
		files := getFileList(args)
		runtime.GOMAXPROCS(config.NumCPUs)

		printField := getFlagString(cmd, "field")
		printPdf := getFlagString(cmd, "image")
		printFreq := getFlagInt(cmd, "print-freq")
		printDump := getFlagBool(cmd, "dump")
		printLog := getFlagBool(cmd, "log")
		printQuiet := getFlagBool(cmd, "quiet")
		outFile := config.OutFile
		printDelay := getFlagInt(cmd, "delay")
		printReset := getFlagBool(cmd, "reset")
		if printDelay < 0 {
			printDelay = 0
		}
		printBins := getFlagInt(cmd, "bins")
		printPass := getFlagBool(cmd, "pass")

		if printFreq > 0 {
			config.ChunkSize = printFreq
		}

		if config.Tabs {
			config.OutDelimiter = rune('\t')
		}

		outw := os.Stdout
		if outFile != "-" {
			tw, err := os.Create(outFile)
			checkError(err)
			outw = tw
		}
		outfh := bufio.NewWriter(outw)

		defer outfh.Flush()
		defer outw.Close()

		binMode := "termfit"
		if printBins > 0 {
			binMode = "fixed"
		}
		h := thist.NewHist([]float64{}, printField, binMode, printBins, true)

		transform := func(x float64) float64 { return x }
		if printLog {
			transform = func(x float64) float64 {
				return math.Log10(x + 1)
			}
		}

		field2col := make(map[string]int)
		var col int
		if config.NoHeaderRow {
			if len(printField) == 0 {
				fmt.Fprintf(os.Stderr, "No field specified!")
				os.Exit(1)
			}
			pcol, err := strconv.Atoi(printField)
			if err != nil {
				fmt.Fprintf(os.Stderr, "Illegal field number: %s\n", printField)
				os.Exit(1)
			}
			col = pcol - 1
			if col < 0 {
				fmt.Fprintf(os.Stderr, "Illegal field number: %d!\n", pcol)
				os.Exit(1)
			}
		}
		if len(printField) == 0 {
			fmt.Fprintf(os.Stderr, "No field specified!\n")
			os.Exit(1)
		}

		var count int
		for _, file := range files {
			csvReader, err := newCSVReaderByConfig(config, file)
			checkError(err)
			csvReader.Run()

			isHeaderLine := !config.NoHeaderRow
			for chunk := range csvReader.Ch {
				checkError(chunk.Err)

				for _, record := range chunk.Data {
					if isHeaderLine {
						for i, column := range record {
							field2col[column] = i
						}

						isHeaderLine = false
						if printPass {
							outfh.Write([]byte(strings.Join(record, string(config.OutDelimiter)) + "\n"))
						}
						continue
					} // header

					i := col
					if !config.NoHeaderRow {
						var ok bool
						i, ok = field2col[printField]
						if !ok {
							fmt.Fprintf(os.Stderr, "Invalid field specified: %s\n", printField)
							os.Exit(1)
						}
					}
					p, err := strconv.ParseFloat(record[i], 64)
					if err == nil {
						count++
						h.Update(transform(p))
						if printPass {
							outfh.Write([]byte(strings.Join(record, string(config.OutDelimiter)) + "\n"))
						}
					} else {
						continue
					}
					if printFreq > 0 && count%printFreq == 0 {
						if printDump {
							os.Stderr.Write([]byte(h.Dump()))
						} else {
							if !printQuiet {
								os.Stderr.Write([]byte(thist.ClearScreenString()))
								os.Stderr.Write([]byte(h.Draw()))
							}
							if printPdf != "" {
								h.SaveImage(printPdf)
							}
						}
						outfh.Flush()
						if printReset {
							h = thist.NewHist([]float64{}, printField, binMode, printBins, true)
						}
						time.Sleep(time.Duration(printDelay) * time.Second)
					}

				} // record
			} //chunk

		} //file
		if printFreq < 0 || count%printFreq != 0 {
			if printDump {
				os.Stderr.Write([]byte(h.Dump()))
			} else {
				if !printQuiet {
					os.Stderr.Write([]byte(thist.ClearScreenString()))
					os.Stderr.Write([]byte(h.Draw()))
				}
			}
			outfh.Flush()
			if printPdf != "" {
				h.SaveImage(printPdf)
			}
		}
	},
}

func init() {
	RootCmd.AddCommand(watchCmd)

	watchCmd.Flags().StringP("field", "f", "", "field to watch")
	watchCmd.Flags().IntP("print-freq", "p", -1, "print frequency (-1 for print after parsing)")
	watchCmd.Flags().StringP("image", "O", "", "print frequency (-1 for print after parsing)")
	watchCmd.Flags().IntP("delay", "w", 1, "sleep this many seconds after online plotting")
	watchCmd.Flags().IntP("bins", "B", -1, "number of histogram bins")
	watchCmd.Flags().BoolP("dump", "y", false, "dump histogram instead of plotting")
	watchCmd.Flags().BoolP("log", "L", false, "log10 transform values")
	watchCmd.Flags().BoolP("reset", "R", false, "reset histogram after every print")
	watchCmd.Flags().BoolP("pass", "x", false, "reset histogram after every print")
	watchCmd.Flags().BoolP("quiet", "Q", false, "reset histogram after every print")
}
<|endoftext|>


<commit_before>package dynect

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
)

const (
	DynAPIPrefix = "https://api.dynect.net/REST"
)

func init() {
	// Set the logging prefix.
	log.SetPrefix("go-dynect ")
}

// A client for use with DynECT's REST API.
type Client struct {
	Token        string
	CustomerName string
	httpclient   *http.Client
	verbose      bool
}

// Creates a new Httpclient.
func NewClient(customerName string) *Client {
	return &Client{
		CustomerName: customerName,
		httpclient:   &http.Client{}}
}

// Enable, or disable verbose output from the client.
//
// This will enable (or disable) logging messages that explain what the client
// is about to do, like the endpoint it is about to make a request to. If the
// request fails with an unexpected HTTP response code, then the response body
// will be logged out, as well.
func (c *Client) Verbose(p bool) {
	c.verbose = p
}

// Establishes a new session with the DynECT API.
func (c *Client) Login(username, password string) error {
	var req = LoginBlock{
		Username:     username,
		Password:     password,
		CustomerName: c.CustomerName}

	var resp LoginResponse

	err := c.Do("POST", "Session", req, &resp)
	if err != nil {
		return err
	}

	c.Token = resp.Data.Token
	return nil
}

func (c *Client) LoggedIn() bool {
	return len(c.Token) > 0
}

func (c *Client) Logout() error {
	return c.Do("DELETE", "Session", nil, nil)
}

func (c *Client) Do(method, endpoint string, requestData, responseData interface{}) error {
	// Throw an error if the user tries to make a request if the client is
	// logged out/unauthenticated, but make an exemption for when the
	// caller is trying to log in.
	if !c.LoggedIn() && method != "POST" && endpoint != "Session" {
		return errors.New("Will not perform request; httpclient is closed")
	}

	var err error

	// Marshal the request data into a byte slice.
	if c.verbose {
		log.Println("Marshaling request data")
	}
	var js []byte
	if requestData != nil {
		js, err = json.Marshal(requestData)
	} else {
		js = []byte("")
	}
	if err != nil {
		return err
	}

	// Create a new http.Request object, and set the necessary headers to
	// authorize the request, and specify the content type.
	url := fmt.Sprintf("%s/%s", DynAPIPrefix, endpoint)
	var req *http.Request
	req, err = http.NewRequest(method, url, bytes.NewReader(js))
	if err != nil {
		return err
	}
	req.Header.Set("Auth-Token", c.Token)
	req.Header.Set("Content-Type", "application/json")

	if c.verbose {
		log.Printf("Making %s request to %q", method, endpoint)
	}

	var resp *http.Response
	resp, err = c.httpclient.Do(req)
	if err != nil {
		if c.verbose {
			respBody, _ := ioutil.ReadAll(resp.Body)
			log.Printf("%s", string(respBody))
		}
		return err
	} else if resp.StatusCode != 200 {
		if c.verbose {
			// Print out the response body.
			respBody, _ := ioutil.ReadAll(resp.Body)
			log.Printf("%s", string(respBody))
		}
		return errors.New(fmt.Sprintf("Bad response, got %q", resp.Status))
	}

	// Unmarshal the response data into the provided struct.
	if c.verbose {
		log.Println("Reading in response data")
	}
	var respBody []byte
	respBody, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	if c.verbose {
		log.Println("Unmarshaling response data")
	}
	err = json.Unmarshal(respBody, &responseData)
	if err != nil {
		respBody, _ := ioutil.ReadAll(resp.Body)
		if resp.ContentLength == 0 || resp.ContentLength == -1 {
			log.Println("Zero-length content body")
		} else {
			log.Printf("%s", string(respBody))
		}
	}

	return err
}
<commit_msg>client: Verbose mode will now print the full URL.<commit_after>package dynect

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
)

const (
	DynAPIPrefix = "https://api.dynect.net/REST"
)

func init() {
	// Set the logging prefix.
	log.SetPrefix("go-dynect ")
}

// A client for use with DynECT's REST API.
type Client struct {
	Token        string
	CustomerName string
	httpclient   *http.Client
	verbose      bool
}

// Creates a new Httpclient.
func NewClient(customerName string) *Client {
	return &Client{
		CustomerName: customerName,
		httpclient:   &http.Client{}}
}

// Enable, or disable verbose output from the client.
//
// This will enable (or disable) logging messages that explain what the client
// is about to do, like the endpoint it is about to make a request to. If the
// request fails with an unexpected HTTP response code, then the response body
// will be logged out, as well.
func (c *Client) Verbose(p bool) {
	c.verbose = p
}

// Establishes a new session with the DynECT API.
func (c *Client) Login(username, password string) error {
	var req = LoginBlock{
		Username:     username,
		Password:     password,
		CustomerName: c.CustomerName}

	var resp LoginResponse

	err := c.Do("POST", "Session", req, &resp)
	if err != nil {
		return err
	}

	c.Token = resp.Data.Token
	return nil
}

func (c *Client) LoggedIn() bool {
	return len(c.Token) > 0
}

func (c *Client) Logout() error {
	return c.Do("DELETE", "Session", nil, nil)
}

func (c *Client) Do(method, endpoint string, requestData, responseData interface{}) error {
	// Throw an error if the user tries to make a request if the client is
	// logged out/unauthenticated, but make an exemption for when the
	// caller is trying to log in.
	if !c.LoggedIn() && method != "POST" && endpoint != "Session" {
		return errors.New("Will not perform request; httpclient is closed")
	}

	var err error

	// Marshal the request data into a byte slice.
	if c.verbose {
		log.Println("Marshaling request data")
	}
	var js []byte
	if requestData != nil {
		js, err = json.Marshal(requestData)
	} else {
		js = []byte("")
	}
	if err != nil {
		return err
	}

	// Create a new http.Request object, and set the necessary headers to
	// authorize the request, and specify the content type.
	url := fmt.Sprintf("%s/%s", DynAPIPrefix, endpoint)
	var req *http.Request
	req, err = http.NewRequest(method, url, bytes.NewReader(js))
	if err != nil {
		return err
	}
	req.Header.Set("Auth-Token", c.Token)
	req.Header.Set("Content-Type", "application/json")

	if c.verbose {
		log.Printf("Making %s request to %q", method, url)
	}

	var resp *http.Response
	resp, err = c.httpclient.Do(req)
	if err != nil {
		if c.verbose {
			respBody, _ := ioutil.ReadAll(resp.Body)
			log.Printf("%s", string(respBody))
		}
		return err
	} else if resp.StatusCode != 200 {
		if c.verbose {
			// Print out the response body.
			respBody, _ := ioutil.ReadAll(resp.Body)
			log.Printf("%s", string(respBody))
		}
		return errors.New(fmt.Sprintf("Bad response, got %q", resp.Status))
	}

	// Unmarshal the response data into the provided struct.
	if c.verbose {
		log.Println("Reading in response data")
	}
	var respBody []byte
	respBody, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	if c.verbose {
		log.Println("Unmarshaling response data")
	}
	err = json.Unmarshal(respBody, &responseData)
	if err != nil {
		respBody, _ := ioutil.ReadAll(resp.Body)
		if resp.ContentLength == 0 || resp.ContentLength == -1 {
			log.Println("Zero-length content body")
		} else {
			log.Printf("%s", string(respBody))
		}
	}

	return err
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/mailru/easyjson/bootstrap"
	// Reference the gen package to be friendly to vendoring tools,
	// as it is an indirect dependency.
	// (The temporary bootstrapping code uses it.)
	_ "github.com/mailru/easyjson/gen"
	"github.com/mailru/easyjson/parser"
)

var buildTags = flag.String("build_tags", "", "build tags to add to generated file")
var snakeCase = flag.Bool("snake_case", false, "use snake_case names instead of CamelCase by default")
var lowerCamelCase = flag.Bool("lower_camel_case", false, "use lowerCamelCase names instead of CamelCase by default")
var noStdMarshalers = flag.Bool("no_std_marshalers", false, "don't generate MarshalJSON/UnmarshalJSON funcs")
var omitEmpty = flag.Bool("omit_empty", false, "omit empty fields by default")
var allStructs = flag.Bool("all", false, "generate marshaler/unmarshalers for all structs in a file")
var simpleBytes = flag.Bool("byte", false, "use simple bytes instead of Base64Bytes for slice of bytes")
var leaveTemps = flag.Bool("leave_temps", false, "do not delete temporary files")
var stubs = flag.Bool("stubs", false, "only generate stubs for marshaler/unmarshaler funcs")
var noformat = flag.Bool("noformat", false, "do not run 'gofmt -w' on output file")
var specifiedName = flag.String("output_filename", "", "specify the filename of the output")
var processPkg = flag.Bool("pkg", false, "process the whole package instead of just the given file")
var disallowUnknownFields = flag.Bool("disallow_unknown_fields", false, "return error if any unknown field in json appeared")

func generate(fname string) (err error) {
	fInfo, err := os.Stat(fname)
	if err != nil {
		return err
	}

	p := parser.Parser{AllStructs: *allStructs}
	if err := p.Parse(fname, fInfo.IsDir()); err != nil {
		return fmt.Errorf("Error parsing %v: %v", fname, err)
	}

	var outName string
	if fInfo.IsDir() {
		outName = filepath.Join(fname, p.PkgName+"_easyjson.go")
	} else {
		if s := strings.TrimSuffix(fname, ".go"); s == fname {
			return errors.New("Filename must end in '.go'")
		} else {
			outName = s + "_easyjson.go"
		}
	}

	if *specifiedName != "" {
		outName = *specifiedName
	}

	var trimmedBuildTags string
	if *buildTags != "" {
		trimmedBuildTags = strings.TrimSpace(*buildTags)
	}

	g := bootstrap.Generator{
		BuildTags:             trimmedBuildTags,
		PkgPath:               p.PkgPath,
		PkgName:               p.PkgName,
		Types:                 p.StructNames,
		SnakeCase:             *snakeCase,
		LowerCamelCase:        *lowerCamelCase,
		NoStdMarshalers:       *noStdMarshalers,
		DisallowUnknownFields: *disallowUnknownFields,
		OmitEmpty:             *omitEmpty,
		LeaveTemps:            *leaveTemps,
		OutName:               outName,
		StubsOnly:             *stubs,
		NoFormat:              *noformat,
    SimpleBytes:           *simpleBytes,
	}

	if err := g.Run(); err != nil {
		return fmt.Errorf("Bootstrap failed: %v", err)
	}
	return nil
}

func main() {
	flag.Parse()

	files := flag.Args()

	gofile := os.Getenv("GOFILE")
	if *processPkg {
		gofile = filepath.Dir(gofile)
	}

	if len(files) == 0 && gofile != "" {
		files = []string{gofile}
	} else if len(files) == 0 {
		flag.Usage()
		os.Exit(1)
	}

	for _, fname := range files {
		if err := generate(fname); err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
	}
}
<commit_msg>fix easyjson main<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/mailru/easyjson/bootstrap"
	// Reference the gen package to be friendly to vendoring tools,
	// as it is an indirect dependency.
	// (The temporary bootstrapping code uses it.)
	_ "github.com/mailru/easyjson/gen"
	"github.com/mailru/easyjson/parser"
)

var buildTags = flag.String("build_tags", "", "build tags to add to generated file")
var snakeCase = flag.Bool("snake_case", false, "use snake_case names instead of CamelCase by default")
var lowerCamelCase = flag.Bool("lower_camel_case", false, "use lowerCamelCase names instead of CamelCase by default")
var noStdMarshalers = flag.Bool("no_std_marshalers", false, "don't generate MarshalJSON/UnmarshalJSON funcs")
var omitEmpty = flag.Bool("omit_empty", false, "omit empty fields by default")
var allStructs = flag.Bool("all", false, "generate marshaler/unmarshalers for all structs in a file")
var simpleBytes = flag.Bool("byte", false, "use simple bytes instead of Base64Bytes for slice of bytes")
var leaveTemps = flag.Bool("leave_temps", false, "do not delete temporary files")
var stubs = flag.Bool("stubs", false, "only generate stubs for marshaler/unmarshaler funcs")
var noformat = flag.Bool("noformat", false, "do not run 'gofmt -w' on output file")
var specifiedName = flag.String("output_filename", "", "specify the filename of the output")
var processPkg = flag.Bool("pkg", false, "process the whole package instead of just the given file")
var disallowUnknownFields = flag.Bool("disallow_unknown_fields", false, "return error if any unknown field in json appeared")

func generate(fname string) (err error) {
	fInfo, err := os.Stat(fname)
	if err != nil {
		return err
	}

	p := parser.Parser{AllStructs: *allStructs}
	if err := p.Parse(fname, fInfo.IsDir()); err != nil {
		return fmt.Errorf("Error parsing %v: %v", fname, err)
	}

	var outName string
	if fInfo.IsDir() {
		outName = filepath.Join(fname, p.PkgName+"_easyjson.go")
	} else {
		if s := strings.TrimSuffix(fname, ".go"); s == fname {
			return errors.New("Filename must end in '.go'")
		} else {
			outName = s + "_easyjson.go"
		}
	}

	if *specifiedName != "" {
		outName = *specifiedName
	}

	var trimmedBuildTags string
	if *buildTags != "" {
		trimmedBuildTags = strings.TrimSpace(*buildTags)
	}

	g := bootstrap.Generator{
		BuildTags:             trimmedBuildTags,
		PkgPath:               p.PkgPath,
		PkgName:               p.PkgName,
		Types:                 p.StructNames,
		SnakeCase:             *snakeCase,
		LowerCamelCase:        *lowerCamelCase,
		NoStdMarshalers:       *noStdMarshalers,
		DisallowUnknownFields: *disallowUnknownFields,
		OmitEmpty:             *omitEmpty,
		LeaveTemps:            *leaveTemps,
		OutName:               outName,
		StubsOnly:             *stubs,
		NoFormat:              *noformat,
		SimpleBytes:           *simpleBytes,
	}

	if err := g.Run(); err != nil {
		return fmt.Errorf("Bootstrap failed: %v", err)
	}
	return nil
}

func main() {
	flag.Parse()

	files := flag.Args()

	gofile := os.Getenv("GOFILE")
	if *processPkg {
		gofile = filepath.Dir(gofile)
	}

	if len(files) == 0 && gofile != "" {
		files = []string{gofile}
	} else if len(files) == 0 {
		flag.Usage()
		os.Exit(1)
	}

	for _, fname := range files {
		if err := generate(fname); err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
	}
}
<|endoftext|>