<commit_before>package tchannel_test

// Copyright (c) 2015 Uber Technologies, Inc.

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"strings"
	"testing"
	"time"

	. "github.com/uber/tchannel/golang"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"github.com/uber/tchannel/golang/testutils"
	"golang.org/x/net/context"
)

func makeRepeatedBytes(n byte) []byte {
	data := make([]byte, int(n))
	for i := byte(0); i < n; i++ {
		data[i] = n
	}
	return data
}

// streamPartialHandler returns a streaming handler that has the following contract:
// read a byte, write N bytes where N = the byte that was read.
// The results are be written as soon as the byte is read.
func streamPartialHandler(t *testing.T) HandlerFunc {
	return func(ctx context.Context, call *InboundCall) {
		response := call.Response()
		onError := func(err error) {
			t.Errorf("Handler error: %v", err)
			response.SendSystemError(fmt.Errorf("failed to read arg2"))
		}

		var arg2 []byte
		if err := NewArgReader(call.Arg2Reader()).Read(&arg2); err != nil {
			onError(fmt.Errorf("failed to read arg2"))
			return
		}

		if err := NewArgWriter(response.Arg2Writer()).Write(nil); err != nil {
			onError(fmt.Errorf(""))
			return
		}

		argReader, err := call.Arg3Reader()
		if err != nil {
			onError(fmt.Errorf("failed to read arg3"))
			return
		}

		argWriter, err := response.Arg3Writer()
		if err != nil {
			onError(fmt.Errorf("arg3 writer failed"))
			return
		}

		// Flush arg3 which will force a frame with just arg2 to be sent.
		// The test reads arg2 before arg3 has been sent.
		if err := argWriter.Flush(); err != nil {
			onError(fmt.Errorf("arg3 flush failed"))
			return
		}

		arg3 := make([]byte, 1)
		for {
			n, err := argReader.Read(arg3)
			if err == io.EOF {
				break
			}
			if n == 0 && err == nil {
				err = fmt.Errorf("read 0 bytes")
			}
			if err != nil {
				onError(fmt.Errorf("arg3 Read failed: %v", err))
				return
			}

			// Magic number to cause a failure
			if arg3[0] == 255 {
				response.SendSystemError(errors.New("intentional failure"))
				return
			}

			// Write the number of bytes as specified by arg3[0]
			if _, err := argWriter.Write(makeRepeatedBytes(arg3[0])); err != nil {
				onError(fmt.Errorf("argWriter Write failed: %v", err))
				return
			}
			if err := argWriter.Flush(); err != nil {
				onError(fmt.Errorf("argWriter flush failed: %v", err))
				return
			}
		}

		if err := argReader.Close(); err != nil {
			onError(fmt.Errorf("argReader Close failed: %v", err))
			return
		}

		if err := argWriter.Close(); err != nil {
			onError(fmt.Errorf("arg3writer Close failed: %v", err))
			return
		}
	}
}

func TestStreamPartialArg(t *testing.T) {
	defer testutils.SetTimeout(t, 2*time.Second)()
	ctx, cancel := NewContext(time.Second)
	defer cancel()

	require.NoError(t, testutils.WithServer(nil, func(ch *Channel, hostPort string) {
		ch.Register(streamPartialHandler(t), "echoStream")

		call, err := ch.BeginCall(ctx, hostPort, ch.PeerInfo().ServiceName, "echoStream", nil)
		require.NoError(t, err, "BeginCall failed")
		require.Nil(t, NewArgWriter(call.Arg2Writer()).Write(nil))

		argWriter, err := call.Arg3Writer()
		require.NoError(t, err, "Arg3Writer failed")

		// Flush arg3 to force the call to start without any arg3.
		require.NoError(t, argWriter.Flush(), "Arg3Writer flush failed")

		// Write out to the stream, and expect to get data
		response := call.Response()

		var arg2 []byte
		require.NoError(t, NewArgReader(response.Arg2Reader()).Read(&arg2), "Arg2Reader failed")
		require.False(t, response.ApplicationError(), "call failed")

		argReader, err := response.Arg3Reader()
		require.NoError(t, err, "Arg3Reader failed")

		verifyBytes := func(n byte) {
			_, err := argWriter.Write([]byte{n})
			require.NoError(t, err, "arg3 write failed")
			require.NoError(t, argWriter.Flush(), "arg3 flush failed")

			arg3 := make([]byte, int(n))
			_, err = io.ReadFull(argReader, arg3)
			require.NoError(t, err, "arg3 read failed")

			assert.Equal(t, makeRepeatedBytes(n), arg3, "arg3 result mismatch")
		}

		verifyBytes(0)
		verifyBytes(5)
		verifyBytes(100)
		verifyBytes(1)

		require.NoError(t, argWriter.Close(), "arg3 close failed")

		// Once closed, we expect the reader to return EOF
		n, err := io.Copy(ioutil.Discard, argReader)
		assert.Equal(t, int64(0), n, "arg2 reader expected to EOF after arg3 writer is closed")
	}))
}

func TestStreamSendError(t *testing.T) {
	defer testutils.SetTimeout(t, 2*time.Second)()
	ctx, cancel := NewContext(time.Second)
	defer cancel()

	require.NoError(t, testutils.WithServer(nil, func(ch *Channel, hostPort string) {
		ch.Register(streamPartialHandler(t), "echoStream")

		call, err := ch.BeginCall(ctx, hostPort, ch.PeerInfo().ServiceName, "echoStream", nil)
		require.NoError(t, err, "BeginCall failed")
		require.Nil(t, NewArgWriter(call.Arg2Writer()).Write(nil))

		argWriter, err := call.Arg3Writer()
		require.NoError(t, err, "Arg3Writer failed")

		// Flush arg3 to force the call to start without any arg3.
		require.NoError(t, argWriter.Flush(), "Arg3Writer flush failed")

		// Write out to the stream, and expect to get data
		response := call.Response()

		var arg2 []byte
		require.NoError(t, NewArgReader(response.Arg2Reader()).Read(&arg2), "Arg2Reader failed")
		require.False(t, response.ApplicationError(), "call failed")

		argReader, err := response.Arg3Reader()
		require.NoError(t, err, "Arg3Reader failed")

		verifyBytes := func(n byte) {
			_, err := argWriter.Write([]byte{n})
			require.NoError(t, err, "arg3 write failed")
			require.NoError(t, argWriter.Flush(), "arg3 flush failed")

			arg3 := make([]byte, int(n))
			_, err = io.ReadFull(argReader, arg3)
			require.NoError(t, err, "arg3 read failed")

			assert.Equal(t, makeRepeatedBytes(n), arg3, "arg3 result mismatch")
		}

		verifyBytes(0)
		verifyBytes(5)
		verifyBytes(100)
		verifyBytes(1)

		// Ask for an error!
		_, err = argWriter.Write([]byte{byte(255)})
		require.NoError(t, err, "arg3 write failed")
		require.NoError(t, argWriter.Flush(), "arg3 flush")

		// Now we expect an error on our next read.
		_, err = ioutil.ReadAll(argReader)
		assert.Error(t, err, "ReadAll should fail")
		assert.True(t, strings.Contains(err.Error(), "intentional failure"), "err %v unexpected", err)
		require.NoError(t, argWriter.Close(), "arg3 close failed")
	}))
}
<commit_msg>Refactor stream tests to share code.<commit_after>package tchannel_test

// Copyright (c) 2015 Uber Technologies, Inc.

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"strings"
	"testing"
	"time"

	. "github.com/uber/tchannel/golang"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"github.com/uber/tchannel/golang/testutils"
	"golang.org/x/net/context"
)

const streamRequestError = byte(255)

func makeRepeatedBytes(n byte) []byte {
	data := make([]byte, int(n))
	for i := byte(0); i < n; i++ {
		data[i] = n
	}
	return data
}

// streamPartialHandler returns a streaming handler that has the following contract:
// read a byte, write N bytes where N = the byte that was read.
// The results are be written as soon as the byte is read.
func streamPartialHandler(t *testing.T) HandlerFunc {
	return func(ctx context.Context, call *InboundCall) {
		response := call.Response()
		onError := func(err error) {
			t.Errorf("Handler error: %v", err)
			response.SendSystemError(fmt.Errorf("failed to read arg2"))
		}

		var arg2 []byte
		if err := NewArgReader(call.Arg2Reader()).Read(&arg2); err != nil {
			onError(fmt.Errorf("failed to read arg2"))
			return
		}

		if err := NewArgWriter(response.Arg2Writer()).Write(nil); err != nil {
			onError(fmt.Errorf(""))
			return
		}

		argReader, err := call.Arg3Reader()
		if err != nil {
			onError(fmt.Errorf("failed to read arg3"))
			return
		}

		argWriter, err := response.Arg3Writer()
		if err != nil {
			onError(fmt.Errorf("arg3 writer failed"))
			return
		}

		// Flush arg3 which will force a frame with just arg2 to be sent.
		// The test reads arg2 before arg3 has been sent.
		if err := argWriter.Flush(); err != nil {
			onError(fmt.Errorf("arg3 flush failed"))
			return
		}

		arg3 := make([]byte, 1)
		for {
			n, err := argReader.Read(arg3)
			if err == io.EOF {
				break
			}
			if n == 0 && err == nil {
				err = fmt.Errorf("read 0 bytes")
			}
			if err != nil {
				onError(fmt.Errorf("arg3 Read failed: %v", err))
				return
			}

			// Magic number to cause a failure
			if arg3[0] == streamRequestError {
				response.SendSystemError(errors.New("intentional failure"))
				return
			}

			// Write the number of bytes as specified by arg3[0]
			if _, err := argWriter.Write(makeRepeatedBytes(arg3[0])); err != nil {
				onError(fmt.Errorf("argWriter Write failed: %v", err))
				return
			}
			if err := argWriter.Flush(); err != nil {
				onError(fmt.Errorf("argWriter flush failed: %v", err))
				return
			}
		}

		if err := argReader.Close(); err != nil {
			onError(fmt.Errorf("argReader Close failed: %v", err))
			return
		}

		if err := argWriter.Close(); err != nil {
			onError(fmt.Errorf("arg3writer Close failed: %v", err))
			return
		}
	}
}

func testStreamArg(t *testing.T, f func(argWriter ArgWriter, argReader io.ReadCloser)) {
	defer testutils.SetTimeout(t, 2*time.Second)()
	ctx, cancel := NewContext(time.Second)
	defer cancel()

	require.NoError(t, testutils.WithServer(nil, func(ch *Channel, hostPort string) {
		ch.Register(streamPartialHandler(t), "echoStream")

		call, err := ch.BeginCall(ctx, hostPort, ch.PeerInfo().ServiceName, "echoStream", nil)
		require.NoError(t, err, "BeginCall failed")
		require.Nil(t, NewArgWriter(call.Arg2Writer()).Write(nil))

		argWriter, err := call.Arg3Writer()
		require.NoError(t, err, "Arg3Writer failed")

		// Flush arg3 to force the call to start without any arg3.
		require.NoError(t, argWriter.Flush(), "Arg3Writer flush failed")

		// Write out to the stream, and expect to get data
		response := call.Response()

		var arg2 []byte
		require.NoError(t, NewArgReader(response.Arg2Reader()).Read(&arg2), "Arg2Reader failed")
		require.False(t, response.ApplicationError(), "call failed")

		argReader, err := response.Arg3Reader()
		require.NoError(t, err, "Arg3Reader failed")

		verifyBytes := func(n byte) {
			_, err := argWriter.Write([]byte{n})
			require.NoError(t, err, "arg3 write failed")
			require.NoError(t, argWriter.Flush(), "arg3 flush failed")

			arg3 := make([]byte, int(n))
			_, err = io.ReadFull(argReader, arg3)
			require.NoError(t, err, "arg3 read failed")

			assert.Equal(t, makeRepeatedBytes(n), arg3, "arg3 result mismatch")
		}

		verifyBytes(0)
		verifyBytes(5)
		verifyBytes(100)
		verifyBytes(1)

		f(argWriter, argReader)
	}))
}

func TestStreamPartialArg(t *testing.T) {
	testStreamArg(t, func(argWriter ArgWriter, argReader io.ReadCloser) {
		require.NoError(t, argWriter.Close(), "arg3 close failed")

		// Once closed, we expect the reader to return EOF
		n, err := io.Copy(ioutil.Discard, argReader)
		assert.Equal(t, int64(0), n, "arg2 reader expected to EOF after arg3 writer is closed")
		assert.NoError(t, err, "Copy should not fail")
	})
}

func TestStreamSendError(t *testing.T) {
	testStreamArg(t, func(argWriter ArgWriter, argReader io.ReadCloser) {
		// Send the magic number to request an error.
		_, err := argWriter.Write([]byte{streamRequestError})
		require.NoError(t, err, "arg3 write failed")
		require.NoError(t, argWriter.Flush(), "arg3 flush")

		// Now we expect an error on our next read.
		_, err = ioutil.ReadAll(argReader)
		assert.Error(t, err, "ReadAll should fail")
		assert.True(t, strings.Contains(err.Error(), "intentional failure"), "err %v unexpected", err)
		require.NoError(t, argWriter.Close(), "arg3 close failed")
	})
}
<|endoftext|>


<commit_before>package epazote

import (
	"bytes"
	"crypto/tls"
	"net"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"time"
)

// URL regex to mach urls's
const URL string = `^((ftp|https?):\/\/)?(\S+(:\S*)?@)?((([1-9]\d?|1\d\d|2[01]\d|22[0-3])(\.(1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.([0-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(([a-zA-Z0-9]([a-zA-Z0-9-]+)?[a-zA-Z0-9]([-\.][a-zA-Z0-9]+)*)|((www\.)?))?(([a-zA-Z\x{00a1}-\x{ffff}0-9]+-?-?)*[a-zA-Z\x{00a1}-\x{ffff}0-9]+)(?:\.([a-zA-Z\x{00a1}-\x{ffff}]{1,}))?))(:(\d{1,5}))?((\/|\?|#)[^\s]*)?$`

var rxURL = regexp.MustCompile(URL)

// ServiceHttpResponse return struct
type ServiceHttpResponse struct {
	Err     error
	Service string
}

// AsyncGet used as a URL validation method
func AsyncGet(s *Services) <-chan ServiceHttpResponse {
	ch := make(chan ServiceHttpResponse, len(*s))

	for k, v := range *s {
		go func(name string, url string, verify bool, h map[string]string) {
			res, err := HTTPGet(url, true, verify, h)
			if err != nil {
				ch <- ServiceHttpResponse{err, name}
				return
			}
			res.Body.Close()
			ch <- ServiceHttpResponse{nil, name}
		}(k, v.URL, v.Insecure, v.Header)
	}

	return ch
}

// HTTPGet creates a new http request
func HTTPGet(url string, follow, insecure bool, h map[string]string, timeout ...int) (*http.Response, error) {
	// timeout in seconds defaults to 5
	var t int = 5

	if len(timeout) > 0 {
		t = timeout[0]
	}

	// if insecure = true, skip ssl verification
	tr := &http.Transport{
		Dial: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial,
		TLSHandshakeTimeout:   60 * time.Second,
		TLSClientConfig:       &tls.Config{InsecureSkipVerify: insecure},
		ResponseHeaderTimeout: time.Duration(t) * time.Second,
	}

	client := &http.Client{}
	client.Transport = tr

	// create a new request
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("User-Agent", "epazote")

	// set custom headers on request
	if h != nil {
		for k, v := range h {
			req.Header.Set(k, v)
		}
	}

	if follow {
		res, err := client.Do(req)
		if err != nil {
			return nil, err
		}
		return res, nil
	}

	// not follow redirects
	var DefaultTransport http.RoundTripper = tr

	res, err := DefaultTransport.RoundTrip(req)
	if err != nil {
		return nil, err
	}
	return res, nil
}

// HTTPPost post service json data
func HTTPPost(url string, data []byte, h map[string]string) (*http.Response, error) {
	// create a new request
	req, err := http.NewRequest("POST", url, bytes.NewBuffer(data))
	if err != nil {
		return nil, err
	}
	req.Header.Set("User-Agent", "epazote")
	req.Header.Set("Content-Type", "application/json")

	// set custom headers on request
	if h != nil {
		for k, v := range h {
			req.Header.Set(k, v)
		}
	}

	client := &http.Client{}

	res, err := client.Do(req)
	if err != nil {
		return nil, err
	}

	return res, nil
}

// IsURL check if the string is an URL https://github.com/asaskevich/govalidator/blob/master/validator.go#L45
func IsURL(str string) bool {
	if str == "" || len(str) >= 2083 || len(str) <= 3 || strings.HasPrefix(str, ".") {
		return false
	}
	u, err := url.Parse(str)
	if err != nil {
		return false
	}
	if strings.HasPrefix(u.Host, ".") {
		return false
	}
	if u.Host == "" && (u.Path != "" && !strings.Contains(u.Path, ".")) {
		return false
	}
	return rxURL.MatchString(str)
}
<commit_msg>rever requests test<commit_after>package epazote

import (
	"bytes"
	"crypto/tls"
	"net"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"time"
)

// URL regex to mach urls's
const URL string = `^((ftp|https?):\/\/)?(\S+(:\S*)?@)?((([1-9]\d?|1\d\d|2[01]\d|22[0-3])(\.(1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.([0-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(([a-zA-Z0-9]([a-zA-Z0-9-]+)?[a-zA-Z0-9]([-\.][a-zA-Z0-9]+)*)|((www\.)?))?(([a-zA-Z\x{00a1}-\x{ffff}0-9]+-?-?)*[a-zA-Z\x{00a1}-\x{ffff}0-9]+)(?:\.([a-zA-Z\x{00a1}-\x{ffff}]{1,}))?))(:(\d{1,5}))?((\/|\?|#)[^\s]*)?$`

var rxURL = regexp.MustCompile(URL)

// ServiceHttpResponse return struct
type ServiceHttpResponse struct {
	Err     error
	Service string
}

// AsyncGet used as a URL validation method
func AsyncGet(s *Services) <-chan ServiceHttpResponse {
	ch := make(chan ServiceHttpResponse, len(*s))

	for k, v := range *s {
		go func(name string, url string, verify bool, h map[string]string) {
			res, err := HTTPGet(url, true, verify, h)
			if err != nil {
				ch <- ServiceHttpResponse{err, name}
				return
			}
			res.Body.Close()
			ch <- ServiceHttpResponse{nil, name}
		}(k, v.URL, v.Insecure, v.Header)
	}

	return ch
}

// HTTPGet creates a new http request
func HTTPGet(url string, follow, insecure bool, h map[string]string, timeout ...int) (*http.Response, error) {
	// timeout in seconds defaults to 5
	var t int = 5

	if len(timeout) > 0 {
		t = timeout[0]
	}

	// if insecure = true, skip ssl verification
	tr := &http.Transport{
		Dial: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial,
		TLSHandshakeTimeout:   60 * time.Second,
		TLSClientConfig:       &tls.Config{InsecureSkipVerify: insecure},
		ResponseHeaderTimeout: time.Duration(t) * time.Second,
	}

	client := &http.Client{}
	client.Transport = tr

	// create a new request
	req, _ := http.NewRequest("GET", url, nil)
	req.Header.Set("User-Agent", "epazote")

	// set custom headers on request
	if h != nil {
		for k, v := range h {
			req.Header.Set(k, v)
		}
	}

	if follow {
		res, err := client.Do(req)
		if err != nil {
			return nil, err
		}
		return res, nil
	}

	// not follow redirects
	var DefaultTransport http.RoundTripper = tr

	res, err := DefaultTransport.RoundTrip(req)
	if err != nil {
		return nil, err
	}
	return res, nil
}

// HTTPPost post service json data
func HTTPPost(url string, data []byte, h map[string]string) (*http.Response, error) {
	// create a new request
	req, _ := http.NewRequest("POST", url, bytes.NewBuffer(data))
	req.Header.Set("User-Agent", "epazote")
	req.Header.Set("Content-Type", "application/json")

	// set custom headers on request
	if h != nil {
		for k, v := range h {
			req.Header.Set(k, v)
		}
	}

	client := &http.Client{}

	res, err := client.Do(req)
	if err != nil {
		return nil, err
	}

	return res, nil
}

// IsURL check if the string is an URL https://github.com/asaskevich/govalidator/blob/master/validator.go#L45
func IsURL(str string) bool {
	if str == "" || len(str) >= 2083 || len(str) <= 3 || strings.HasPrefix(str, ".") {
		return false
	}
	u, err := url.Parse(str)
	if err != nil {
		return false
	}
	if strings.HasPrefix(u.Host, ".") {
		return false
	}
	if u.Host == "" && (u.Path != "" && !strings.Contains(u.Path, ".")) {
		return false
	}
	return rxURL.MatchString(str)
}
<|endoftext|>


<commit_before>// +build windows

package win_perf_counters

import (
	"errors"
	"fmt"
	"strings"
	"unsafe"

	"github.com/influxdata/telegraf"
	"github.com/influxdata/telegraf/plugins/inputs"
)

var sampleConfig = `
  ## By default this plugin returns basic CPU and Disk statistics.
  ## See the README file for more examples.
  ## Uncomment examples below or write your own as you see fit. If the system
  ## being polled for data does not have the Object at startup of the Telegraf
  ## agent, it will not be gathered.
  ## Settings:
  # PrintValid = false # Print All matching performance counters

  [[inputs.win_perf_counters.object]]
    # Processor usage, alternative to native, reports on a per core.
    ObjectName = "Processor"
    Instances = ["*"]
    Counters = [
      "%% Idle Time", "%% Interrupt Time",
      "%% Privileged Time", "%% User Time",
      "%% Processor Time"
    ]
    Measurement = "win_cpu"
    # Set to true to include _Total instance when querying for all (*).
    # IncludeTotal=false
    # Print out when the performance counter is missing from object, counter or instance.
    # WarnOnMissing = false

  [[inputs.win_perf_counters.object]]
    # Disk times and queues
    ObjectName = "LogicalDisk"
    Instances = ["*"]
    Counters = [
      "%% Idle Time", "%% Disk Time","%% Disk Read Time",
      "%% Disk Write Time", "%% User Time", "Current Disk Queue Length"
    ]
    Measurement = "win_disk"

  [[inputs.win_perf_counters.object]]
    ObjectName = "System"
    Counters = ["Context Switches/sec","System Calls/sec"]
    Instances = ["------"]
    Measurement = "win_system"

  [[inputs.win_perf_counters.object]]
    # Example query where the Instance portion must be removed to get data back,
    # such as from the Memory object.
    ObjectName = "Memory"
    Counters = [
      "Available Bytes", "Cache Faults/sec", "Demand Zero Faults/sec",
      "Page Faults/sec", "Pages/sec", "Transition Faults/sec",
      "Pool Nonpaged Bytes", "Pool Paged Bytes"
    ]
    Instances = ["------"] # Use 6 x - to remove the Instance bit from the query.
    Measurement = "win_mem"
`

// Valid queries end up in this map.
var gItemList = make(map[int]*item)

var configParsed bool
var testConfigParsed bool
var testObject string

type Win_PerfCounters struct {
	PrintValid      bool
	TestName        string
	PreVistaSupport bool
	Object          []perfobject
}

type perfobject struct {
	ObjectName    string
	Counters      []string
	Instances     []string
	Measurement   string
	WarnOnMissing bool
	FailOnMissing bool
	IncludeTotal  bool
}

// Parsed configuration ends up here after it has been validated for valid
// Performance Counter paths
type itemList struct {
	items map[int]*item
}

type item struct {
	query         string
	objectName    string
	counter       string
	instance      string
	measurement   string
	include_total bool
	handle        PDH_HQUERY
	counterHandle PDH_HCOUNTER
}

var sanitizedChars = strings.NewReplacer("/sec", "_persec", "/Sec", "_persec",
	" ", "_", "%", "Percent", `\`, "")

func (m *Win_PerfCounters) AddItem(metrics *itemList, query string, objectName string, counter string, instance string,
	measurement string, include_total bool) error {

	var handle PDH_HQUERY
	var counterHandle PDH_HCOUNTER
	ret := PdhOpenQuery(0, 0, &handle)
	if m.PreVistaSupport {
		ret = PdhAddCounter(handle, query, 0, &counterHandle)
	} else {
		ret = PdhAddEnglishCounter(handle, query, 0, &counterHandle)
	}

	// Call PdhCollectQueryData one time to check existance of the counter
	ret = PdhCollectQueryData(handle)
	if ret != ERROR_SUCCESS {
		PdhCloseQuery(handle)
		return errors.New(PdhFormatError(ret))
	}

	temp := &item{query, objectName, counter, instance, measurement,
		include_total, handle, counterHandle}
	index := len(gItemList)
	gItemList[index] = temp

	if metrics.items == nil {
		metrics.items = make(map[int]*item)
	}
	metrics.items[index] = temp
	return nil
}

func (m *Win_PerfCounters) Description() string {
	return "Input plugin to query Performance Counters on Windows operating systems"
}

func (m *Win_PerfCounters) SampleConfig() string {
	return sampleConfig
}

func (m *Win_PerfCounters) ParseConfig(metrics *itemList) error {
	var query string

	configParsed = true

	if len(m.Object) > 0 {
		for _, PerfObject := range m.Object {
			for _, counter := range PerfObject.Counters {
				for _, instance := range PerfObject.Instances {
					objectname := PerfObject.ObjectName

					if instance == "------" {
						query = "\\" + objectname + "\\" + counter
					} else {
						query = "\\" + objectname + "(" + instance + ")\\" + counter
					}

					err := m.AddItem(metrics, query, objectname, counter, instance,
						PerfObject.Measurement, PerfObject.IncludeTotal)

					if err == nil {
						if m.PrintValid {
							fmt.Printf("Valid: %s\n", query)
						}
					} else {
						if PerfObject.FailOnMissing || PerfObject.WarnOnMissing {
							fmt.Printf("Invalid query: '%s'. Error: %s", query, err.Error())
						}
						if PerfObject.FailOnMissing {
							return err
						}
					}
				}
			}
		}

		return nil
	} else {
		err := errors.New("No performance objects configured!")
		return err
	}
}

func (m *Win_PerfCounters) Cleanup(metrics *itemList) {
	// Cleanup

	for _, metric := range metrics.items {
		ret := PdhCloseQuery(metric.handle)
		_ = ret
	}
}

func (m *Win_PerfCounters) CleanupTestMode() {
	// Cleanup for the testmode.

	for _, metric := range gItemList {
		ret := PdhCloseQuery(metric.handle)
		_ = ret
	}
}

func (m *Win_PerfCounters) Gather(acc telegraf.Accumulator) error {
	metrics := itemList{}

	// Both values are empty in normal use.
	if m.TestName != testObject {
		// Cleanup any handles before emptying the global variable containing valid queries.
		m.CleanupTestMode()
		gItemList = make(map[int]*item)
		testObject = m.TestName
		testConfigParsed = true
		configParsed = false
	}

	// We only need to parse the config during the init, it uses the global variable after.
	if configParsed == false {

		err := m.ParseConfig(&metrics)
		if err != nil {
			return err
		}
	}

	var bufSize uint32
	var bufCount uint32
	var size uint32 = uint32(unsafe.Sizeof(PDH_FMT_COUNTERVALUE_ITEM_DOUBLE{}))
	var emptyBuf [1]PDH_FMT_COUNTERVALUE_ITEM_DOUBLE // need at least 1 addressable null ptr.

	// For iterate over the known metrics and get the samples.
	for _, metric := range gItemList {
		// collect
		ret := PdhCollectQueryData(metric.handle)
		if ret == ERROR_SUCCESS {
			ret = PdhGetFormattedCounterArrayDouble(metric.counterHandle, &bufSize,
				&bufCount, &emptyBuf[0]) // uses null ptr here according to MSDN.
			if ret == PDH_MORE_DATA {
				filledBuf := make([]PDH_FMT_COUNTERVALUE_ITEM_DOUBLE, bufCount*size)
				if len(filledBuf) == 0 {
					continue
				}
				ret = PdhGetFormattedCounterArrayDouble(metric.counterHandle,
					&bufSize, &bufCount, &filledBuf[0])
				for i := 0; i < int(bufCount); i++ {
					c := filledBuf[i]
					var s string = UTF16PtrToString(c.SzName)

					var add bool

					if metric.include_total {
						// If IncludeTotal is set, include all.
						add = true
					} else if metric.instance == "*" && !strings.Contains(s, "_Total") {
						// Catch if set to * and that it is not a '*_Total*' instance.
						add = true
					} else if metric.instance == s {
						// Catch if we set it to total or some form of it
						add = true
					} else if strings.Contains(metric.instance, "#") && strings.HasPrefix(metric.instance, s) {
						// If you are using a multiple instance identifier such as "w3wp#1"
						// phd.dll returns only the first 2 characters of the identifier.
						add = true
						s = metric.instance
					} else if metric.instance == "------" {
						add = true
					}

					if add {
						fields := make(map[string]interface{})
						tags := make(map[string]string)
						if s != "" {
							tags["instance"] = s
						}
						tags["objectname"] = metric.objectName
						fields[sanitizedChars.Replace(metric.counter)] =
							float32(c.FmtValue.DoubleValue)

						measurement := sanitizedChars.Replace(metric.measurement)
						if measurement == "" {
							measurement = "win_perf_counters"
						}
						acc.AddFields(measurement, fields, tags)
					}
				}

				filledBuf = nil
				// Need to at least set bufSize to zero, because if not, the function will not
				// return PDH_MORE_DATA and will not set the bufSize.
				bufCount = 0
				bufSize = 0
			}
		}
	}

	return nil
}

func init() {
	inputs.Add("win_perf_counters", func() telegraf.Input { return &Win_PerfCounters{} })
}
<commit_msg>Fix multiple plugin loading in win_perf_counters (#2800)<commit_after>// +build windows

package win_perf_counters

import (
	"errors"
	"fmt"
	"strings"
	"unsafe"

	"github.com/influxdata/telegraf"
	"github.com/influxdata/telegraf/plugins/inputs"
)

var sampleConfig = `
  ## By default this plugin returns basic CPU and Disk statistics.
  ## See the README file for more examples.
  ## Uncomment examples below or write your own as you see fit. If the system
  ## being polled for data does not have the Object at startup of the Telegraf
  ## agent, it will not be gathered.
  ## Settings:
  # PrintValid = false # Print All matching performance counters

  [[inputs.win_perf_counters.object]]
    # Processor usage, alternative to native, reports on a per core.
    ObjectName = "Processor"
    Instances = ["*"]
    Counters = [
      "%% Idle Time", "%% Interrupt Time",
      "%% Privileged Time", "%% User Time",
      "%% Processor Time"
    ]
    Measurement = "win_cpu"
    # Set to true to include _Total instance when querying for all (*).
    # IncludeTotal=false
    # Print out when the performance counter is missing from object, counter or instance.
    # WarnOnMissing = false

  [[inputs.win_perf_counters.object]]
    # Disk times and queues
    ObjectName = "LogicalDisk"
    Instances = ["*"]
    Counters = [
      "%% Idle Time", "%% Disk Time","%% Disk Read Time",
      "%% Disk Write Time", "%% User Time", "Current Disk Queue Length"
    ]
    Measurement = "win_disk"

  [[inputs.win_perf_counters.object]]
    ObjectName = "System"
    Counters = ["Context Switches/sec","System Calls/sec"]
    Instances = ["------"]
    Measurement = "win_system"

  [[inputs.win_perf_counters.object]]
    # Example query where the Instance portion must be removed to get data back,
    # such as from the Memory object.
    ObjectName = "Memory"
    Counters = [
      "Available Bytes", "Cache Faults/sec", "Demand Zero Faults/sec",
      "Page Faults/sec", "Pages/sec", "Transition Faults/sec",
      "Pool Nonpaged Bytes", "Pool Paged Bytes"
    ]
    Instances = ["------"] # Use 6 x - to remove the Instance bit from the query.
    Measurement = "win_mem"
`

type Win_PerfCounters struct {
	PrintValid      bool
	PreVistaSupport bool
	Object          []perfobject

	configParsed bool
	itemCache    []*item
}

type perfobject struct {
	ObjectName    string
	Counters      []string
	Instances     []string
	Measurement   string
	WarnOnMissing bool
	FailOnMissing bool
	IncludeTotal  bool
}

type item struct {
	query         string
	objectName    string
	counter       string
	instance      string
	measurement   string
	include_total bool
	handle        PDH_HQUERY
	counterHandle PDH_HCOUNTER
}

var sanitizedChars = strings.NewReplacer("/sec", "_persec", "/Sec", "_persec",
	" ", "_", "%", "Percent", `\`, "")

func (m *Win_PerfCounters) AddItem(query string, objectName string, counter string, instance string,
	measurement string, include_total bool) error {

	var handle PDH_HQUERY
	var counterHandle PDH_HCOUNTER
	ret := PdhOpenQuery(0, 0, &handle)
	if m.PreVistaSupport {
		ret = PdhAddCounter(handle, query, 0, &counterHandle)
	} else {
		ret = PdhAddEnglishCounter(handle, query, 0, &counterHandle)
	}

	// Call PdhCollectQueryData one time to check existance of the counter
	ret = PdhCollectQueryData(handle)
	if ret != ERROR_SUCCESS {
		PdhCloseQuery(handle)
		return errors.New(PdhFormatError(ret))
	}

	newItem := &item{query, objectName, counter, instance, measurement,
		include_total, handle, counterHandle}
	m.itemCache = append(m.itemCache, newItem)

	return nil
}

func (m *Win_PerfCounters) Description() string {
	return "Input plugin to query Performance Counters on Windows operating systems"
}

func (m *Win_PerfCounters) SampleConfig() string {
	return sampleConfig
}

func (m *Win_PerfCounters) ParseConfig() error {
	var query string

	if len(m.Object) > 0 {
		for _, PerfObject := range m.Object {
			for _, counter := range PerfObject.Counters {
				for _, instance := range PerfObject.Instances {
					objectname := PerfObject.ObjectName

					if instance == "------" {
						query = "\\" + objectname + "\\" + counter
					} else {
						query = "\\" + objectname + "(" + instance + ")\\" + counter
					}

					err := m.AddItem(query, objectname, counter, instance,
						PerfObject.Measurement, PerfObject.IncludeTotal)

					if err == nil {
						if m.PrintValid {
							fmt.Printf("Valid: %s\n", query)
						}
					} else {
						if PerfObject.FailOnMissing || PerfObject.WarnOnMissing {
							fmt.Printf("Invalid query: '%s'. Error: %s", query, err.Error())
						}
						if PerfObject.FailOnMissing {
							return err
						}
					}
				}
			}
		}

		return nil
	} else {
		err := errors.New("No performance objects configured!")
		return err
	}
}

func (m *Win_PerfCounters) Gather(acc telegraf.Accumulator) error {
	// Parse the config once
	if !m.configParsed {
		err := m.ParseConfig()
		m.configParsed = true
		if err != nil {
			return err
		}
	}

	var bufSize uint32
	var bufCount uint32
	var size uint32 = uint32(unsafe.Sizeof(PDH_FMT_COUNTERVALUE_ITEM_DOUBLE{}))
	var emptyBuf [1]PDH_FMT_COUNTERVALUE_ITEM_DOUBLE // need at least 1 addressable null ptr.

	// For iterate over the known metrics and get the samples.
	for _, metric := range m.itemCache {
		// collect
		ret := PdhCollectQueryData(metric.handle)
		if ret == ERROR_SUCCESS {
			ret = PdhGetFormattedCounterArrayDouble(metric.counterHandle, &bufSize,
				&bufCount, &emptyBuf[0]) // uses null ptr here according to MSDN.
			if ret == PDH_MORE_DATA {
				filledBuf := make([]PDH_FMT_COUNTERVALUE_ITEM_DOUBLE, bufCount*size)
				if len(filledBuf) == 0 {
					continue
				}
				ret = PdhGetFormattedCounterArrayDouble(metric.counterHandle,
					&bufSize, &bufCount, &filledBuf[0])
				for i := 0; i < int(bufCount); i++ {
					c := filledBuf[i]
					var s string = UTF16PtrToString(c.SzName)

					var add bool

					if metric.include_total {
						// If IncludeTotal is set, include all.
						add = true
					} else if metric.instance == "*" && !strings.Contains(s, "_Total") {
						// Catch if set to * and that it is not a '*_Total*' instance.
						add = true
					} else if metric.instance == s {
						// Catch if we set it to total or some form of it
						add = true
					} else if strings.Contains(metric.instance, "#") && strings.HasPrefix(metric.instance, s) {
						// If you are using a multiple instance identifier such as "w3wp#1"
						// phd.dll returns only the first 2 characters of the identifier.
						add = true
						s = metric.instance
					} else if metric.instance == "------" {
						add = true
					}

					if add {
						fields := make(map[string]interface{})
						tags := make(map[string]string)
						if s != "" {
							tags["instance"] = s
						}
						tags["objectname"] = metric.objectName
						fields[sanitizedChars.Replace(metric.counter)] =
							float32(c.FmtValue.DoubleValue)

						measurement := sanitizedChars.Replace(metric.measurement)
						if measurement == "" {
							measurement = "win_perf_counters"
						}
						acc.AddFields(measurement, fields, tags)
					}
				}

				filledBuf = nil
				// Need to at least set bufSize to zero, because if not, the function will not
				// return PDH_MORE_DATA and will not set the bufSize.
				bufCount = 0
				bufSize = 0
			}
		}
	}

	return nil
}

func init() {
	inputs.Add("win_perf_counters", func() telegraf.Input { return &Win_PerfCounters{} })
}
<|endoftext|>


<commit_before>package graphicsmagick

import (
	"fmt"
	"github.com/pierrre/imageserver"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
)

const tempDirPrefix = "imageserver_"

type GraphicsMagickProcessor struct {
	Executable       string
	TempDir          string
	AllowedFormats   []string
	DefaultQualities map[string]string
}

func (converter *GraphicsMagickProcessor) Process(sourceImage *imageserver.Image, parameters imageserver.Parameters) (image *imageserver.Image, err error) {
	var arguments []string

	arguments = append(arguments, "mogrify")

	arguments, width, height, err := converter.buildArgumentsResize(arguments, parameters)
	if err != nil {
		return
	}

	arguments, err = converter.buildArgumentsBackground(arguments, parameters)
	if err != nil {
		return
	}

	arguments, err = converter.buildArgumentsExtent(arguments, parameters, width, height)
	if err != nil {
		return
	}

	arguments, format, hasFileExtension, err := converter.buildArgumentsFormat(arguments, parameters, sourceImage)
	if err != nil {
		return
	}

	arguments, err = converter.buildArgumentsQuality(arguments, parameters, format)
	if err != nil {
		return
	}

	if len(arguments) == 1 {
		image = sourceImage
		return
	}

	tempDir, err := ioutil.TempDir(converter.TempDir, tempDirPrefix)
	if err != nil {
		return
	}
	defer os.RemoveAll(tempDir)

	file := filepath.Join(tempDir, "image")
	arguments = append(arguments, file)
	err = ioutil.WriteFile(file, sourceImage.Data, os.FileMode(0600))
	if err != nil {
		return
	}

	cmd := exec.Command(converter.Executable, arguments...)
	err = cmd.Run()
	if err != nil {
		return
	}

	if hasFileExtension {
		file = fmt.Sprintf("%s.%s", file, format)
	}
	data, err := ioutil.ReadFile(file)
	if err != nil {
		return
	}

	image = &imageserver.Image{}
	image.Data = data
	image.Type = format

	return
}

func (converter *GraphicsMagickProcessor) buildArgumentsResize(in []string, parameters imageserver.Parameters) (arguments []string, width int, height int, err error) {
	arguments = in

	width, _ = parameters.GetInt("gm.width")
	if width < 0 {
		err = fmt.Errorf("Invalid width")
		return
	}

	height, _ = parameters.GetInt("gm.height")
	if height < 0 {
		err = fmt.Errorf("Invalid height")
		return
	}

	if width != 0 || height != 0 {
		widthString := ""
		if width != 0 {
			widthString = strconv.Itoa(width)
		}
		heightString := ""
		if height != 0 {
			heightString = strconv.Itoa(height)
		}
		resize := fmt.Sprintf("%sx%s", widthString, heightString)

		if fill, _ := parameters.GetBool("gm.fill"); fill {
			resize = resize + "^"
		}

		if ignoreRatio, _ := parameters.GetBool("gm.ignore_ratio"); ignoreRatio {
			resize = resize + "!"
		}

		if onlyShrinkLarger, _ := parameters.GetBool("gm.only_shrink_larger"); onlyShrinkLarger {
			resize = resize + ">"
		}

		if onlyEnlargeSmaller, _ := parameters.GetBool("gm.only_enlarge_smaller"); onlyEnlargeSmaller {
			resize = resize + "<"
		}

		arguments = append(arguments, "-resize", resize)
	}

	return
}

func (converter *GraphicsMagickProcessor) buildArgumentsBackground(in []string, parameters imageserver.Parameters) (arguments []string, err error) {
	arguments = in

	background, _ := parameters.GetString("gm.background")

	if backgroundLength := len(background); backgroundLength > 0 {
		if backgroundLength != 6 && backgroundLength != 8 && backgroundLength != 3 && backgroundLength != 4 {
			err = fmt.Errorf("Invalid background")
			return
		}

		for _, r := range background {
			if (r < '0' || r > '9') && (r < 'a' || r > 'f') {
				err = fmt.Errorf("Invalid background")
				return
			}
		}

		arguments = append(arguments, "-background", fmt.Sprintf("#%s", background))
	}

	return
}

func (converter *GraphicsMagickProcessor) buildArgumentsExtent(in []string, parameters imageserver.Parameters, width int, height int) (arguments []string, err error) {
	arguments = in

	if width != 0 && height != 0 {
		if extent, _ := parameters.GetBool("gm.extent"); extent {
			arguments = append(arguments, "-gravity", "center")
			arguments = append(arguments, "-extent", fmt.Sprintf("%dx%d", width, height))
		}
	}

	return
}

func (converter *GraphicsMagickProcessor) buildArgumentsFormat(in []string, parameters imageserver.Parameters, sourceImage *imageserver.Image) (arguments []string, format string, hasFileExtension bool, err error) {
	arguments = in

	format, _ = parameters.GetString("gm.format")

	formatSpecified := true
	if len(format) == 0 {
		format = sourceImage.Type
		formatSpecified = false
	}

	if converter.AllowedFormats != nil {
		ok := false
		for _, f := range converter.AllowedFormats {
			if f == format {
				ok = true
				break
			}
		}
		if !ok {
			err = fmt.Errorf("Invalid format")
			return
		}
	}

	if formatSpecified {
		arguments = append(arguments, "-format", format)
	}

	hasFileExtension = formatSpecified

	return
}

func (converter *GraphicsMagickProcessor) buildArgumentsQuality(in []string, parameters imageserver.Parameters, format string) (arguments []string, err error) {
	arguments = in

	quality, _ := parameters.GetString("gm.quality")

	if len(quality) == 0 && len(arguments) == 1 {
		return
	}

	if len(quality) == 0 && converter.DefaultQualities != nil {
		if q, ok := converter.DefaultQualities[format]; ok {
			quality = q
		}
	}

	if len(quality) > 0 {
		qualityInt, e := strconv.Atoi(quality)
		if e != nil {
			err = e
			return
		}

		if qualityInt < 0 {
			err = fmt.Errorf("Invalid quality")
			return
		}

		if format == "jpeg" {
			if qualityInt < 0 || qualityInt > 100 {
				err = fmt.Errorf("Invalid quality")
				return
			}
		}

		arguments = append(arguments, "-quality", quality)
	}

	return
}
<commit_msg>Add error for grraphicsmagick processor<commit_after>package graphicsmagick

import (
	"fmt"
	"github.com/pierrre/imageserver"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
)

const tempDirPrefix = "imageserver_"

type GraphicsMagickProcessor struct {
	Executable       string
	TempDir          string
	AllowedFormats   []string
	DefaultQualities map[string]string
}

func (converter *GraphicsMagickProcessor) Process(sourceImage *imageserver.Image, parameters imageserver.Parameters) (image *imageserver.Image, err error) {
	var arguments []string

	arguments = append(arguments, "mogrify")

	arguments, width, height, err := converter.buildArgumentsResize(arguments, parameters)
	if err != nil {
		return
	}

	arguments, err = converter.buildArgumentsBackground(arguments, parameters)
	if err != nil {
		return
	}

	arguments, err = converter.buildArgumentsExtent(arguments, parameters, width, height)
	if err != nil {
		return
	}

	arguments, format, hasFileExtension, err := converter.buildArgumentsFormat(arguments, parameters, sourceImage)
	if err != nil {
		return
	}

	arguments, err = converter.buildArgumentsQuality(arguments, parameters, format)
	if err != nil {
		return
	}

	if len(arguments) == 1 {
		image = sourceImage
		return
	}

	tempDir, err := ioutil.TempDir(converter.TempDir, tempDirPrefix)
	if err != nil {
		return
	}
	defer os.RemoveAll(tempDir)

	file := filepath.Join(tempDir, "image")
	arguments = append(arguments, file)
	err = ioutil.WriteFile(file, sourceImage.Data, os.FileMode(0600))
	if err != nil {
		return
	}

	cmd := exec.Command(converter.Executable, arguments...)
	err = cmd.Run()
	if err != nil {
		err = imageserver.NewError("Error during execution of GraphicsMagick")
		return
	}

	if hasFileExtension {
		file = fmt.Sprintf("%s.%s", file, format)
	}
	data, err := ioutil.ReadFile(file)
	if err != nil {
		return
	}

	image = &imageserver.Image{}
	image.Data = data
	image.Type = format

	return
}

func (converter *GraphicsMagickProcessor) buildArgumentsResize(in []string, parameters imageserver.Parameters) (arguments []string, width int, height int, err error) {
	arguments = in

	width, _ = parameters.GetInt("gm.width")
	if width < 0 {
		err = imageserver.NewError("Invalid width parameter")
		return
	}

	height, _ = parameters.GetInt("gm.height")
	if height < 0 {
		err = imageserver.NewError("Invalid height parameter")
		return
	}

	if width != 0 || height != 0 {
		widthString := ""
		if width != 0 {
			widthString = strconv.Itoa(width)
		}
		heightString := ""
		if height != 0 {
			heightString = strconv.Itoa(height)
		}
		resize := fmt.Sprintf("%sx%s", widthString, heightString)

		if fill, _ := parameters.GetBool("gm.fill"); fill {
			resize = resize + "^"
		}

		if ignoreRatio, _ := parameters.GetBool("gm.ignore_ratio"); ignoreRatio {
			resize = resize + "!"
		}

		if onlyShrinkLarger, _ := parameters.GetBool("gm.only_shrink_larger"); onlyShrinkLarger {
			resize = resize + ">"
		}

		if onlyEnlargeSmaller, _ := parameters.GetBool("gm.only_enlarge_smaller"); onlyEnlargeSmaller {
			resize = resize + "<"
		}

		arguments = append(arguments, "-resize", resize)
	}

	return
}

func (converter *GraphicsMagickProcessor) buildArgumentsBackground(in []string, parameters imageserver.Parameters) (arguments []string, err error) {
	arguments = in

	background, _ := parameters.GetString("gm.background")

	if backgroundLength := len(background); backgroundLength > 0 {
		if backgroundLength != 6 && backgroundLength != 8 && backgroundLength != 3 && backgroundLength != 4 {
			err = imageserver.NewError("Invalid background parameter")
			return
		}

		for _, r := range background {
			if (r < '0' || r > '9') && (r < 'a' || r > 'f') {
				err = imageserver.NewError("Invalid background parameter")
				return
			}
		}

		arguments = append(arguments, "-background", fmt.Sprintf("#%s", background))
	}

	return
}

func (converter *GraphicsMagickProcessor) buildArgumentsExtent(in []string, parameters imageserver.Parameters, width int, height int) (arguments []string, err error) {
	arguments = in

	if width != 0 && height != 0 {
		if extent, _ := parameters.GetBool("gm.extent"); extent {
			arguments = append(arguments, "-gravity", "center")
			arguments = append(arguments, "-extent", fmt.Sprintf("%dx%d", width, height))
		}
	}

	return
}

func (converter *GraphicsMagickProcessor) buildArgumentsFormat(in []string, parameters imageserver.Parameters, sourceImage *imageserver.Image) (arguments []string, format string, hasFileExtension bool, err error) {
	arguments = in

	format, _ = parameters.GetString("gm.format")

	formatSpecified := true
	if len(format) == 0 {
		format = sourceImage.Type
		formatSpecified = false
	}

	if converter.AllowedFormats != nil {
		ok := false
		for _, f := range converter.AllowedFormats {
			if f == format {
				ok = true
				break
			}
		}
		if !ok {
			err = imageserver.NewError("Invalid format parameter")
			return
		}
	}

	if formatSpecified {
		arguments = append(arguments, "-format", format)
	}

	hasFileExtension = formatSpecified

	return
}

func (converter *GraphicsMagickProcessor) buildArgumentsQuality(in []string, parameters imageserver.Parameters, format string) (arguments []string, err error) {
	arguments = in

	quality, _ := parameters.GetString("gm.quality")

	if len(quality) == 0 && len(arguments) == 1 {
		return
	}

	if len(quality) == 0 && converter.DefaultQualities != nil {
		if q, ok := converter.DefaultQualities[format]; ok {
			quality = q
		}
	}

	if len(quality) > 0 {
		var qualityInt int
		qualityInt, err = strconv.Atoi(quality)
		if err != nil {
			err = imageserver.NewError("Invalid quality parameter (parse int error)")
			return
		}

		if qualityInt < 0 {
			err = imageserver.NewError("Invalid quality parameter (less than 0)")
			return
		}

		if format == "jpeg" {
			if qualityInt < 0 || qualityInt > 100 {
				err = imageserver.NewError("Invalid quality parameter (must be between 0 and 100)")
				return
			}
		}

		arguments = append(arguments, "-quality", quality)
	}

	return
}
<|endoftext|>


<commit_before>package ipallocator

import (
	"sync"
)

// iPSet is a thread-safe sorted set and a stack.
type iPSet struct {
	sync.RWMutex
	set []string
}

// Push takes a string and adds it to the set. If the elem aready exists, it has no effect.
func (s *iPSet) Push(elem string) {
	s.RLock()
	for i, e := range s.set {
		if e == elem {
			s.RUnlock()
			return
		}
	}
	s.RUnlock()

	s.Lock()
	s.set = append(s.set, elem)
	// Make sure the list is always sorted
	sort.Strings(s.set)
	s.Unlock()
}

// Pop returns the first elemen from the list and removes it.
// If the list is empty, it returns an empty string
func (s *iPSet) Pop() string {
	s.RLock()

	for i, e := range s.set {
		ret := e
		s.RUnlock()
		s.Lock()
		s.set = append(s.set[:i], s.set[i+1:]...)
		s.Unlock()
		return e
	}
	s.RUnlock()
	return ""
}

// Remove removes an element from the list.
// If the element is not found, it has no effect.
func (s *iPSet) Remove(elem string) {
	for i, e := range s.set {
		if e == elem {
			s.set = append(s.set[:i], s.set[i+1:]...)
			return
		}
	}
}

// Len returns the length of the list.
func (s *iPSet) Len() int {
	return len(s.set)
}
<commit_msg>Implement Containers in set<commit_after>package ipallocator

import (
	"sync"
)

// iPSet is a thread-safe sorted set and a stack.
type iPSet struct {
	sync.RWMutex
	set []string
}

// Push takes a string and adds it to the set. If the elem aready exists, it has no effect.
func (s *iPSet) Push(elem string) {
	s.RLock()
	for i, e := range s.set {
		if e == elem {
			s.RUnlock()
			return
		}
	}
	s.RUnlock()

	s.Lock()
	s.set = append(s.set, elem)
	// Make sure the list is always sorted
	sort.Strings(s.set)
	s.Unlock()
}

// Pop returns the first elemen from the list and removes it.
// If the list is empty, it returns an empty string
func (s *iPSet) Pop() string {
	s.RLock()

	for i, e := range s.set {
		ret := e
		s.RUnlock()
		s.Lock()
		s.set = append(s.set[:i], s.set[i+1:]...)
		s.Unlock()
		return e
	}
	s.RUnlock()
	return ""
}

// Exists checks if the given element present in the list.
func (s *iPSet) Exists(elem string) bool {
	for _, e := range s.set {
		if e == elem {
			return true
		}
	}
	return false
}

// Remove removes an element from the list.
// If the element is not found, it has no effect.
func (s *iPSet) Remove(elem string) {
	for i, e := range s.set {
		if e == elem {
			s.set = append(s.set[:i], s.set[i+1:]...)
			return
		}
	}
}

// Len returns the length of the list.
func (s *iPSet) Len() int {
	return len(s.set)
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
	"time"

	"github.com/mattn/go-sqlite3"

	"github.com/lxc/lxd/shared"
)

var (
	// DbErrAlreadyDefined hapens when the given entry already exists,
	// for example a container.
	DbErrAlreadyDefined = fmt.Errorf("The container/snapshot already exists")

	/* NoSuchObjectError is in the case of joins (and probably other) queries,
	 * we don't get back sql.ErrNoRows when no rows are returned, even though we do
	 * on selects without joins. Instead, you can use this error to
	 * propagate up and generate proper 404s to the client when something
	 * isn't found so we don't abuse sql.ErrNoRows any more than we
	 * already do.
	 */
	NoSuchObjectError = fmt.Errorf("No such object")
)

// CURRENT_SCHEMA contains the current SQLite SQL Schema.
const CURRENT_SCHEMA string = `
CREATE TABLE IF NOT EXISTS certificates (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    fingerprint VARCHAR(255) NOT NULL,
    type INTEGER NOT NULL,
    name VARCHAR(255) NOT NULL,
    certificate TEXT NOT NULL,
    UNIQUE (fingerprint)
);
CREATE TABLE IF NOT EXISTS config (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    key VARCHAR(255) NOT NULL,
    value TEXT,
    UNIQUE (key)
);
CREATE TABLE IF NOT EXISTS containers (
    id INTEGER primary key AUTOINCREMENT NOT NULL,
    name VARCHAR(255) NOT NULL,
    architecture INTEGER NOT NULL,
    type INTEGER NOT NULL,
    ephemeral INTEGER NOT NULL DEFAULT 0,
    stateful INTEGER NOT NULL DEFAULT 0,
    creation_date DATETIME,
    last_use_date DATETIME,
    UNIQUE (name)
);
CREATE TABLE IF NOT EXISTS containers_config (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    container_id INTEGER NOT NULL,
    key VARCHAR(255) NOT NULL,
    value TEXT,
    FOREIGN KEY (container_id) REFERENCES containers (id) ON DELETE CASCADE,
    UNIQUE (container_id, key)
);
CREATE TABLE IF NOT EXISTS containers_devices (
    id INTEGER primary key AUTOINCREMENT NOT NULL,
    container_id INTEGER NOT NULL,
    name VARCHAR(255) NOT NULL,
    type INTEGER NOT NULL default 0,
    FOREIGN KEY (container_id) REFERENCES containers (id) ON DELETE CASCADE,
    UNIQUE (container_id, name)
);
CREATE TABLE IF NOT EXISTS containers_devices_config (
    id INTEGER primary key AUTOINCREMENT NOT NULL,
    container_device_id INTEGER NOT NULL,
    key VARCHAR(255) NOT NULL,
    value TEXT,
    FOREIGN KEY (container_device_id) REFERENCES containers_devices (id) ON DELETE CASCADE,
    UNIQUE (container_device_id, key)
);
CREATE TABLE IF NOT EXISTS containers_profiles (
    id INTEGER primary key AUTOINCREMENT NOT NULL,
    container_id INTEGER NOT NULL,
    profile_id INTEGER NOT NULL,
    apply_order INTEGER NOT NULL default 0,
    UNIQUE (container_id, profile_id),
    FOREIGN KEY (container_id) REFERENCES containers(id) ON DELETE CASCADE,
    FOREIGN KEY (profile_id) REFERENCES profiles(id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS images (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    cached INTEGER NOT NULL DEFAULT 0,
    fingerprint VARCHAR(255) NOT NULL,
    filename VARCHAR(255) NOT NULL,
    size INTEGER NOT NULL,
    public INTEGER NOT NULL DEFAULT 0,
    auto_update INTEGER NOT NULL DEFAULT 0,
    architecture INTEGER NOT NULL,
    creation_date DATETIME,
    expiry_date DATETIME,
    upload_date DATETIME NOT NULL,
    last_use_date DATETIME,
    UNIQUE (fingerprint)
);
CREATE TABLE IF NOT EXISTS images_aliases (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    name VARCHAR(255) NOT NULL,
    image_id INTEGER NOT NULL,
    description VARCHAR(255),
    FOREIGN KEY (image_id) REFERENCES images (id) ON DELETE CASCADE,
    UNIQUE (name)
);
CREATE TABLE IF NOT EXISTS images_properties (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    image_id INTEGER NOT NULL,
    type INTEGER NOT NULL,
    key VARCHAR(255) NOT NULL,
    value TEXT,
    FOREIGN KEY (image_id) REFERENCES images (id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS images_source (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    image_id INTEGER NOT NULL,
    server TEXT NOT NULL,
    protocol INTEGER NOT NULL,
    certificate TEXT NOT NULL,
    alias VARCHAR(255) NOT NULL,
    FOREIGN KEY (image_id) REFERENCES images (id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS networks (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    name VARCHAR(255) NOT NULL,
    UNIQUE (name)
);
CREATE TABLE IF NOT EXISTS networks_config (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    network_id INTEGER NOT NULL,
    key VARCHAR(255) NOT NULL,
    value TEXT,
    UNIQUE (network_id, key),
    FOREIGN KEY (network_id) REFERENCES networks (id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS patches (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    name VARCHAR(255) NOT NULL,
    applied_at DATETIME NOT NULL,
    UNIQUE (name)
);
CREATE TABLE IF NOT EXISTS profiles (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    name VARCHAR(255) NOT NULL,
    description TEXT,
    UNIQUE (name)
);
CREATE TABLE IF NOT EXISTS profiles_config (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    profile_id INTEGER NOT NULL,
    key VARCHAR(255) NOT NULL,
    value VARCHAR(255),
    UNIQUE (profile_id, key),
    FOREIGN KEY (profile_id) REFERENCES profiles(id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS profiles_devices (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    profile_id INTEGER NOT NULL,
    name VARCHAR(255) NOT NULL,
    type INTEGER NOT NULL default 0,
    UNIQUE (profile_id, name),
    FOREIGN KEY (profile_id) REFERENCES profiles (id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS profiles_devices_config (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    profile_device_id INTEGER NOT NULL,
    key VARCHAR(255) NOT NULL,
    value TEXT,
    UNIQUE (profile_device_id, key),
    FOREIGN KEY (profile_device_id) REFERENCES profiles_devices (id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS schema (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    version INTEGER NOT NULL,
    updated_at DATETIME NOT NULL,
    UNIQUE (version)
);`

// Create the initial (current) schema for a given SQLite DB connection.
func createDb(db *sql.DB) (err error) {
	latestVersion := dbGetSchema(db)

	if latestVersion != 0 {
		return nil
	}

	_, err = db.Exec(CURRENT_SCHEMA)
	if err != nil {
		return err
	}

	// There isn't an entry for schema version, let's put it in.
	insertStmt := `INSERT INTO schema (version, updated_at) values (?, strftime("%s"));`
	_, err = db.Exec(insertStmt, dbGetLatestSchema())
	if err != nil {
		return err
	}

	err = dbProfileCreateDefault(db)
	if err != nil {
		return err
	}

	return dbProfileCreateDocker(db)
}

func dbGetSchema(db *sql.DB) (v int) {
	arg1 := []interface{}{}
	arg2 := []interface{}{&v}
	q := "SELECT max(version) FROM schema"
	err := dbQueryRowScan(db, q, arg1, arg2)
	if err != nil {
		return 0
	}
	return v
}

func dbGetLatestSchema() int {
	return dbUpdates[len(dbUpdates)-1].version
}

// Create a database connection object and return it.
func initializeDbObject(d *Daemon, path string) (err error) {
	var openPath string

	timeout := 5 // TODO - make this command-line configurable?

	// These are used to tune the transaction BEGIN behavior instead of using the
	// similar "locking_mode" pragma (locking for the whole database connection).
	openPath = fmt.Sprintf("%s?_busy_timeout=%d&_txlock=exclusive", path, timeout*1000)

	// Open the database. If the file doesn't exist it is created.
	d.db, err = sql.Open("sqlite3", openPath)
	if err != nil {
		return err
	}

	// Create the DB if it doesn't exist.
	err = createDb(d.db)
	if err != nil {
		return fmt.Errorf("Error creating database: %s", err)
	}

	// Run PRAGMA statements now since they are *per-connection*.
	d.db.Exec("PRAGMA foreign_keys=ON;") // This allows us to use ON DELETE CASCADE

	// Apply any update
	err = dbUpdatesApplyAll(d)
	if err != nil {
		return err
	}

	return nil
}

func isDbLockedError(err error) bool {
	if err == nil {
		return false
	}
	if err == sqlite3.ErrLocked || err == sqlite3.ErrBusy {
		return true
	}
	if err.Error() == "database is locked" {
		return true
	}
	return false
}

func isNoMatchError(err error) bool {
	if err == nil {
		return false
	}
	if err.Error() == "sql: no rows in result set" {
		return true
	}
	return false
}

func dbBegin(db *sql.DB) (*sql.Tx, error) {
	for i := 0; i < 100; i++ {
		tx, err := db.Begin()
		if err == nil {
			return tx, nil
		}
		if !isDbLockedError(err) {
			shared.LogDebugf("DbBegin: error %q", err)
			return nil, err
		}
		time.Sleep(100 * time.Millisecond)
	}

	shared.LogDebugf("DbBegin: DB still locked")
	shared.PrintStack()
	return nil, fmt.Errorf("DB is locked")
}

func txCommit(tx *sql.Tx) error {
	for i := 0; i < 100; i++ {
		err := tx.Commit()
		if err == nil {
			return nil
		}
		if !isDbLockedError(err) {
			shared.LogDebugf("Txcommit: error %q", err)
			return err
		}
		time.Sleep(100 * time.Millisecond)
	}

	shared.LogDebugf("Txcommit: db still locked")
	shared.PrintStack()
	return fmt.Errorf("DB is locked")
}

func dbQueryRowScan(db *sql.DB, q string, args []interface{}, outargs []interface{}) error {
	for i := 0; i < 100; i++ {
		err := db.QueryRow(q, args...).Scan(outargs...)
		if err == nil {
			return nil
		}
		if isNoMatchError(err) {
			return err
		}
		if !isDbLockedError(err) {
			return err
		}
		time.Sleep(100 * time.Millisecond)
	}

	shared.LogDebugf("DbQueryRowScan: query %q args %q, DB still locked", q, args)
	shared.PrintStack()
	return fmt.Errorf("DB is locked")
}

func dbQuery(db *sql.DB, q string, args ...interface{}) (*sql.Rows, error) {
	for i := 0; i < 100; i++ {
		result, err := db.Query(q, args...)
		if err == nil {
			return result, nil
		}
		if !isDbLockedError(err) {
			shared.LogDebugf("DbQuery: query %q error %q", q, err)
			return nil, err
		}
		time.Sleep(100 * time.Millisecond)
	}

	shared.LogDebugf("DbQuery: query %q args %q, DB still locked", q, args)
	shared.PrintStack()
	return nil, fmt.Errorf("DB is locked")
}

func doDbQueryScan(db *sql.DB, q string, args []interface{}, outargs []interface{}) ([][]interface{}, error) {
	rows, err := db.Query(q, args...)
	if err != nil {
		return [][]interface{}{}, err
	}
	defer rows.Close()
	result := [][]interface{}{}
	for rows.Next() {
		ptrargs := make([]interface{}, len(outargs))
		for i := range outargs {
			switch t := outargs[i].(type) {
			case string:
				str := ""
				ptrargs[i] = &str
			case int:
				integer := 0
				ptrargs[i] = &integer
			case int64:
				integer := int64(0)
				ptrargs[i] = &integer
			default:
				return [][]interface{}{}, fmt.Errorf("Bad interface type: %s", t)
			}
		}
		err = rows.Scan(ptrargs...)
		if err != nil {
			return [][]interface{}{}, err
		}
		newargs := make([]interface{}, len(outargs))
		for i := range ptrargs {
			switch t := outargs[i].(type) {
			case string:
				newargs[i] = *ptrargs[i].(*string)
			case int:
				newargs[i] = *ptrargs[i].(*int)
			case int64:
				newargs[i] = *ptrargs[i].(*int64)
			default:
				return [][]interface{}{}, fmt.Errorf("Bad interface type: %s", t)
			}
		}
		result = append(result, newargs)
	}
	err = rows.Err()
	if err != nil {
		return [][]interface{}{}, err
	}
	return result, nil
}

/*
 * . q is the database query
 * . inargs is an array of interfaces containing the query arguments
 * . outfmt is an array of interfaces containing the right types of output
 *   arguments, i.e.
 *      var arg1 string
 *      var arg2 int
 *      outfmt := {}interface{}{arg1, arg2}
 *
 * The result will be an array (one per output row) of arrays (one per output argument)
 * of interfaces, containing pointers to the actual output arguments.
 */
func dbQueryScan(db *sql.DB, q string, inargs []interface{}, outfmt []interface{}) ([][]interface{}, error) {
	for i := 0; i < 100; i++ {
		result, err := doDbQueryScan(db, q, inargs, outfmt)
		if err == nil {
			return result, nil
		}
		if !isDbLockedError(err) {
			shared.LogDebugf("DbQuery: query %q error %q", q, err)
			return nil, err
		}
		time.Sleep(100 * time.Millisecond)
	}

	shared.LogDebugf("DbQueryscan: query %q inargs %q, DB still locked", q, inargs)
	shared.PrintStack()
	return nil, fmt.Errorf("DB is locked")
}

func dbExec(db *sql.DB, q string, args ...interface{}) (sql.Result, error) {
	for i := 0; i < 100; i++ {
		result, err := db.Exec(q, args...)
		if err == nil {
			return result, nil
		}
		if !isDbLockedError(err) {
			shared.LogDebugf("DbExec: query %q error %q", q, err)
			return nil, err
		}
		time.Sleep(100 * time.Millisecond)
	}

	shared.LogDebugf("DbExec: query %q args %q, DB still locked", q, args)
	shared.PrintStack()
	return nil, fmt.Errorf("DB is locked")
}
<commit_msg>patches: Mark all patches as applied on create<commit_after>package main

import (
	"database/sql"
	"fmt"
	"time"

	"github.com/mattn/go-sqlite3"

	"github.com/lxc/lxd/shared"
)

var (
	// DbErrAlreadyDefined hapens when the given entry already exists,
	// for example a container.
	DbErrAlreadyDefined = fmt.Errorf("The container/snapshot already exists")

	/* NoSuchObjectError is in the case of joins (and probably other) queries,
	 * we don't get back sql.ErrNoRows when no rows are returned, even though we do
	 * on selects without joins. Instead, you can use this error to
	 * propagate up and generate proper 404s to the client when something
	 * isn't found so we don't abuse sql.ErrNoRows any more than we
	 * already do.
	 */
	NoSuchObjectError = fmt.Errorf("No such object")
)

// CURRENT_SCHEMA contains the current SQLite SQL Schema.
const CURRENT_SCHEMA string = `
CREATE TABLE IF NOT EXISTS certificates (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    fingerprint VARCHAR(255) NOT NULL,
    type INTEGER NOT NULL,
    name VARCHAR(255) NOT NULL,
    certificate TEXT NOT NULL,
    UNIQUE (fingerprint)
);
CREATE TABLE IF NOT EXISTS config (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    key VARCHAR(255) NOT NULL,
    value TEXT,
    UNIQUE (key)
);
CREATE TABLE IF NOT EXISTS containers (
    id INTEGER primary key AUTOINCREMENT NOT NULL,
    name VARCHAR(255) NOT NULL,
    architecture INTEGER NOT NULL,
    type INTEGER NOT NULL,
    ephemeral INTEGER NOT NULL DEFAULT 0,
    stateful INTEGER NOT NULL DEFAULT 0,
    creation_date DATETIME,
    last_use_date DATETIME,
    UNIQUE (name)
);
CREATE TABLE IF NOT EXISTS containers_config (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    container_id INTEGER NOT NULL,
    key VARCHAR(255) NOT NULL,
    value TEXT,
    FOREIGN KEY (container_id) REFERENCES containers (id) ON DELETE CASCADE,
    UNIQUE (container_id, key)
);
CREATE TABLE IF NOT EXISTS containers_devices (
    id INTEGER primary key AUTOINCREMENT NOT NULL,
    container_id INTEGER NOT NULL,
    name VARCHAR(255) NOT NULL,
    type INTEGER NOT NULL default 0,
    FOREIGN KEY (container_id) REFERENCES containers (id) ON DELETE CASCADE,
    UNIQUE (container_id, name)
);
CREATE TABLE IF NOT EXISTS containers_devices_config (
    id INTEGER primary key AUTOINCREMENT NOT NULL,
    container_device_id INTEGER NOT NULL,
    key VARCHAR(255) NOT NULL,
    value TEXT,
    FOREIGN KEY (container_device_id) REFERENCES containers_devices (id) ON DELETE CASCADE,
    UNIQUE (container_device_id, key)
);
CREATE TABLE IF NOT EXISTS containers_profiles (
    id INTEGER primary key AUTOINCREMENT NOT NULL,
    container_id INTEGER NOT NULL,
    profile_id INTEGER NOT NULL,
    apply_order INTEGER NOT NULL default 0,
    UNIQUE (container_id, profile_id),
    FOREIGN KEY (container_id) REFERENCES containers(id) ON DELETE CASCADE,
    FOREIGN KEY (profile_id) REFERENCES profiles(id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS images (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    cached INTEGER NOT NULL DEFAULT 0,
    fingerprint VARCHAR(255) NOT NULL,
    filename VARCHAR(255) NOT NULL,
    size INTEGER NOT NULL,
    public INTEGER NOT NULL DEFAULT 0,
    auto_update INTEGER NOT NULL DEFAULT 0,
    architecture INTEGER NOT NULL,
    creation_date DATETIME,
    expiry_date DATETIME,
    upload_date DATETIME NOT NULL,
    last_use_date DATETIME,
    UNIQUE (fingerprint)
);
CREATE TABLE IF NOT EXISTS images_aliases (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    name VARCHAR(255) NOT NULL,
    image_id INTEGER NOT NULL,
    description VARCHAR(255),
    FOREIGN KEY (image_id) REFERENCES images (id) ON DELETE CASCADE,
    UNIQUE (name)
);
CREATE TABLE IF NOT EXISTS images_properties (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    image_id INTEGER NOT NULL,
    type INTEGER NOT NULL,
    key VARCHAR(255) NOT NULL,
    value TEXT,
    FOREIGN KEY (image_id) REFERENCES images (id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS images_source (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    image_id INTEGER NOT NULL,
    server TEXT NOT NULL,
    protocol INTEGER NOT NULL,
    certificate TEXT NOT NULL,
    alias VARCHAR(255) NOT NULL,
    FOREIGN KEY (image_id) REFERENCES images (id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS networks (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    name VARCHAR(255) NOT NULL,
    UNIQUE (name)
);
CREATE TABLE IF NOT EXISTS networks_config (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    network_id INTEGER NOT NULL,
    key VARCHAR(255) NOT NULL,
    value TEXT,
    UNIQUE (network_id, key),
    FOREIGN KEY (network_id) REFERENCES networks (id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS patches (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    name VARCHAR(255) NOT NULL,
    applied_at DATETIME NOT NULL,
    UNIQUE (name)
);
CREATE TABLE IF NOT EXISTS profiles (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    name VARCHAR(255) NOT NULL,
    description TEXT,
    UNIQUE (name)
);
CREATE TABLE IF NOT EXISTS profiles_config (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    profile_id INTEGER NOT NULL,
    key VARCHAR(255) NOT NULL,
    value VARCHAR(255),
    UNIQUE (profile_id, key),
    FOREIGN KEY (profile_id) REFERENCES profiles(id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS profiles_devices (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    profile_id INTEGER NOT NULL,
    name VARCHAR(255) NOT NULL,
    type INTEGER NOT NULL default 0,
    UNIQUE (profile_id, name),
    FOREIGN KEY (profile_id) REFERENCES profiles (id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS profiles_devices_config (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    profile_device_id INTEGER NOT NULL,
    key VARCHAR(255) NOT NULL,
    value TEXT,
    UNIQUE (profile_device_id, key),
    FOREIGN KEY (profile_device_id) REFERENCES profiles_devices (id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS schema (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    version INTEGER NOT NULL,
    updated_at DATETIME NOT NULL,
    UNIQUE (version)
);`

// Create the initial (current) schema for a given SQLite DB connection.
func createDb(db *sql.DB) (err error) {
	latestVersion := dbGetSchema(db)

	if latestVersion != 0 {
		return nil
	}

	_, err = db.Exec(CURRENT_SCHEMA)
	if err != nil {
		return err
	}

	// There isn't an entry for schema version, let's put it in.
	insertStmt := `INSERT INTO schema (version, updated_at) values (?, strftime("%s"));`
	_, err = db.Exec(insertStmt, dbGetLatestSchema())
	if err != nil {
		return err
	}

	// Mark all existing patches as applied
	for _, p := range patches {
		dbPatchesMarkApplied(db, p.name)
	}

	err = dbProfileCreateDefault(db)
	if err != nil {
		return err
	}

	return dbProfileCreateDocker(db)
}

func dbGetSchema(db *sql.DB) (v int) {
	arg1 := []interface{}{}
	arg2 := []interface{}{&v}
	q := "SELECT max(version) FROM schema"
	err := dbQueryRowScan(db, q, arg1, arg2)
	if err != nil {
		return 0
	}
	return v
}

func dbGetLatestSchema() int {
	return dbUpdates[len(dbUpdates)-1].version
}

// Create a database connection object and return it.
func initializeDbObject(d *Daemon, path string) (err error) {
	var openPath string

	timeout := 5 // TODO - make this command-line configurable?

	// These are used to tune the transaction BEGIN behavior instead of using the
	// similar "locking_mode" pragma (locking for the whole database connection).
	openPath = fmt.Sprintf("%s?_busy_timeout=%d&_txlock=exclusive", path, timeout*1000)

	// Open the database. If the file doesn't exist it is created.
	d.db, err = sql.Open("sqlite3", openPath)
	if err != nil {
		return err
	}

	// Create the DB if it doesn't exist.
	err = createDb(d.db)
	if err != nil {
		return fmt.Errorf("Error creating database: %s", err)
	}

	// Run PRAGMA statements now since they are *per-connection*.
	d.db.Exec("PRAGMA foreign_keys=ON;") // This allows us to use ON DELETE CASCADE

	// Apply any update
	err = dbUpdatesApplyAll(d)
	if err != nil {
		return err
	}

	return nil
}

func isDbLockedError(err error) bool {
	if err == nil {
		return false
	}
	if err == sqlite3.ErrLocked || err == sqlite3.ErrBusy {
		return true
	}
	if err.Error() == "database is locked" {
		return true
	}
	return false
}

func isNoMatchError(err error) bool {
	if err == nil {
		return false
	}
	if err.Error() == "sql: no rows in result set" {
		return true
	}
	return false
}

func dbBegin(db *sql.DB) (*sql.Tx, error) {
	for i := 0; i < 100; i++ {
		tx, err := db.Begin()
		if err == nil {
			return tx, nil
		}
		if !isDbLockedError(err) {
			shared.LogDebugf("DbBegin: error %q", err)
			return nil, err
		}
		time.Sleep(100 * time.Millisecond)
	}

	shared.LogDebugf("DbBegin: DB still locked")
	shared.PrintStack()
	return nil, fmt.Errorf("DB is locked")
}

func txCommit(tx *sql.Tx) error {
	for i := 0; i < 100; i++ {
		err := tx.Commit()
		if err == nil {
			return nil
		}
		if !isDbLockedError(err) {
			shared.LogDebugf("Txcommit: error %q", err)
			return err
		}
		time.Sleep(100 * time.Millisecond)
	}

	shared.LogDebugf("Txcommit: db still locked")
	shared.PrintStack()
	return fmt.Errorf("DB is locked")
}

func dbQueryRowScan(db *sql.DB, q string, args []interface{}, outargs []interface{}) error {
	for i := 0; i < 100; i++ {
		err := db.QueryRow(q, args...).Scan(outargs...)
		if err == nil {
			return nil
		}
		if isNoMatchError(err) {
			return err
		}
		if !isDbLockedError(err) {
			return err
		}
		time.Sleep(100 * time.Millisecond)
	}

	shared.LogDebugf("DbQueryRowScan: query %q args %q, DB still locked", q, args)
	shared.PrintStack()
	return fmt.Errorf("DB is locked")
}

func dbQuery(db *sql.DB, q string, args ...interface{}) (*sql.Rows, error) {
	for i := 0; i < 100; i++ {
		result, err := db.Query(q, args...)
		if err == nil {
			return result, nil
		}
		if !isDbLockedError(err) {
			shared.LogDebugf("DbQuery: query %q error %q", q, err)
			return nil, err
		}
		time.Sleep(100 * time.Millisecond)
	}

	shared.LogDebugf("DbQuery: query %q args %q, DB still locked", q, args)
	shared.PrintStack()
	return nil, fmt.Errorf("DB is locked")
}

func doDbQueryScan(db *sql.DB, q string, args []interface{}, outargs []interface{}) ([][]interface{}, error) {
	rows, err := db.Query(q, args...)
	if err != nil {
		return [][]interface{}{}, err
	}
	defer rows.Close()
	result := [][]interface{}{}
	for rows.Next() {
		ptrargs := make([]interface{}, len(outargs))
		for i := range outargs {
			switch t := outargs[i].(type) {
			case string:
				str := ""
				ptrargs[i] = &str
			case int:
				integer := 0
				ptrargs[i] = &integer
			case int64:
				integer := int64(0)
				ptrargs[i] = &integer
			default:
				return [][]interface{}{}, fmt.Errorf("Bad interface type: %s", t)
			}
		}
		err = rows.Scan(ptrargs...)
		if err != nil {
			return [][]interface{}{}, err
		}
		newargs := make([]interface{}, len(outargs))
		for i := range ptrargs {
			switch t := outargs[i].(type) {
			case string:
				newargs[i] = *ptrargs[i].(*string)
			case int:
				newargs[i] = *ptrargs[i].(*int)
			case int64:
				newargs[i] = *ptrargs[i].(*int64)
			default:
				return [][]interface{}{}, fmt.Errorf("Bad interface type: %s", t)
			}
		}
		result = append(result, newargs)
	}
	err = rows.Err()
	if err != nil {
		return [][]interface{}{}, err
	}
	return result, nil
}

/*
 * . q is the database query
 * . inargs is an array of interfaces containing the query arguments
 * . outfmt is an array of interfaces containing the right types of output
 *   arguments, i.e.
 *      var arg1 string
 *      var arg2 int
 *      outfmt := {}interface{}{arg1, arg2}
 *
 * The result will be an array (one per output row) of arrays (one per output argument)
 * of interfaces, containing pointers to the actual output arguments.
 */
func dbQueryScan(db *sql.DB, q string, inargs []interface{}, outfmt []interface{}) ([][]interface{}, error) {
	for i := 0; i < 100; i++ {
		result, err := doDbQueryScan(db, q, inargs, outfmt)
		if err == nil {
			return result, nil
		}
		if !isDbLockedError(err) {
			shared.LogDebugf("DbQuery: query %q error %q", q, err)
			return nil, err
		}
		time.Sleep(100 * time.Millisecond)
	}

	shared.LogDebugf("DbQueryscan: query %q inargs %q, DB still locked", q, inargs)
	shared.PrintStack()
	return nil, fmt.Errorf("DB is locked")
}

func dbExec(db *sql.DB, q string, args ...interface{}) (sql.Result, error) {
	for i := 0; i < 100; i++ {
		result, err := db.Exec(q, args...)
		if err == nil {
			return result, nil
		}
		if !isDbLockedError(err) {
			shared.LogDebugf("DbExec: query %q error %q", q, err)
			return nil, err
		}
		time.Sleep(100 * time.Millisecond)
	}

	shared.LogDebugf("DbExec: query %q args %q, DB still locked", q, args)
	shared.PrintStack()
	return nil, fmt.Errorf("DB is locked")
}
<|endoftext|>


<commit_before>package hypervisor

import (
	"net"
	"time"
)

const (
	StateStarting      = 0
	StateRunning       = 1
	StateFailedToStart = 2
	StateStopping      = 3
	StateStopped       = 4
	StateDestroying    = 5
)

type AcknowledgeVmRequest struct {
	IpAddress net.IP
}

type AcknowledgeVmResponse struct {
	Error string
}

type AddAddressesToPoolRequest struct {
	Addresses []Address
}

type AddAddressesToPoolResponse struct {
	Error string
}

type Address struct {
	IpAddress  net.IP `json:",omitempty"`
	MacAddress string
}

type AddSubnetsRequest struct {
	Subnets []Subnet
}

type AddSubnetsResponse struct {
	Error string
}

type ChangeVmOwnerUsersRequest struct {
	IpAddress  net.IP
	OwnerUsers []string
}

type ChangeVmOwnerUsersResponse struct {
	Error string
}

type ChangeVmTagsRequest struct {
	IpAddress net.IP
	Tags      map[string]string
}

type ChangeVmTagsResponse struct {
	Error string
}

type CreateVmRequest struct {
	DhcpTimeout      time.Duration
	ImageDataSize    uint64
	ImageTimeout     time.Duration
	MinimumFreeBytes uint64
	RoundupPower     uint64
	SecondaryVolumes []Volume
	UserDataSize     uint64
	VmInfo
} // RAW image data (length=ImageDataSize) and user data (length=UserDataSize)
// are streamed afterwards.

type CreateVmResponse struct { // Multiple responses are sent.
	DhcpTimedOut    bool
	Final           bool // If true, this is the final response.
	IpAddress       net.IP
	ProgressMessage string
	Error           string
}

type DestroyVmRequest struct {
	IpAddress net.IP
}

type DestroyVmResponse struct {
	Error string
}

type DiscardVmOldImageRequest struct {
	IpAddress net.IP
}

type DiscardVmOldImageResponse struct {
	Error string
}

type DiscardVmOldUserDataRequest struct {
	IpAddress net.IP
}

type DiscardVmOldUserDataResponse struct {
	Error string
}

// The GetUpdates() RPC is fully streamed.
// The client sends no information to the server.
// The server sends a stream of Update messages.

type Update struct {
	HaveAddressPool  bool               `json:",omitempty"`
	AddressPool      []Address          `json:",omitempty"` // Used & free.
	HaveNumFree      bool               `json:",omitempty"`
	NumFreeAddresses uint               `json:",omitempty"`
	HaveSubnets      bool               `json:",omitempty"`
	Subnets          []Subnet           `json:",omitempty"`
	VMs              map[string]*VmInfo `json:",omitempty"` // Key: IP address.
}

type GetVmInfoRequest struct {
	IpAddress net.IP
}

type GetVmInfoResponse struct {
	VmInfo VmInfo
	Error  string
}

type ListVMsRequest struct {
	Sort bool
}

type ListVMsResponse struct {
	IpAddresses []net.IP
}

type ProbeVmPortRequest struct {
	IpAddress  net.IP
	PortNumber uint
	Timeout    time.Duration
}

type ProbeVmPortResponse struct {
	PortIsOpen bool
	Error      string
}

type RemoveExcessAddressesFromPoolRequest struct {
	MaximumFreeAddresses uint
}

type RemoveExcessAddressesFromPoolResponse struct {
	Error string
}

type ReplaceVmImageRequest struct {
	DhcpTimeout      time.Duration
	ImageDataSize    uint64
	ImageName        string `json:",omitempty"`
	ImageTimeout     time.Duration
	ImageURL         string `json:",omitempty"`
	IpAddress        net.IP
	MinimumFreeBytes uint64
	RoundupPower     uint64
} // RAW image data (length=ImageDataSize) is streamed afterwards.

type ReplaceVmImageResponse struct { // Multiple responses are sent.
	DhcpTimedOut    bool
	Final           bool // If true, this is the final response.
	ProgressMessage string
	Error           string
}

type ReplaceVmUserDataRequest struct {
	IpAddress net.IP
	Size      uint64
} // User data (length=Size) are streamed afterwards.

type ReplaceVmUserDataResponse struct {
	Error string
}

type RestoreVmImageRequest struct {
	IpAddress net.IP
}

type RestoreVmImageResponse struct {
	Error string
}

type RestoreVmUserDataRequest struct {
	IpAddress net.IP
}

type RestoreVmUserDataResponse struct {
	Error string
}

type StartVmRequest struct {
	DhcpTimeout time.Duration
	IpAddress   net.IP
}

type StartVmResponse struct {
	DhcpTimedOut bool
	Error        string
}

type StopVmRequest struct {
	IpAddress net.IP
}

type StopVmResponse struct {
	Error string
}

type State uint

type Subnet struct {
	Id                string
	IpGateway         net.IP
	IpMask            net.IP // net.IPMask can't be JSON {en,de}coded.
	DomainNameServers []net.IP
}

type TraceVmMetadataRequest struct {
	IpAddress net.IP
}

type TraceVmMetadataResponse struct {
	Error string
} // A stream of strings (trace paths) follow.

type VmInfo struct {
	Address       Address
	ImageName     string `json:",omitempty"`
	ImageURL      string `json:",omitempty"`
	MemoryInMiB   uint64
	MilliCPUs     uint
	OwnerGroups   []string `json:",omitempty"`
	OwnerUsers    []string
	SpreadVolumes bool
	State         State
	Tags          map[string]string `json:",omitempty"`
	SubnetId      string
	Volumes       []Volume
}

type Volume struct {
	Size uint64
}
<commit_msg>Add proto/hypervisor.Update.HaveVMs field.<commit_after>package hypervisor

import (
	"net"
	"time"
)

const (
	StateStarting      = 0
	StateRunning       = 1
	StateFailedToStart = 2
	StateStopping      = 3
	StateStopped       = 4
	StateDestroying    = 5
)

type AcknowledgeVmRequest struct {
	IpAddress net.IP
}

type AcknowledgeVmResponse struct {
	Error string
}

type AddAddressesToPoolRequest struct {
	Addresses []Address
}

type AddAddressesToPoolResponse struct {
	Error string
}

type Address struct {
	IpAddress  net.IP `json:",omitempty"`
	MacAddress string
}

type AddSubnetsRequest struct {
	Subnets []Subnet
}

type AddSubnetsResponse struct {
	Error string
}

type ChangeVmOwnerUsersRequest struct {
	IpAddress  net.IP
	OwnerUsers []string
}

type ChangeVmOwnerUsersResponse struct {
	Error string
}

type ChangeVmTagsRequest struct {
	IpAddress net.IP
	Tags      map[string]string
}

type ChangeVmTagsResponse struct {
	Error string
}

type CreateVmRequest struct {
	DhcpTimeout      time.Duration
	ImageDataSize    uint64
	ImageTimeout     time.Duration
	MinimumFreeBytes uint64
	RoundupPower     uint64
	SecondaryVolumes []Volume
	UserDataSize     uint64
	VmInfo
} // RAW image data (length=ImageDataSize) and user data (length=UserDataSize)
// are streamed afterwards.

type CreateVmResponse struct { // Multiple responses are sent.
	DhcpTimedOut    bool
	Final           bool // If true, this is the final response.
	IpAddress       net.IP
	ProgressMessage string
	Error           string
}

type DestroyVmRequest struct {
	IpAddress net.IP
}

type DestroyVmResponse struct {
	Error string
}

type DiscardVmOldImageRequest struct {
	IpAddress net.IP
}

type DiscardVmOldImageResponse struct {
	Error string
}

type DiscardVmOldUserDataRequest struct {
	IpAddress net.IP
}

type DiscardVmOldUserDataResponse struct {
	Error string
}

// The GetUpdates() RPC is fully streamed.
// The client sends no information to the server.
// The server sends a stream of Update messages.

type Update struct {
	HaveAddressPool  bool               `json:",omitempty"`
	AddressPool      []Address          `json:",omitempty"` // Used & free.
	HaveNumFree      bool               `json:",omitempty"`
	NumFreeAddresses uint               `json:",omitempty"`
	HaveSubnets      bool               `json:",omitempty"`
	Subnets          []Subnet           `json:",omitempty"`
	HaveVMs          bool               `json:",omitempty"`
	VMs              map[string]*VmInfo `json:",omitempty"` // Key: IP address.
}

type GetVmInfoRequest struct {
	IpAddress net.IP
}

type GetVmInfoResponse struct {
	VmInfo VmInfo
	Error  string
}

type ListVMsRequest struct {
	Sort bool
}

type ListVMsResponse struct {
	IpAddresses []net.IP
}

type ProbeVmPortRequest struct {
	IpAddress  net.IP
	PortNumber uint
	Timeout    time.Duration
}

type ProbeVmPortResponse struct {
	PortIsOpen bool
	Error      string
}

type RemoveExcessAddressesFromPoolRequest struct {
	MaximumFreeAddresses uint
}

type RemoveExcessAddressesFromPoolResponse struct {
	Error string
}

type ReplaceVmImageRequest struct {
	DhcpTimeout      time.Duration
	ImageDataSize    uint64
	ImageName        string `json:",omitempty"`
	ImageTimeout     time.Duration
	ImageURL         string `json:",omitempty"`
	IpAddress        net.IP
	MinimumFreeBytes uint64
	RoundupPower     uint64
} // RAW image data (length=ImageDataSize) is streamed afterwards.

type ReplaceVmImageResponse struct { // Multiple responses are sent.
	DhcpTimedOut    bool
	Final           bool // If true, this is the final response.
	ProgressMessage string
	Error           string
}

type ReplaceVmUserDataRequest struct {
	IpAddress net.IP
	Size      uint64
} // User data (length=Size) are streamed afterwards.

type ReplaceVmUserDataResponse struct {
	Error string
}

type RestoreVmImageRequest struct {
	IpAddress net.IP
}

type RestoreVmImageResponse struct {
	Error string
}

type RestoreVmUserDataRequest struct {
	IpAddress net.IP
}

type RestoreVmUserDataResponse struct {
	Error string
}

type StartVmRequest struct {
	DhcpTimeout time.Duration
	IpAddress   net.IP
}

type StartVmResponse struct {
	DhcpTimedOut bool
	Error        string
}

type StopVmRequest struct {
	IpAddress net.IP
}

type StopVmResponse struct {
	Error string
}

type State uint

type Subnet struct {
	Id                string
	IpGateway         net.IP
	IpMask            net.IP // net.IPMask can't be JSON {en,de}coded.
	DomainNameServers []net.IP
}

type TraceVmMetadataRequest struct {
	IpAddress net.IP
}

type TraceVmMetadataResponse struct {
	Error string
} // A stream of strings (trace paths) follow.

type VmInfo struct {
	Address       Address
	ImageName     string `json:",omitempty"`
	ImageURL      string `json:",omitempty"`
	MemoryInMiB   uint64
	MilliCPUs     uint
	OwnerGroups   []string `json:",omitempty"`
	OwnerUsers    []string
	SpreadVolumes bool
	State         State
	Tags          map[string]string `json:",omitempty"`
	SubnetId      string
	Volumes       []Volume
}

type Volume struct {
	Size uint64
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"net/http"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Networking", func() {
	var c *client.Client

	BeforeEach(func() {
		var err error
		c, err = loadClient()
		Expect(err).NotTo(HaveOccurred())
	})

	// Create a unique namespace for this test.
	ns := "nettest-" + randomSuffix()

	It("should function for pods", func() {
		if testContext.provider == "vagrant" {
			By("Skipping test which is broken for vagrant (See https://github.com/GoogleCloudPlatform/kubernetes/issues/3580)")
			return
		}

		// Obtain a list of nodes so we can place one webserver container on each node.
		nodes, err := c.Nodes().List()
		if err != nil {
			Failf("Failed to list nodes: %v", err)
		}
		peers := len(nodes.Items)
		if peers == 0 {
			Failf("Failed to find any nodes")
		}

		// Test basic external connectivity.
		resp, err := http.Get("http://google.com/")
		if err != nil {
			Failf("unable to talk to the external internet: %v", err)
		}
		if resp.StatusCode != http.StatusOK {
			Failf("unexpected error code. expected 200, got: %v (%v)", resp.StatusCode, resp)
		}

		name := "nettest"

		By(fmt.Sprintf("Creating service with name %s in namespace %s", name, ns))
		svc, err := c.Services(ns).Create(&api.Service{
			ObjectMeta: api.ObjectMeta{
				Name: name,
				Labels: map[string]string{
					"name": name,
				},
			},
			Spec: api.ServiceSpec{
				Port:       8080,
				TargetPort: util.NewIntOrStringFromInt(8080),
				Selector: map[string]string{
					"name": name,
				},
			},
		})
		if err != nil {
			Failf("unable to create test service %s: %v", svc.Name, err)
		}
		// Clean up service
		defer func() {
			defer GinkgoRecover()
			By("Cleaning up the service")
			if err = c.Services(ns).Delete(svc.Name); err != nil {
				Failf("unable to delete svc %v: %v", svc.Name, err)
			}
		}()

		By("Creating a webserver pod on each node")
		podNames := []string{}
		for i, node := range nodes.Items {
			podName := fmt.Sprintf("%s-%d", name, i)
			podNames = append(podNames, podName)
			Logf("Creating pod %s on node %s", podName, node.Name)
			_, err := c.Pods(ns).Create(&api.Pod{
				ObjectMeta: api.ObjectMeta{
					Name: podName,
					Labels: map[string]string{
						"name": name,
					},
				},
				Spec: api.PodSpec{
					Containers: []api.Container{
						{
							Name:  "webserver",
							Image: "kubernetes/nettest:1.1",
							Command: []string{
								"-service=" + name,
								fmt.Sprintf("-peers=%d", peers),
								"-namespace=" + ns},
							Ports: []api.ContainerPort{{ContainerPort: 8080}},
						},
					},
					Host:          node.Name,
					RestartPolicy: api.RestartPolicyNever,
				},
			})
			Expect(err).NotTo(HaveOccurred())
		}
		// Clean up the pods
		defer func() {
			defer GinkgoRecover()
			By("Cleaning up the webserver pods")
			for _, podName := range podNames {
				if err = c.Pods(ns).Delete(podName); err != nil {
					Logf("Failed to delete pod %s: %v", podName, err)
				}
			}
		}()

		By("Wait for the webserver pods to be ready")
		for _, podName := range podNames {
			err = waitForPodRunningInNamespace(c, podName, ns)
			Expect(err).NotTo(HaveOccurred())
		}

		By("Waiting for connectivity to be verified")
		const maxAttempts = 60
		passed := false
		var body []byte
		for i := 0; i < maxAttempts && !passed; i++ {
			time.Sleep(2 * time.Second)
			body, err = c.Get().
				Namespace(ns).
				Prefix("proxy").
				Resource("services").
				Name(svc.Name).
				Suffix("status").
				Do().Raw()
			if err != nil {
				Logf("Attempt %v/%v: service/pod still starting. (error: '%v')", i, maxAttempts, err)
				continue
			}
			switch string(body) {
			case "pass":
				Logf("Passed on attempt %v. Cleaning up.", i)
				passed = true
				break
			case "running":
				Logf("Attempt %v/%v: test still running", i, maxAttempts)
				break
			case "fail":
				if body, err = c.Get().
					Namespace(ns).Prefix("proxy").
					Resource("services").
					Name(svc.Name).Suffix("read").
					Do().Raw(); err != nil {
					Failf("Failed on attempt %v. Cleaning up. Error reading details: %v", i, err)
				} else {
					Failf("Failed on attempt %v. Cleaning up. Details:\n%s", i, string(body))
				}
				break
			}
		}

		if !passed {
			if body, err = c.Get().
				Namespace(ns).
				Prefix("proxy").
				Resource("services").
				Name(svc.Name).
				Suffix("read").
				Do().Raw(); err != nil {
				Failf("Timed out. Cleaning up. Error reading details: %v", err)
			} else {
				Failf("Timed out. Cleaning up. Details:\n%s", string(body))
			}
		}
		Expect(string(body)).To(Equal("pass"))
	})

	It("should provide unchanging URLs", func() {
		tests := []struct {
			path string
		}{
			{path: "/validate"},
			{path: "/healthz"},
			// TODO: test proxy links here
		}
		for _, test := range tests {
			By(fmt.Sprintf("testing: %s", test.path))
			data, err := c.RESTClient.Get().
				Namespace(ns).
				AbsPath(test.path).
				Do().Raw()
			if err != nil {
				Failf("Failed: %v\nBody: %s", err, string(data))
			}
		}
	})
})
<commit_msg>Add extra diagnostics to networking e2e test<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"net/http"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Networking", func() {
	var c *client.Client

	BeforeEach(func() {
		var err error
		c, err = loadClient()
		Expect(err).NotTo(HaveOccurred())
	})

	// Create a unique namespace for this test.
	ns := "nettest-" + randomSuffix()

	It("should function for pods", func() {
		if testContext.provider == "vagrant" {
			By("Skipping test which is broken for vagrant (See https://github.com/GoogleCloudPlatform/kubernetes/issues/3580)")
			return
		}

		// Obtain a list of nodes so we can place one webserver container on each node.
		nodes, err := c.Nodes().List()
		if err != nil {
			Failf("Failed to list nodes: %v", err)
		}
		peers := len(nodes.Items)
		if peers == 0 {
			Failf("Failed to find any nodes")
		}

		// Test basic external connectivity.
		resp, err := http.Get("http://google.com/")
		if err != nil {
			Failf("unable to talk to the external internet: %v", err)
		}
		if resp.StatusCode != http.StatusOK {
			Failf("unexpected error code. expected 200, got: %v (%v)", resp.StatusCode, resp)
		}

		name := "nettest"

		By(fmt.Sprintf("Creating service with name %s in namespace %s", name, ns))
		svc, err := c.Services(ns).Create(&api.Service{
			ObjectMeta: api.ObjectMeta{
				Name: name,
				Labels: map[string]string{
					"name": name,
				},
			},
			Spec: api.ServiceSpec{
				Port:       8080,
				TargetPort: util.NewIntOrStringFromInt(8080),
				Selector: map[string]string{
					"name": name,
				},
			},
		})
		if err != nil {
			Failf("unable to create test service %s: %v", svc.Name, err)
		}
		// Clean up service
		defer func() {
			defer GinkgoRecover()
			By("Cleaning up the service")
			if err = c.Services(ns).Delete(svc.Name); err != nil {
				Failf("unable to delete svc %v: %v", svc.Name, err)
			}
		}()

		By("Creating a webserver pod on each node")
		podNames := []string{}
		for i, node := range nodes.Items {
			podName := fmt.Sprintf("%s-%d", name, i)
			podNames = append(podNames, podName)
			Logf("Creating pod %s on node %s", podName, node.Name)
			_, err := c.Pods(ns).Create(&api.Pod{
				ObjectMeta: api.ObjectMeta{
					Name: podName,
					Labels: map[string]string{
						"name": name,
					},
				},
				Spec: api.PodSpec{
					Containers: []api.Container{
						{
							Name:  "webserver",
							Image: "kubernetes/nettest:1.1",
							Command: []string{
								"-service=" + name,
								fmt.Sprintf("-peers=%d", peers),
								"-namespace=" + ns},
							Ports: []api.ContainerPort{{ContainerPort: 8080}},
						},
					},
					Host:          node.Name,
					RestartPolicy: api.RestartPolicyNever,
				},
			})
			Expect(err).NotTo(HaveOccurred())
		}
		// Clean up the pods
		defer func() {
			defer GinkgoRecover()
			By("Cleaning up the webserver pods")
			for _, podName := range podNames {
				if err = c.Pods(ns).Delete(podName); err != nil {
					Logf("Failed to delete pod %s: %v", podName, err)
				}
			}
		}()

		By("Wait for the webserver pods to be ready")
		for _, podName := range podNames {
			err = waitForPodRunningInNamespace(c, podName, ns)
			Expect(err).NotTo(HaveOccurred())
		}

		By("Waiting for connectivity to be verified")
		const maxAttempts = 60
		passed := false
		var body []byte
		for i := 0; i < maxAttempts && !passed; i++ {
			time.Sleep(2 * time.Second)
			Logf("About to make a proxy status call")
			start := time.Now()
			body, err = c.Get().
				Namespace(ns).
				Prefix("proxy").
				Resource("services").
				Name(svc.Name).
				Suffix("status").
				DoRaw()
			Logf("Proxy status call returned in %v", time.Since(start))
			if err != nil {
				Logf("Attempt %v/%v: service/pod still starting. (error: '%v')", i, maxAttempts, err)
				continue
			}
			switch string(body) {
			case "pass":
				Logf("Passed on attempt %v. Cleaning up.", i)
				passed = true
				break
			case "running":
				Logf("Attempt %v/%v: test still running", i, maxAttempts)
				break
			case "fail":
				if body, err = c.Get().
					Namespace(ns).Prefix("proxy").
					Resource("services").
					Name(svc.Name).Suffix("read").
					DoRaw(); err != nil {
					Failf("Failed on attempt %v. Cleaning up. Error reading details: %v", i, err)
				} else {
					Failf("Failed on attempt %v. Cleaning up. Details:\n%s", i, string(body))
				}
				break
			}
		}

		if !passed {
			if body, err = c.Get().
				Namespace(ns).
				Prefix("proxy").
				Resource("services").
				Name(svc.Name).
				Suffix("read").
				DoRaw(); err != nil {
				Failf("Timed out. Cleaning up. Error reading details: %v", err)
			} else {
				Failf("Timed out. Cleaning up. Details:\n%s", string(body))
			}
		}
		Expect(string(body)).To(Equal("pass"))
	})

	It("should provide unchanging URLs", func() {
		tests := []struct {
			path string
		}{
			{path: "/validate"},
			{path: "/healthz"},
			// TODO: test proxy links here
		}
		for _, test := range tests {
			By(fmt.Sprintf("testing: %s", test.path))
			data, err := c.RESTClient.Get().
				Namespace(ns).
				AbsPath(test.path).
				DoRaw()
			if err != nil {
				Failf("Failed: %v\nBody: %s", err, string(data))
			}
		}
	})
})
<|endoftext|>