<commit_before>package graph

import (
	"fmt"
	"io"
	"strings"
	"strconv"
)

func (node VertexId) String() string {
	return strconv.Itoa(int(uint(node)))
}

func (conn Connection) String() string {
	return fmt.Sprintf("%v->%v", conn.Tail, conn.Head)
}

// Typed connection as a string.
//
// Format:
//  * 1--2 for undirected connection
//  * 1->2 for directed connection
//  * 1<-2 for reversed directed connection
//  * 1><2 if there is no connection between 1 and 2
//  * 1!!2 for unexpected error (if function can't recognize connection type)
func (conn TypedConnection) String() string {
	switch conn.Type {
		case CT_UNDIRECTED:
			return fmt.Sprintf("%v--%v", conn.Tail, conn.Head)
		case CT_DIRECTED:
			return fmt.Sprintf("%v->%v", conn.Tail, conn.Head)
		case CT_DIRECTED_REVERSED:
			return fmt.Sprintf("%v<-%v", conn.Tail, conn.Head)
		case CT_NONE:
			return fmt.Sprintf("%v><%v", conn.Tail, conn.Head)
	}
	return fmt.Sprintf("%v!!%v", conn.Tail, conn.Head)
}

func styleMapToString(style map[string]string) string {
	chunks := make([]string, len(style))
	i := 0
	for k, v := range style {
		chunks[i] = fmt.Sprintf("%v=\"%v\"", k, v)
		i++
	}
	return "[" + strings.Join(chunks, ",") + "]"
}

type DotNodeStyleFunc func(node VertexId) map[string]string
type DotConnectionStyleFunc func(conn TypedConnection) map[string]string

// Basic dot style function for vertexes.
//
// Generates only one style property: vertex label, which match vertex id.
func SimpleNodeStyle(node VertexId) map[string]string {
	style := make(map[string]string)
	style["label"] = node.String()
	return style
}

// Basic dot style function for connections.
//
// Empty style, except undirected connection. For undirected connection
// "dir=both" style is set.
func SimpleConnectionStyle(conn TypedConnection) map[string]string {
	res := make(map[string]string)
	if conn.Type==CT_UNDIRECTED {
		// for undirected connection by default set "both" direction
		res["dir"] = "both"
	}
	return res
}

// Plot graph vertexes to dot format.
//
// nodesIter -- iterable object over graph vertexes
// wr -- writer interface
// styleFunc -- style function for vertexes. For example, see SimpleNodeStyle()
func PlotVertexesToDot(nodesIter VertexesIterable, wr io.Writer, styleFunc DotNodeStyleFunc) {
	if styleFunc==nil {
		styleFunc = SimpleNodeStyle
	}
	for node := range nodesIter.VertexesIter() {
		wr.Write([]byte("n" + node.String() + styleMapToString(styleFunc(node)) + ";\n"))
	}
}

// Plot graph connections to dot format.
//
// connIter -- iterable object over graph connections
// wr -- writer interface
// styleFunc -- style function for typed connections. For example, see SimpleConnectionStyle()
func PlotConnectionsToDot(connIter TypedConnectionsIterable, wr io.Writer, styleFunc DotConnectionStyleFunc) {
	if styleFunc==nil {
		styleFunc = SimpleConnectionStyle
	}
	for conn := range connIter.TypedConnectionsIter() {
		wr.Write([]byte(fmt.Sprintf("n%v->n%v%v;\n",
			conn.Tail.String(),
			conn.Head.String(),
			styleMapToString(styleFunc(conn)))))
	}
}

// Plot directed graph to dot format.
//
// gr -- directed graph interface
// wr -- writer interface
// nodeStyleFunc -- style function for vertexes. For example, see SimpleNodeStyle()
// connStyleFunc -- style function for typed connections. For example, see SimpleConnectionStyle()
func PlotDgraphToDot(gr DirectedGraphReader, wr io.Writer, nodeStyleFunc DotNodeStyleFunc, connStyleFunc DotConnectionStyleFunc) {
	wr.Write([]byte("digraph messages {\n"))
	PlotVertexesToDot(gr, wr, nodeStyleFunc)
	PlotConnectionsToDot(ArcsToTypedConnIterable(gr), wr, connStyleFunc)
	wr.Write([]byte("}\n"))
}

// Plot mixed graph to dot format.
//
// gr -- directed graph interface
// wr -- writer interface
// nodeStyleFunc -- style function for vertexes. For example, see SimpleNodeStyle()
// connStyleFunc -- style function for typed connections. For example, see SimpleConnectionStyle()
func PlotMgraphToDot(gr MixedGraphReader, wr io.Writer, nodeStyleFunc DotNodeStyleFunc, connStyleFunc DotConnectionStyleFunc) {
	wr.Write([]byte("digraph messages {\n"))
	PlotVertexesToDot(gr, wr, nodeStyleFunc)
	PlotConnectionsToDot(gr, wr, connStyleFunc)
	wr.Write([]byte("}\n"))
}

// Plot undirected graph to dot format.
//
// gr -- directed graph interface
// wr -- writer interface
// nodeStyleFunc -- style function for vertexes. For example, see SimpleNodeStyle()
// connStyleFunc -- style function for typed connections. For example, see SimpleConnectionStyle()
func PlotUgraphToDot(gr UndirectedGraphReader, wr io.Writer, nodeStyleFunc DotNodeStyleFunc, connStyleFunc DotConnectionStyleFunc) {
	wr.Write([]byte("graph messages {\n"))
	PlotVertexesToDot(gr, wr, nodeStyleFunc)
	PlotConnectionsToDot(EdgesToTypedConnIterable(gr), wr, connStyleFunc)
	wr.Write([]byte("}\n"))
}
<commit_msg>fixed graphviz output to dot format<commit_after>package graph

import (
	"fmt"
	"io"
	"strings"
	"strconv"
)

func (node VertexId) String() string {
	return strconv.Itoa(int(uint(node)))
}

func (conn Connection) String() string {
	return fmt.Sprintf("%v->%v", conn.Tail, conn.Head)
}

// Typed connection as a string.
//
// Format:
//  * 1--2 for undirected connection
//  * 1->2 for directed connection
//  * 1<-2 for reversed directed connection
//  * 1><2 if there is no connection between 1 and 2
//  * 1!!2 for unexpected error (if function can't recognize connection type)
func (conn TypedConnection) String() string {
	switch conn.Type {
		case CT_UNDIRECTED:
			return fmt.Sprintf("%v--%v", conn.Tail, conn.Head)
		case CT_DIRECTED:
			return fmt.Sprintf("%v->%v", conn.Tail, conn.Head)
		case CT_DIRECTED_REVERSED:
			return fmt.Sprintf("%v<-%v", conn.Tail, conn.Head)
		case CT_NONE:
			return fmt.Sprintf("%v><%v", conn.Tail, conn.Head)
	}
	return fmt.Sprintf("%v!!%v", conn.Tail, conn.Head)
}

func styleMapToString(style map[string]string) string {
	chunks := make([]string, len(style))
	i := 0
	for k, v := range style {
		chunks[i] = fmt.Sprintf("%v=\"%v\"", k, v)
		i++
	}
	return "[" + strings.Join(chunks, ",") + "]"
}

type DotNodeStyleFunc func(node VertexId) map[string]string
type DotConnectionStyleFunc func(conn TypedConnection) map[string]string

// Basic dot style function for vertexes.
//
// Generates only one style property: vertex label, which match vertex id.
func SimpleNodeStyle(node VertexId) map[string]string {
	style := make(map[string]string)
	style["label"] = node.String()
	return style
}

// Basic dot style function for connections.
//
// Empty style, except undirected connection. For undirected connection
// "dir=both" style is set.
func SimpleConnectionStyle(conn TypedConnection) map[string]string {
	res := make(map[string]string)
	if conn.Type==CT_UNDIRECTED {
		// for undirected connection by default set "both" direction
		res["dir"] = "both"
	}
	return res
}

// Plot graph vertexes to dot format.
//
// nodesIter -- iterable object over graph vertexes
// wr -- writer interface
// styleFunc -- style function for vertexes. For example, see SimpleNodeStyle()
func PlotVertexesToDot(nodesIter VertexesIterable, wr io.Writer, styleFunc DotNodeStyleFunc) {
	if styleFunc==nil {
		styleFunc = SimpleNodeStyle
	}
	for node := range nodesIter.VertexesIter() {
		wr.Write([]byte("n" + node.String() + styleMapToString(styleFunc(node)) + ";\n"))
	}
}

// Plot graph connections to dot format.
//
// connIter -- iterable object over graph connections
// separator -- string betweet two vertexes. "->" for digraph and "--" for graph in graphviz file format
// wr -- writer interface
// styleFunc -- style function for typed connections. For example, see SimpleConnectionStyle()
func PlotConnectionsToDot(connIter TypedConnectionsIterable, separator string, wr io.Writer, styleFunc DotConnectionStyleFunc) {
	if styleFunc==nil {
		styleFunc = SimpleConnectionStyle
	}
	for conn := range connIter.TypedConnectionsIter() {
		wr.Write([]byte(fmt.Sprintf("n%v" + separator + "n%v%v;\n",
			conn.Tail.String(),
			conn.Head.String(),
			styleMapToString(styleFunc(conn)))))
	}
}

// Plot directed graph to dot format.
//
// gr -- directed graph interface
// wr -- writer interface
// nodeStyleFunc -- style function for vertexes. For example, see SimpleNodeStyle()
// connStyleFunc -- style function for typed connections. For example, see SimpleConnectionStyle()
func PlotDgraphToDot(gr DirectedGraphReader, wr io.Writer, nodeStyleFunc DotNodeStyleFunc, connStyleFunc DotConnectionStyleFunc) {
	wr.Write([]byte("digraph messages {\n"))
	PlotVertexesToDot(gr, wr, nodeStyleFunc)
	PlotConnectionsToDot(ArcsToTypedConnIterable(gr), "->", wr, connStyleFunc)
	wr.Write([]byte("}\n"))
}

// Plot mixed graph to dot format.
//
// gr -- directed graph interface
// wr -- writer interface
// nodeStyleFunc -- style function for vertexes. For example, see SimpleNodeStyle()
// connStyleFunc -- style function for typed connections. For example, see SimpleConnectionStyle()
func PlotMgraphToDot(gr MixedGraphReader, wr io.Writer, nodeStyleFunc DotNodeStyleFunc, connStyleFunc DotConnectionStyleFunc) {
	wr.Write([]byte("digraph messages {\n"))
	PlotVertexesToDot(gr, wr, nodeStyleFunc)
	PlotConnectionsToDot(gr, "->", wr, connStyleFunc)
	wr.Write([]byte("}\n"))
}

// Plot undirected graph to dot format.
//
// gr -- directed graph interface
// wr -- writer interface
// nodeStyleFunc -- style function for vertexes. For example, see SimpleNodeStyle()
// connStyleFunc -- style function for typed connections. For example, see SimpleConnectionStyle()
func PlotUgraphToDot(gr UndirectedGraphReader, wr io.Writer, nodeStyleFunc DotNodeStyleFunc, connStyleFunc DotConnectionStyleFunc) {
	wr.Write([]byte("graph messages {\n"))
	PlotVertexesToDot(gr, wr, nodeStyleFunc)
	PlotConnectionsToDot(EdgesToTypedConnIterable(gr), "--", wr, connStyleFunc)
	wr.Write([]byte("}\n"))
}
<|endoftext|>


<commit_before>package algoliasearch

import (
	"bytes"
	_ "crypto/sha512" // Fix certificates
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"
)

const (
	version = "2.6.0"
)

// Define the constants used to specify the type of request.
const (
	search = 1 << iota
	write
	read
)

// Seed the RNG used to shuffle the hosts slice (see `defaultHosts` function).
func init() {
	rand.Seed(int64(time.Now().Nanosecond()))
}

// Transport is responsible for the connection and the retry strategy to
// Algolia servers.
type Transport struct {
	activeHost        string
	activeSince       time.Time
	apiKey            string
	appId             string
	dialTimeout       time.Duration
	headers           map[string]string
	hosts             []string
	httpClient        *http.Client
	keepAliveDuration time.Duration
}

// NewTransport instantiates a new Transport with the default Algolia hosts to
// connect to.
func NewTransport(appId, apiKey string) *Transport {
	return &Transport{
		activeHost:        "",
		apiKey:            apiKey,
		appId:             appId,
		dialTimeout:       1 * time.Second,
		headers:           defaultHeaders(appId, apiKey),
		hosts:             defaultHosts(appId),
		httpClient:        defaultHttpClient(),
		keepAliveDuration: 5 * 60 * time.Second,
	}
}

// NewTransport instantiates a new Transport with the specificed hosts as main
// servers to connect to.
func NewTransportWithHosts(appId, apiKey string, hosts []string) *Transport {
	return &Transport{
		activeHost:        "",
		apiKey:            apiKey,
		appId:             appId,
		dialTimeout:       1 * time.Second,
		headers:           defaultHeaders(appId, apiKey),
		hosts:             hosts,
		httpClient:        defaultHttpClient(),
		keepAliveDuration: 5 * 60 * time.Second,
	}
}

// defaultHeaders is used to set the default HTTP headers to use with each
// requests.
func defaultHeaders(appId, apiKey string) map[string]string {
	return map[string]string{
		"Connection":               "keep-alive",
		"User-Agent":               "Algolia for Go (" + version + ")",
		"X-Algolia-API-Key":        apiKey,
		"X-Algolia-Application-Id": appId,
	}
}

// defaultHosts returns the list of the default Algolia hosts to use. The
// entries are shuffled.
func defaultHosts(appId string) []string {
	hosts := []string{
		appId + "-1.algolianet.com",
		appId + "-2.algolianet.com",
		appId + "-3.algolianet.com",
	}

	shuffled := make([]string, len(hosts))
	for i, v := range rand.Perm(len(hosts)) {
		shuffled[i] = hosts[v]
	}

	return shuffled
}

// defaultHttpClient returns the `*http.Client` which will perform all the
// requests. All the timeout settings are explicitely defined here.
func defaultHttpClient() *http.Client {
	return &http.Client{
		Timeout:   time.Second * 30,
		Transport: defaultTransport(1 * time.Second),
	}
}

// defaultTransport returns the `*http.Transport` which starts and maintain the
// connection with the server. The `dialTimeout` is used to specify the timeout
// beyond which the connection is considered as failed (used to control DNS
// lookup timeouts).
func defaultTransport(dialTimeout time.Duration) *http.Transport {
	return &http.Transport{
		Dial: (&net.Dialer{
			KeepAlive: 180 * time.Second,
			Timeout:   dialTimeout,
		}).Dial,
		DisableKeepAlives:   false,
		MaxIdleConnsPerHost: 2,
		TLSHandshakeTimeout: 2 * time.Second,
	}
}

// addHeaders add the key/value pairs from `headers` to the header list of the
// `req` request.
func addHeaders(req *http.Request, headers map[string]string) {
	for k, v := range headers {
		req.Header.Add(k, v)
	}
}

// setExtraHeader lets the user (through the exported `Client.SetExtraHeader`)
// add custom headers to the requests.
func (t *Transport) setExtraHeader(key, value string) {
	t.headers[key] = value
}

// setTimeout lets the user (through the exported `Client.SetTimeout`) replace
// the default values of `TLSHandshakeTimeout` (via `connectTimeout`) and
// `ResponseHeaderTimeout` (via `readTimeout`).
func (t *Transport) setTimeout(connectTimeout, readTimeout time.Duration) {
	switch transport := t.httpClient.Transport.(type) {
	case *http.Transport:
		transport.TLSHandshakeTimeout = connectTimeout
		transport.ResponseHeaderTimeout = readTimeout
	default:
		fmt.Fprintln(os.Stderr, "Timeouts not set for nonstandard underlying Transport")
	}
}

// request is the method used by the `Client` to perform the request against
// the Algolia servers (or to the list of specified hosts).
func (t *Transport) request(method, path string, body interface{}, typeCall int) ([]byte, error) {
	var res []byte
	var err error

	for _, host := range t.hostsToTry(typeCall) {
		res, err = t.tryRequest(method, host, path, body)
		if err == nil {
			t.resetDialTimeout()
			t.activeSince = time.Now()
			t.activeHost = host
			return res, nil
		}
		t.increaseDialTimeout()
	}

	t.activeHost = ""
	return nil, err
}

// hostsToTry returns the list of hosts to try ordered by priority according to
// the type of request (write vs. read/search) and if a previous host was
// marked as active.
func (t *Transport) hostsToTry(typeCall int) []string {
	// If it's a `write` request, we first try `APPID.algolia.net` and then the
	// `APPID-{1,2,3}.algolianet.com` endpoints (shuffled).
	if typeCall == write {
		return append([]string{t.appId + ".algolia.net"}, t.hosts...)
	}

	var hosts []string

	// If a previous host was active since less than k.keepAliveDuration
	// seconds and was not the `APPID-dsn.algolia.net` endpoint, then it is
	// used first in the list of endpoints to try.
	if t.activeHost != "" &&
		!strings.Contains(t.activeHost, "-dsn.algolia.net") &&
		time.Now().Sub(t.activeSince) <= t.keepAliveDuration {
		hosts = []string{t.activeHost}
	}

	// Whether or not an active host is first in the list of hosts to try, the
	// following entries in the list is the DSN endpoint followed by the
	// `APPID-{1,2,3}.algolianet.com` endpoints (shuffled).
	hosts = append(hosts, t.appId+"-dsn.algolia.net")
	hosts = append(hosts, t.hosts...)
	return hosts

}

// tryRequest is the underlying method which actually performs the request. It
// returns the response as a byte slice or a non-nil error if anything went
// wrong.
func (t *Transport) tryRequest(method, host, path string, body interface{}) ([]byte, error) {
	// Build the request
	req, err := t.buildRequest(method, host, path, body)
	if err != nil {
		return nil, err
	}

	// Perform the request
	res, err := t.httpClient.Do(req)
	if err != nil {
		return nil, fmt.Errorf("Cannot perform request [%s] %s (%s): %s", method, path, host, err)
	}

	// Read response's body
	bodyRes, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, fmt.Errorf("Cannot read response body: %s", err)
	}
	res.Body.Close()

	// Return the body as an error if the status code is not 2XX
	code := res.StatusCode
	if !(200 <= code && code < 300) {
		return nil, errors.New(string(bodyRes))
	}

	return bodyRes, nil
}

// buildRequest returns a valid `http.Request` with the headers and body (if
// any) correctly set. The return error is non-nil if the request is invalid or
// if the body, if non-nil, is not a valid JSON.
func (t *Transport) buildRequest(method, host, path string, body interface{}) (*http.Request, error) {
	var req *http.Request
	var err error
	urlStr := "https://" + host + path

	if body == nil {
		// As the body is nil, an empty body request is instantiated
		req, err = http.NewRequest(method, urlStr, nil)
		if err != nil {
			return nil, fmt.Errorf("Cannot instantiate request: [%s] %s", method, urlStr)
		}
	} else {
		// As the body is non-nil, the content is read
		data, err := json.Marshal(body)
		if err != nil {
			return nil, errors.New("Invalid JSON in the query")
		}
		reader := bytes.NewReader(data)

		// The request is then instantiated with the body content
		req, err = http.NewRequest(method, urlStr, reader)
		if err != nil {
			return nil, fmt.Errorf("Cannot instantiate request: [%s] %s", method, urlStr)
		}

		// Add content specific headers
		req.Header.Add("Content-Length", strconv.Itoa(len(string(data))))
		req.Header.Add("Content-Type", "application/json; charset=utf-8")
	}

	// Add default and Algolia specific headers
	addHeaders(req, t.headers)

	if strings.Contains(path, "/*/") {
		req.URL = &url.URL{
			Scheme: "https",
			Host:   host,
			Opaque: "//" + host + path, //Remove url encoding
		}
	}

	return req, nil
}

// resetDialTimeout increases the `Timeout` value of the underlying dialer by 1
// second.
func (t *Transport) increaseDialTimeout() {
	t.dialTimeout = t.dialTimeout + time.Second
	t.httpClient.Transport = defaultTransport(t.dialTimeout)
}

// resetDialTimeout resets the `Timeout` value of the underlying dialer to 1
// second.
func (t *Transport) resetDialTimeout() {
	t.dialTimeout = 1 * time.Second
	t.httpClient.Transport = defaultTransport(t.dialTimeout)
}
<commit_msg>fix: Use the same retry strategy for both write and read requests<commit_after>package algoliasearch

import (
	"bytes"
	_ "crypto/sha512" // Fix certificates
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"
)

const (
	version = "2.6.0"
)

// Define the constants used to specify the type of request.
const (
	search = 1 << iota
	write
	read
)

// Seed the RNG used to shuffle the hosts slice (see `defaultHosts` function).
func init() {
	rand.Seed(int64(time.Now().Nanosecond()))
}

// Transport is responsible for the connection and the retry strategy to
// Algolia servers.
type Transport struct {
	activeReadHost    string
	activeReadSince   time.Time
	activeWriteHost   string
	activeWriteSince  time.Time
	apiKey            string
	appId             string
	dialTimeout       time.Duration
	headers           map[string]string
	hosts             []string
	httpClient        *http.Client
	keepAliveDuration time.Duration
}

// NewTransport instantiates a new Transport with the default Algolia hosts to
// connect to.
func NewTransport(appId, apiKey string) *Transport {
	return &Transport{
		activeReadHost:    "",
		activeWriteHost:   "",
		apiKey:            apiKey,
		appId:             appId,
		dialTimeout:       1 * time.Second,
		headers:           defaultHeaders(appId, apiKey),
		hosts:             defaultHosts(appId),
		httpClient:        defaultHttpClient(),
		keepAliveDuration: 5 * 60 * time.Second,
	}
}

// NewTransport instantiates a new Transport with the specificed hosts as main
// servers to connect to.
func NewTransportWithHosts(appId, apiKey string, hosts []string) *Transport {
	return &Transport{
		activeReadHost:    "",
		activeWriteHost:   "",
		apiKey:            apiKey,
		appId:             appId,
		dialTimeout:       1 * time.Second,
		headers:           defaultHeaders(appId, apiKey),
		hosts:             hosts,
		httpClient:        defaultHttpClient(),
		keepAliveDuration: 5 * 60 * time.Second,
	}
}

// defaultHeaders is used to set the default HTTP headers to use with each
// requests.
func defaultHeaders(appId, apiKey string) map[string]string {
	return map[string]string{
		"Connection":               "keep-alive",
		"User-Agent":               "Algolia for Go (" + version + ")",
		"X-Algolia-API-Key":        apiKey,
		"X-Algolia-Application-Id": appId,
	}
}

// defaultHosts returns the list of the default Algolia hosts to use. The
// entries are shuffled.
func defaultHosts(appId string) []string {
	hosts := []string{
		appId + "-1.algolianet.com",
		appId + "-2.algolianet.com",
		appId + "-3.algolianet.com",
	}

	shuffled := make([]string, len(hosts))
	for i, v := range rand.Perm(len(hosts)) {
		shuffled[i] = hosts[v]
	}

	return shuffled
}

// defaultHttpClient returns the `*http.Client` which will perform all the
// requests. All the timeout settings are explicitely defined here.
func defaultHttpClient() *http.Client {
	return &http.Client{
		Timeout:   time.Second * 30,
		Transport: defaultTransport(1 * time.Second),
	}
}

// defaultTransport returns the `*http.Transport` which starts and maintain the
// connection with the server. The `dialTimeout` is used to specify the timeout
// beyond which the connection is considered as failed (used to control DNS
// lookup timeouts).
func defaultTransport(dialTimeout time.Duration) *http.Transport {
	return &http.Transport{
		Dial: (&net.Dialer{
			KeepAlive: 180 * time.Second,
			Timeout:   dialTimeout,
		}).Dial,
		DisableKeepAlives:   false,
		MaxIdleConnsPerHost: 2,
		TLSHandshakeTimeout: 2 * time.Second,
	}
}

// addHeaders add the key/value pairs from `headers` to the header list of the
// `req` request.
func addHeaders(req *http.Request, headers map[string]string) {
	for k, v := range headers {
		req.Header.Add(k, v)
	}
}

// setExtraHeader lets the user (through the exported `Client.SetExtraHeader`)
// add custom headers to the requests.
func (t *Transport) setExtraHeader(key, value string) {
	t.headers[key] = value
}

// setTimeout lets the user (through the exported `Client.SetTimeout`) replace
// the default values of `TLSHandshakeTimeout` (via `connectTimeout`) and
// `ResponseHeaderTimeout` (via `readTimeout`).
func (t *Transport) setTimeout(connectTimeout, readTimeout time.Duration) {
	switch transport := t.httpClient.Transport.(type) {
	case *http.Transport:
		transport.TLSHandshakeTimeout = connectTimeout
		transport.ResponseHeaderTimeout = readTimeout
	default:
		fmt.Fprintln(os.Stderr, "Timeouts not set for nonstandard underlying Transport")
	}
}

// request is the method used by the `Client` to perform the request against
// the Algolia servers (or to the list of specified hosts).
func (t *Transport) request(method, path string, body interface{}, typeCall int) ([]byte, error) {
	var res []byte
	var err error

	for _, host := range t.hostsToTry(typeCall) {
		res, err = t.tryRequest(method, host, path, body)
		if err == nil {
			t.resetDialTimeout()
			if typeCall == write {
				t.activeWriteSince = time.Now()
				t.activeWriteHost = host
			} else {
				t.activeReadSince = time.Now()
				t.activeReadHost = host
			}
			return res, nil
		}
		t.increaseDialTimeout()
	}

	if typeCall == write {
		t.activeWriteHost = ""
	} else {
		t.activeReadHost = ""
	}

	return nil, err
}

// hostsToTry returns the list of hosts to try ordered by priority according to
// the type of request (write vs. read/search) and if a previous host was
// marked as active.
func (t *Transport) hostsToTry(typeCall int) []string {
	var hosts []string

	if typeCall == write {
		// In case the request is a write query, we put the last active write
		// host first in the list of hosts to try if it was used in the last
		// `keepAliveDuration` seconds. We then put the main algolia.net host.
		if t.activeWriteHost != "" &&
			t.activeWriteHost != t.appId+".algolia.net" &&
			time.Now().Sub(t.activeWriteSince) <= t.keepAliveDuration {
			hosts = []string{t.activeWriteHost}
		}
		hosts = append(hosts, t.appId+".algolia.net")
	} else {
		// In case the request is not a write query, we put the last active
		// read host first in the list of hosts to try if it was used in the
		// last `keepAliveDuration` seconds. We then put the DSN host.
		if t.activeReadHost != "" &&
			t.activeReadHost != t.appId+"-dsn.algolia.net" &&
			time.Now().Sub(t.activeReadSince) <= t.keepAliveDuration {
			hosts = []string{t.activeReadHost}
		}
		hosts = append(hosts, t.appId+"-dsn.algolia.net")
	}

	// In any case, we append all the original hosts (default ones or the ones
	// specified by the user) to the list of hosts to try.
	hosts = append(hosts, t.hosts...)
	return hosts

}

// tryRequest is the underlying method which actually performs the request. It
// returns the response as a byte slice or a non-nil error if anything went
// wrong.
func (t *Transport) tryRequest(method, host, path string, body interface{}) ([]byte, error) {
	// Build the request
	req, err := t.buildRequest(method, host, path, body)
	if err != nil {
		return nil, err
	}

	// Perform the request
	res, err := t.httpClient.Do(req)
	if err != nil {
		return nil, fmt.Errorf("Cannot perform request [%s] %s (%s): %s", method, path, host, err)
	}

	// Read response's body
	bodyRes, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, fmt.Errorf("Cannot read response body: %s", err)
	}
	res.Body.Close()

	// Return the body as an error if the status code is not 2XX
	code := res.StatusCode
	if !(200 <= code && code < 300) {
		return nil, errors.New(string(bodyRes))
	}

	return bodyRes, nil
}

// buildRequest returns a valid `http.Request` with the headers and body (if
// any) correctly set. The return error is non-nil if the request is invalid or
// if the body, if non-nil, is not a valid JSON.
func (t *Transport) buildRequest(method, host, path string, body interface{}) (*http.Request, error) {
	var req *http.Request
	var err error
	urlStr := "https://" + host + path

	if body == nil {
		// As the body is nil, an empty body request is instantiated
		req, err = http.NewRequest(method, urlStr, nil)
		if err != nil {
			return nil, fmt.Errorf("Cannot instantiate request: [%s] %s", method, urlStr)
		}
	} else {
		// As the body is non-nil, the content is read
		data, err := json.Marshal(body)
		if err != nil {
			return nil, errors.New("Invalid JSON in the query")
		}
		reader := bytes.NewReader(data)

		// The request is then instantiated with the body content
		req, err = http.NewRequest(method, urlStr, reader)
		if err != nil {
			return nil, fmt.Errorf("Cannot instantiate request: [%s] %s", method, urlStr)
		}

		// Add content specific headers
		req.Header.Add("Content-Length", strconv.Itoa(len(string(data))))
		req.Header.Add("Content-Type", "application/json; charset=utf-8")
	}

	// Add default and Algolia specific headers
	addHeaders(req, t.headers)

	if strings.Contains(path, "/*/") {
		req.URL = &url.URL{
			Scheme: "https",
			Host:   host,
			Opaque: "//" + host + path, //Remove url encoding
		}
	}

	return req, nil
}

// resetDialTimeout increases the `Timeout` value of the underlying dialer by 1
// second.
func (t *Transport) increaseDialTimeout() {
	t.dialTimeout = t.dialTimeout + time.Second
	t.httpClient.Transport = defaultTransport(t.dialTimeout)
}

// resetDialTimeout resets the `Timeout` value of the underlying dialer to 1
// second.
func (t *Transport) resetDialTimeout() {
	t.dialTimeout = 1 * time.Second
	t.httpClient.Transport = defaultTransport(t.dialTimeout)
}
<|endoftext|>


<commit_before>package hypervisor

import (
	"encoding/json"
	"fmt"
	"sync"

	"github.com/hyperhq/hypercontainer-utils/hlog"
	"github.com/hyperhq/runv/api"
	hyperstartapi "github.com/hyperhq/runv/hyperstart/api/json"
	"github.com/hyperhq/runv/hypervisor/types"
	"github.com/hyperhq/runv/lib/utils"
)

const CURRENT_PERSIST_VERSION = 20171210
const COMPATIBLE_PERSIST_VERSION = CURRENT_PERSIST_VERSION

// check to ensure CURRENT_PERSIST_VERSION >= COMPATIBLE_PERSIST_VERSION
const _ uint = CURRENT_PERSIST_VERSION - COMPATIBLE_PERSIST_VERSION

type VmHwStatus struct {
	PciAddr  int    //next available pci addr for pci hotplug
	ScsiId   int    //next available scsi id for scsi hotplug
	AttachId uint64 //next available attachId for attached tty
	GuestCid uint32 //vsock guest cid
}

type PersistVolumeInfo struct {
	Name         string
	Filename     string
	Format       string
	Fstype       string
	DeviceName   string
	ScsiId       int
	ContainerIds []string
	IsRootVol    bool
	Containers   []int // deprecated
	MontPoints   []string
}

type PersistNetworkInfo struct {
	Id         string
	Index      int
	PciAddr    int
	HostDevice string
	DeviceName string
	NewName    string
	IpAddr     string
	Mac        string
	Mtu        uint64
}

type PersistInfo struct {
	PersistVersion int
	Id             string
	Paused         bool
	DriverInfo     map[string]interface{}
	VmSpec         *hyperstartapi.Pod
	HwStat         *VmHwStatus
	VolumeList     []*PersistVolumeInfo
	NetworkList    []*PersistNetworkInfo
	PortList       []*api.PortDescription
	Containers     map[string]*api.ContainerDescription
}

func (p *PersistInfo) LogLevel(level hlog.LogLevel) bool {
	return hlog.IsLogLevel(level)
}

func (p *PersistInfo) LogPrefix() string {
	return fmt.Sprintf("PSB[%s] ", p.Id)
}

func (p *PersistInfo) Log(level hlog.LogLevel, args ...interface{}) {
	hlog.HLog(level, p, 1, args...)
}

func (ctx *VmContext) dump() (*PersistInfo, error) {
	dr, err := ctx.DCtx.Dump()
	if err != nil {
		return nil, err
	}

	nc := ctx.networks
	info := &PersistInfo{
		PersistVersion: CURRENT_PERSIST_VERSION,
		Id:             ctx.Id,
		Paused:         ctx.PauseState == PauseStatePaused,
		DriverInfo:     dr,
		VmSpec:         ctx.networks.sandboxInfo(),
		HwStat:         ctx.dumpHwInfo(),
		VolumeList:     make([]*PersistVolumeInfo, len(ctx.volumes)+len(ctx.containers)),
		NetworkList:    make([]*PersistNetworkInfo, len(nc.eth)+len(nc.lo)),
		PortList:       make([]*api.PortDescription, len(nc.ports)),
		Containers:     make(map[string]*api.ContainerDescription),
	}

	vid := 0
	for _, vol := range ctx.volumes {
		v := vol.dump()
		for id := range vol.observers {
			v.ContainerIds = append(v.ContainerIds, id)
		}
		info.VolumeList[vid] = v
		vid++
	}

	for i, p := range nc.ports {
		info.PortList[i] = &api.PortDescription{
			HostPort:      p.HostPort,
			ContainerPort: p.ContainerPort,
			Protocol:      p.Protocol,
		}
	}
	nid := 0
	for _, nic := range nc.lo {
		info.NetworkList[nid] = &PersistNetworkInfo{
			Id:         nic.Id,
			Index:      nic.Index,
			PciAddr:    nic.PCIAddr,
			HostDevice: nic.HostDevice,
			DeviceName: nic.DeviceName,
			IpAddr:     nic.IpAddr,
		}
		nid++
	}
	nc.slotLock.RLock()
	for _, nic := range nc.eth {
		info.NetworkList[nid] = &PersistNetworkInfo{
			Id:         nic.Id,
			Index:      nic.Index,
			PciAddr:    nic.PCIAddr,
			HostDevice: nic.HostDevice,
			DeviceName: nic.DeviceName,
			NewName:    nic.NewName,
			IpAddr:     nic.IpAddr,
			Mac:        nic.MacAddr,
			Mtu:        nic.Mtu,
		}
		nid++
	}
	defer nc.slotLock.RUnlock()

	for _, c := range ctx.containers {
		rootVolume := c.root.dump()
		rootVolume.ContainerIds = []string{c.Id}
		rootVolume.IsRootVol = true
		info.VolumeList[vid] = rootVolume
		info.Containers[c.Id] = c.ContainerDescription
		vid++
	}

	return info, nil
}

func (ctx *VmContext) dumpHwInfo() *VmHwStatus {
	return &VmHwStatus{
		PciAddr:  ctx.pciAddr,
		ScsiId:   ctx.scsiId,
		AttachId: ctx.hyperstart.LastStreamSeq(),
		GuestCid: ctx.GuestCid,
	}
}

func (ctx *VmContext) loadHwStatus(pinfo *PersistInfo) error {
	ctx.pciAddr = pinfo.HwStat.PciAddr
	ctx.scsiId = pinfo.HwStat.ScsiId
	ctx.GuestCid = pinfo.HwStat.GuestCid
	if ctx.GuestCid != 0 {
		if !VsockCidManager.MarkCidInuse(ctx.GuestCid) {
			return fmt.Errorf("conflicting vsock guest cid %d: already in use", ctx.GuestCid)
		}
		ctx.Boot.EnableVsock = true
	}
	return nil
}

func (blk *DiskDescriptor) dump() *PersistVolumeInfo {
	return &PersistVolumeInfo{
		Name:       blk.Name,
		Filename:   blk.Filename,
		Format:     blk.Format,
		Fstype:     blk.Fstype,
		DeviceName: blk.DeviceName,
		ScsiId:     blk.ScsiId,
	}
}

func (vol *PersistVolumeInfo) blockInfo() *DiskDescriptor {
	return &DiskDescriptor{
		Name:       vol.Name,
		Filename:   vol.Filename,
		Format:     vol.Format,
		Fstype:     vol.Fstype,
		DeviceName: vol.DeviceName,
		ScsiId:     vol.ScsiId,
	}
}

func (nc *NetworkContext) load(pinfo *PersistInfo) {
	nc.SandboxConfig = &api.SandboxConfig{
		Hostname: pinfo.VmSpec.Hostname,
		Dns:      pinfo.VmSpec.Dns,
	}
	portWhilteList := pinfo.VmSpec.PortmappingWhiteLists
	if portWhilteList != nil {
		nc.Neighbors = &api.NeighborNetworks{
			InternalNetworks: portWhilteList.InternalNetworks,
			ExternalNetworks: portWhilteList.ExternalNetworks,
		}
	}

	for i, p := range pinfo.PortList {
		nc.ports[i] = p
	}
	for _, pi := range pinfo.NetworkList {
		ifc := &InterfaceCreated{
			Id:         pi.Id,
			Index:      pi.Index,
			PCIAddr:    pi.PciAddr,
			HostDevice: pi.HostDevice,
			DeviceName: pi.DeviceName,
			NewName:    pi.NewName,
			IpAddr:     pi.IpAddr,
			Mtu:        pi.Mtu,
			MacAddr:    pi.Mac,
		}
		// if empty, may be old data, generate one for compatibility.
		if ifc.Id == "" {
			ifc.Id = utils.RandStr(8, "alpha")
		}
		// use device name distinguish from lo and eth
		if ifc.DeviceName == DEFAULT_LO_DEVICE_NAME {
			nc.lo[pi.IpAddr] = ifc
		} else {
			nc.eth[pi.Index] = ifc
		}
		nc.idMap[pi.Id] = ifc
	}
}

func vmDeserialize(s []byte) (*PersistInfo, error) {
	info := &PersistInfo{}
	// TODO: REMOVE THIS
	err := json.Unmarshal(s, info)
	return info, err
}

func (pinfo *PersistInfo) serialize() ([]byte, error) {
	return json.Marshal(pinfo)
}

func (pinfo *PersistInfo) vmContext(hub chan VmEvent, client chan *types.VmResponse) (*VmContext, error) {
	if pinfo.PersistVersion > CURRENT_PERSIST_VERSION {
		return nil, fmt.Errorf("error: detech saved data of newer version: PersistInfo's PersistVersion is %d, but compiled code version is %d", pinfo.PersistVersion, CURRENT_PERSIST_VERSION)
	}
	if pinfo.PersistVersion < COMPATIBLE_PERSIST_VERSION {
		return nil, fmt.Errorf("error: detech saved data of older version, but no compatible handler for it: PersistInfo's PersistVersion is %d, but lowest compatible version is %d", pinfo.PersistVersion, COMPATIBLE_PERSIST_VERSION)
	}

	dc, err := HDriver.LoadContext(pinfo.DriverInfo)
	if err != nil {
		pinfo.Log(ERROR, "cannot load driver context: %v", err)
		return nil, err
	}

	ctx, err := InitContext(pinfo.Id, hub, client, dc, &BootConfig{})
	if err != nil {
		return nil, err
	}

	if pinfo.Paused {
		ctx.PauseState = PauseStatePaused
	}

	err = ctx.loadHwStatus(pinfo)
	if err != nil {
		return nil, err
	}

	ctx.networks.load(pinfo)

	// map container id to image DiskDescriptor
	imageMap := make(map[string]*DiskDescriptor)
	// map container id to volume DiskContext list
	volumeMap := make(map[string][]*DiskContext)
	for _, vol := range pinfo.VolumeList {
		binfo := vol.blockInfo()
		if vol.IsRootVol {
			if len(vol.ContainerIds) != 1 {
				return nil, fmt.Errorf("persistent data corrupt, root volume mismatch")
			}
			imageMap[vol.ContainerIds[0]] = binfo
			continue
		}
		ctx.volumes[binfo.Name] = &DiskContext{
			DiskDescriptor: binfo,
			sandbox:        ctx,
			observers:      make(map[string]*sync.WaitGroup),
			lock:           &sync.RWMutex{},
			// FIXME: is there any trouble if we set it as ready when restoring from persistence
			ready: true,
		}
		for _, id := range vol.ContainerIds {
			volumeMap[id] = append(volumeMap[id], ctx.volumes[binfo.Name])
		}
	}

	for cid, des := range pinfo.Containers {
		bInfo, ok := imageMap[cid]
		if !ok {
			return nil, fmt.Errorf("persistent data corrupt, lack of container root volume")
		}
		cc := &ContainerContext{
			ContainerDescription: des,
			sandbox:              ctx,
			logPrefix:            fmt.Sprintf("SB[%s] Con[%s] ", ctx.Id, cid),
			root: &DiskContext{
				DiskDescriptor: bInfo,
				sandbox:        ctx,
				isRootVol:      true,
				ready:          true,
				observers:      make(map[string]*sync.WaitGroup),
				lock:           &sync.RWMutex{},
			},
		}
		// restore wg for volumes attached to container
		wgDisk := &sync.WaitGroup{}
		volList, ok := volumeMap[cid]
		if ok {
			cc.Volumes = make(map[string]*api.VolumeReference, len(volList))
			for _, vol := range volList {
				// for unwait attached volumes when removing container
				cc.Volumes[vol.Name] = &api.VolumeReference{}
				vol.wait(cc.Id, wgDisk)
			}
		}
		cc.root.wait(cc.Id, wgDisk)

		ctx.containers[cc.Id] = cc
	}

	return ctx, nil
}
<commit_msg>use api.SandboxConfig instead of VmSpec for persistence<commit_after>package hypervisor

import (
	"encoding/json"
	"fmt"
	"sync"

	"github.com/hyperhq/hypercontainer-utils/hlog"
	"github.com/hyperhq/runv/api"
	"github.com/hyperhq/runv/hypervisor/types"
	"github.com/hyperhq/runv/lib/utils"
)

const CURRENT_PERSIST_VERSION = 20171210
const COMPATIBLE_PERSIST_VERSION = CURRENT_PERSIST_VERSION

// check to ensure CURRENT_PERSIST_VERSION >= COMPATIBLE_PERSIST_VERSION
const _ uint = CURRENT_PERSIST_VERSION - COMPATIBLE_PERSIST_VERSION

type VmHwStatus struct {
	PciAddr  int    //next available pci addr for pci hotplug
	ScsiId   int    //next available scsi id for scsi hotplug
	AttachId uint64 //next available attachId for attached tty
	GuestCid uint32 //vsock guest cid
}

type PersistVolumeInfo struct {
	Name         string
	Filename     string
	Format       string
	Fstype       string
	DeviceName   string
	ScsiId       int
	ContainerIds []string
	IsRootVol    bool
	Containers   []int // deprecated
	MontPoints   []string
}

type PersistNetworkInfo struct {
	Id         string
	Index      int
	PciAddr    int
	HostDevice string
	DeviceName string
	NewName    string
	IpAddr     string
	Mac        string
	Mtu        uint64
}

type PersistInfo struct {
	PersistVersion int
	Id             string
	Paused         bool
	DriverInfo     map[string]interface{}
	SandboxConfig  *api.SandboxConfig
	HwStat         *VmHwStatus
	VolumeList     []*PersistVolumeInfo
	NetworkList    []*PersistNetworkInfo
	PortList       []*api.PortDescription
	Containers     map[string]*api.ContainerDescription
}

func (p *PersistInfo) LogLevel(level hlog.LogLevel) bool {
	return hlog.IsLogLevel(level)
}

func (p *PersistInfo) LogPrefix() string {
	return fmt.Sprintf("PSB[%s] ", p.Id)
}

func (p *PersistInfo) Log(level hlog.LogLevel, args ...interface{}) {
	hlog.HLog(level, p, 1, args...)
}

func (ctx *VmContext) dump() (*PersistInfo, error) {
	dr, err := ctx.DCtx.Dump()
	if err != nil {
		return nil, err
	}

	nc := ctx.networks
	info := &PersistInfo{
		PersistVersion: CURRENT_PERSIST_VERSION,
		Id:             ctx.Id,
		Paused:         ctx.PauseState == PauseStatePaused,
		DriverInfo:     dr,
		SandboxConfig:  ctx.networks.SandboxConfig,
		HwStat:         ctx.dumpHwInfo(),
		VolumeList:     make([]*PersistVolumeInfo, len(ctx.volumes)+len(ctx.containers)),
		NetworkList:    make([]*PersistNetworkInfo, len(nc.eth)+len(nc.lo)),
		PortList:       make([]*api.PortDescription, len(nc.ports)),
		Containers:     make(map[string]*api.ContainerDescription),
	}

	vid := 0
	for _, vol := range ctx.volumes {
		v := vol.dump()
		for id := range vol.observers {
			v.ContainerIds = append(v.ContainerIds, id)
		}
		info.VolumeList[vid] = v
		vid++
	}

	for i, p := range nc.ports {
		info.PortList[i] = &api.PortDescription{
			HostPort:      p.HostPort,
			ContainerPort: p.ContainerPort,
			Protocol:      p.Protocol,
		}
	}
	nid := 0
	for _, nic := range nc.lo {
		info.NetworkList[nid] = &PersistNetworkInfo{
			Id:         nic.Id,
			Index:      nic.Index,
			PciAddr:    nic.PCIAddr,
			HostDevice: nic.HostDevice,
			DeviceName: nic.DeviceName,
			IpAddr:     nic.IpAddr,
		}
		nid++
	}
	nc.slotLock.RLock()
	for _, nic := range nc.eth {
		info.NetworkList[nid] = &PersistNetworkInfo{
			Id:         nic.Id,
			Index:      nic.Index,
			PciAddr:    nic.PCIAddr,
			HostDevice: nic.HostDevice,
			DeviceName: nic.DeviceName,
			NewName:    nic.NewName,
			IpAddr:     nic.IpAddr,
			Mac:        nic.MacAddr,
			Mtu:        nic.Mtu,
		}
		nid++
	}
	defer nc.slotLock.RUnlock()

	for _, c := range ctx.containers {
		rootVolume := c.root.dump()
		rootVolume.ContainerIds = []string{c.Id}
		rootVolume.IsRootVol = true
		info.VolumeList[vid] = rootVolume
		info.Containers[c.Id] = c.ContainerDescription
		vid++
	}

	return info, nil
}

func (ctx *VmContext) dumpHwInfo() *VmHwStatus {
	return &VmHwStatus{
		PciAddr:  ctx.pciAddr,
		ScsiId:   ctx.scsiId,
		AttachId: ctx.hyperstart.LastStreamSeq(),
		GuestCid: ctx.GuestCid,
	}
}

func (ctx *VmContext) loadHwStatus(pinfo *PersistInfo) error {
	ctx.pciAddr = pinfo.HwStat.PciAddr
	ctx.scsiId = pinfo.HwStat.ScsiId
	ctx.GuestCid = pinfo.HwStat.GuestCid
	if ctx.GuestCid != 0 {
		if !VsockCidManager.MarkCidInuse(ctx.GuestCid) {
			return fmt.Errorf("conflicting vsock guest cid %d: already in use", ctx.GuestCid)
		}
		ctx.Boot.EnableVsock = true
	}
	return nil
}

func (blk *DiskDescriptor) dump() *PersistVolumeInfo {
	return &PersistVolumeInfo{
		Name:       blk.Name,
		Filename:   blk.Filename,
		Format:     blk.Format,
		Fstype:     blk.Fstype,
		DeviceName: blk.DeviceName,
		ScsiId:     blk.ScsiId,
	}
}

func (vol *PersistVolumeInfo) blockInfo() *DiskDescriptor {
	return &DiskDescriptor{
		Name:       vol.Name,
		Filename:   vol.Filename,
		Format:     vol.Format,
		Fstype:     vol.Fstype,
		DeviceName: vol.DeviceName,
		ScsiId:     vol.ScsiId,
	}
}

func (nc *NetworkContext) load(pinfo *PersistInfo) {
	nc.SandboxConfig = pinfo.SandboxConfig

	for i, p := range pinfo.PortList {
		nc.ports[i] = p
	}
	for _, pi := range pinfo.NetworkList {
		ifc := &InterfaceCreated{
			Id:         pi.Id,
			Index:      pi.Index,
			PCIAddr:    pi.PciAddr,
			HostDevice: pi.HostDevice,
			DeviceName: pi.DeviceName,
			NewName:    pi.NewName,
			IpAddr:     pi.IpAddr,
			Mtu:        pi.Mtu,
			MacAddr:    pi.Mac,
		}
		// if empty, may be old data, generate one for compatibility.
		if ifc.Id == "" {
			ifc.Id = utils.RandStr(8, "alpha")
		}
		// use device name distinguish from lo and eth
		if ifc.DeviceName == DEFAULT_LO_DEVICE_NAME {
			nc.lo[pi.IpAddr] = ifc
		} else {
			nc.eth[pi.Index] = ifc
		}
		nc.idMap[pi.Id] = ifc
	}
}

func vmDeserialize(s []byte) (*PersistInfo, error) {
	info := &PersistInfo{}
	// TODO: REMOVE THIS
	err := json.Unmarshal(s, info)
	return info, err
}

func (pinfo *PersistInfo) serialize() ([]byte, error) {
	return json.Marshal(pinfo)
}

func (pinfo *PersistInfo) vmContext(hub chan VmEvent, client chan *types.VmResponse) (*VmContext, error) {
	if pinfo.PersistVersion > CURRENT_PERSIST_VERSION {
		return nil, fmt.Errorf("error: detech saved data of newer version: PersistInfo's PersistVersion is %d, but compiled code version is %d", pinfo.PersistVersion, CURRENT_PERSIST_VERSION)
	}
	if pinfo.PersistVersion < COMPATIBLE_PERSIST_VERSION {
		return nil, fmt.Errorf("error: detech saved data of older version, but no compatible handler for it: PersistInfo's PersistVersion is %d, but lowest compatible version is %d", pinfo.PersistVersion, COMPATIBLE_PERSIST_VERSION)
	}

	dc, err := HDriver.LoadContext(pinfo.DriverInfo)
	if err != nil {
		pinfo.Log(ERROR, "cannot load driver context: %v", err)
		return nil, err
	}

	ctx, err := InitContext(pinfo.Id, hub, client, dc, &BootConfig{})
	if err != nil {
		return nil, err
	}

	if pinfo.Paused {
		ctx.PauseState = PauseStatePaused
	}

	err = ctx.loadHwStatus(pinfo)
	if err != nil {
		return nil, err
	}

	ctx.networks.load(pinfo)

	// map container id to image DiskDescriptor
	imageMap := make(map[string]*DiskDescriptor)
	// map container id to volume DiskContext list
	volumeMap := make(map[string][]*DiskContext)
	for _, vol := range pinfo.VolumeList {
		binfo := vol.blockInfo()
		if vol.IsRootVol {
			if len(vol.ContainerIds) != 1 {
				return nil, fmt.Errorf("persistent data corrupt, root volume mismatch")
			}
			imageMap[vol.ContainerIds[0]] = binfo
			continue
		}
		ctx.volumes[binfo.Name] = &DiskContext{
			DiskDescriptor: binfo,
			sandbox:        ctx,
			observers:      make(map[string]*sync.WaitGroup),
			lock:           &sync.RWMutex{},
			// FIXME: is there any trouble if we set it as ready when restoring from persistence
			ready: true,
		}
		for _, id := range vol.ContainerIds {
			volumeMap[id] = append(volumeMap[id], ctx.volumes[binfo.Name])
		}
	}

	for cid, des := range pinfo.Containers {
		bInfo, ok := imageMap[cid]
		if !ok {
			return nil, fmt.Errorf("persistent data corrupt, lack of container root volume")
		}
		cc := &ContainerContext{
			ContainerDescription: des,
			sandbox:              ctx,
			logPrefix:            fmt.Sprintf("SB[%s] Con[%s] ", ctx.Id, cid),
			root: &DiskContext{
				DiskDescriptor: bInfo,
				sandbox:        ctx,
				isRootVol:      true,
				ready:          true,
				observers:      make(map[string]*sync.WaitGroup),
				lock:           &sync.RWMutex{},
			},
		}
		// restore wg for volumes attached to container
		wgDisk := &sync.WaitGroup{}
		volList, ok := volumeMap[cid]
		if ok {
			cc.Volumes = make(map[string]*api.VolumeReference, len(volList))
			for _, vol := range volList {
				// for unwait attached volumes when removing container
				cc.Volumes[vol.Name] = &api.VolumeReference{}
				vol.wait(cc.Id, wgDisk)
			}
		}
		cc.root.wait(cc.Id, wgDisk)

		ctx.containers[cc.Id] = cc
	}

	return ctx, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package client

import (
	"bytes"
	"crypto/sha1"
	"fmt"
	"http"
	"io"
	"io/ioutil"
	"json"
	"log"
	"os"
	"strings"

	"camli/blobref"
	"camli/misc"
)

var _ = log.Printf

type UploadHandle struct {
	BlobRef  *blobref.BlobRef
	Size     int64 // or -1 if size isn't known
	Contents io.Reader
}

type PutResult struct {
	BlobRef *blobref.BlobRef
	Size    int64
	Skipped bool // already present on blobserver
}

func (pr *PutResult) SizedBlobRef() blobref.SizedBlobRef {
	return blobref.SizedBlobRef{pr.BlobRef, pr.Size}
}

type statResponse struct {
	HaveMap                    map[string]blobref.SizedBlobRef
	maxUploadSize              int64
	uploadUrl                  string
	uploadUrlExpirationSeconds int
	canLongPoll                bool
}

type ResponseFormatError os.Error

func newResFormatError(s string, arg ...interface{}) ResponseFormatError {
	return ResponseFormatError(fmt.Errorf(s, arg...))
}

// TODO-GO: if outerr is replaced by a "_", gotest(!) fails with a 6g error.
func parseStatResponse(r io.Reader) (sr *statResponse, outerr os.Error) {
	var (
		ok   bool
		err  os.Error
		s    = &statResponse{HaveMap: make(map[string]blobref.SizedBlobRef)}
		jmap = make(map[string]interface{})
	)
	if err = json.NewDecoder(io.LimitReader(r, 5<<20)).Decode(&jmap); err != nil {
		return nil, ResponseFormatError(err)
	}
	defer func() {
		if sr == nil {
			log.Printf("parseStatResponse got map: %#v", jmap)
		}
	}()

	s.uploadUrl, ok = jmap["uploadUrl"].(string)
	if !ok {
		return nil, newResFormatError("no 'uploadUrl' in stat response")
	}

	if n, ok := jmap["maxUploadSize"].(float64); ok {
		s.maxUploadSize = int64(n)
	} else {
		return nil, newResFormatError("no 'maxUploadSize' in stat response")
	}

	if n, ok := jmap["uploadUrlExpirationSeconds"].(float64); ok {
		s.uploadUrlExpirationSeconds = int(n)
	} else {
		return nil, newResFormatError("no 'uploadUrlExpirationSeconds' in stat response")
	}

	if v, ok := jmap["canLongPoll"].(bool); ok {
		s.canLongPoll = v
	}

	alreadyHave, ok := jmap["stat"].([]interface{})
	if !ok {
		return nil, newResFormatError("no 'stat' key in stat response")
	}

	for _, li := range alreadyHave {
		m, ok := li.(map[string]interface{})
		if !ok {
			return nil, newResFormatError("'stat' list value of unexpected type %T", li)
		}
		blobRefStr, ok := m["blobRef"].(string)
		if !ok {
			return nil, newResFormatError("'stat' list item has non-string 'blobRef' key")
		}
		size, ok := m["size"].(float64)
		if !ok {
			return nil, newResFormatError("'stat' list item has non-number 'size' key")
		}
		br := blobref.Parse(blobRefStr)
		if br == nil {
			return nil, newResFormatError("'stat' list item has invalid 'blobRef' key")
		}
		s.HaveMap[br.String()] = blobref.SizedBlobRef{br, int64(size)}
	}

	return s, nil
}

func NewUploadHandleFromString(data string) *UploadHandle {
	s1 := sha1.New()
	s1.Write([]byte(data))
	bref := blobref.FromHash("sha1", s1)
	buf := bytes.NewBufferString(data)
	return &UploadHandle{BlobRef: bref, Size: int64(len(data)), Contents: buf}
}

func (c *Client) jsonFromResponse(requestName string, resp *http.Response) (map[string]interface{}, os.Error) {
	if resp.StatusCode != 200 {
		log.Printf("After %s request, failed to JSON from response; status code is %d", requestName, resp.StatusCode)
		io.Copy(os.Stderr, resp.Body)
		return nil, os.NewError(fmt.Sprintf("After %s request, HTTP response code is %d; no JSON to parse.", requestName, resp.StatusCode))
	}
	// TODO: LimitReader here for paranoia
	buf := new(bytes.Buffer)
	io.Copy(buf, resp.Body)
	resp.Body.Close()
	jmap := make(map[string]interface{})
	if jerr := json.Unmarshal(buf.Bytes(), &jmap); jerr != nil {
		return nil, jerr
	}
	return jmap, nil
}

func (c *Client) Stat(dest chan<- blobref.SizedBlobRef, blobs []*blobref.BlobRef, waitSeconds int) os.Error {
	if len(blobs) == 0 {
		return nil
	}

	// TODO: if len(blobs) > 1000 or something, cut this up into
	// multiple http requests, and also if the server returns a
	// 400 error, per the blob-stat-protocol.txt document.
	var buf bytes.Buffer
	fmt.Fprintf(&buf, "camliversion=1")
	for n, blob := range blobs {
		if blob == nil {
			panic("nil blob")
		}
		fmt.Fprintf(&buf, "&blob%d=%s", n+1, blob)
	}

	if waitSeconds > 0 {
		fmt.Fprintf(&buf, "&maxwaitsec=%d", waitSeconds)
	}

	req, err := http.NewRequest("POST", fmt.Sprintf("%s/camli/stat", c.server), strings.NewReader(buf.String()))
	if err != nil {
		return err
	}
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	req.ContentLength = int64(buf.Len())

	resp, err := c.httpClient.Do(req)
	if err != nil {
		return fmt.Errorf("stat HTTP error: %v", err)
	}

	if resp.StatusCode != 200 {
		return fmt.Errorf("stat response had http status %d", resp.StatusCode)
	}

	stat, err := parseStatResponse(resp.Body)
	if err != nil {
		return err
	}

	for _, sb := range stat.HaveMap {
		dest <- sb
	}
	return nil
}

func (c *Client) Upload(h *UploadHandle) (*PutResult, os.Error) {
	error := func(msg string, arg ...interface{}) (*PutResult, os.Error) {
		err := fmt.Errorf(msg, arg...)
		c.log.Print(err.String())
		return nil, err
	}

	c.statsMutex.Lock()
	c.stats.UploadRequests.Blobs++
	if h.Size != -1 {
		c.stats.UploadRequests.Bytes += h.Size
	}
	c.statsMutex.Unlock()

	blobRefString := h.BlobRef.String()

	// Pre-upload.  Check whether the blob already exists on the
	// server and if not, the URL to upload it to.
	url := fmt.Sprintf("%s/camli/stat", c.server)
	requestBody := "camliversion=1&blob1=" + blobRefString
	req := c.newRequest("POST", url)
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	req.Body = ioutil.NopCloser(strings.NewReader(requestBody))
	req.ContentLength = int64(len(requestBody))
	req.TransferEncoding = nil

	resp, err := c.httpClient.Do(req)
	if err != nil {
		return error("stat http error: %v", err)
	}

	if resp.StatusCode != 200 {
		return error("stat response had http status %d", resp.StatusCode)
	}

	stat, err := parseStatResponse(resp.Body)
	if err != nil {
		return nil, err
	}

	pr := &PutResult{BlobRef: h.BlobRef, Size: h.Size}
	if _, ok := stat.HaveMap[h.BlobRef.String()]; ok {
		pr.Skipped = true

		// Consume the buffer that was provided, just for
		// consistency. But if it's a closer, do that
		// instead. But if they didn't provide a size,
		// we consume it anyway just to get the size
		// for stats.
		closer, _ := h.Contents.(io.Closer)
		if h.Size >= 0 && closer != nil {
			closer.Close()
		} else {
			n, err := io.Copy(ioutil.Discard, h.Contents)
			if err != nil {
				return nil, err
			}
			if h.Size == -1 {
				pr.Size = n
				c.statsMutex.Lock()
				c.stats.UploadRequests.Bytes += pr.Size
				c.statsMutex.Unlock()
			}
		}
		return pr, nil
	}

	// TODO: use a proper random boundary
	boundary := "sdf8sd8f7s9df9s7df9sd7sdf9s879vs7d8v7sd8v7sd8v"

	// TODO-GO: add a multipart writer class.
	multiPartHeader := fmt.Sprintf(
		"--%s\r\nContent-Type: application/octet-stream\r\n"+
			"Content-Disposition: form-data; name=\"%s\"; filename=\"%s\"\r\n\r\n",
		boundary,
		h.BlobRef, h.BlobRef)
	multiPartFooter := "\r\n--" + boundary + "--\r\n"

	c.log.Printf("Uploading to URL: %s", stat.uploadUrl)
	req = c.newRequest("POST", stat.uploadUrl)
	req.Header.Set("Content-Type", "multipart/form-data; boundary="+boundary)

	contentsSize := int64(0)
	req.Body = ioutil.NopCloser(io.MultiReader(
		strings.NewReader(multiPartHeader),
		misc.CountingReader{h.Contents, &contentsSize},
		strings.NewReader(multiPartFooter)))

	if h.Size >= 0 {
		req.ContentLength = int64(len(multiPartHeader)) + h.Size + int64(len(multiPartFooter))
	}
	req.TransferEncoding = nil
	resp, err = c.httpClient.Do(req)
	if err != nil {
		return error("upload http error: %v", err)
	}

	if h.Size >= 0 {
		if contentsSize != h.Size {
			return error("UploadHandle declared size %d but Contents length was %d", h.Size, contentsSize)
		}
	} else {
		h.Size = contentsSize
	}

	// The only valid HTTP responses are 200 and 303.
	if resp.StatusCode != 200 && resp.StatusCode != 303 {
		return error("invalid http response %d in upload response", resp.StatusCode)
	}

	if resp.StatusCode == 303 {
		otherLocation := resp.Header.Get("Location")
		if otherLocation == "" {
			return error("303 without a Location")
		}
		baseUrl, _ := http.ParseURL(stat.uploadUrl)
		absUrl, err := baseUrl.ParseURL(otherLocation)
		if err != nil {
			return error("303 Location URL relative resolve error: %v", err)
		}
		otherLocation = absUrl.String()
		resp, err = http.Get(otherLocation)
		if err != nil {
			return error("error following 303 redirect after upload: %v", err)
		}
	}

	ures, err := c.jsonFromResponse("upload", resp)
	if err != nil {
		return error("json parse from upload error: %v", err)
	}

	errorText, ok := ures["errorText"].(string)
	if ok {
		c.log.Printf("Blob server reports error: %s", errorText)
	}

	received, ok := ures["received"].([]interface{})
	if !ok {
		return error("upload json validity error: no 'received'")
	}

	for _, rit := range received {
		it, ok := rit.(map[string]interface{})
		if !ok {
			return error("upload json validity error: 'received' is malformed")
		}
		if it["blobRef"] == blobRefString {
			switch size := it["size"].(type) {
			case nil:
				return error("upload json validity error: 'received' is missing 'size'")
			case float64:
				if int64(size) == h.Size {
					// Success!
					c.statsMutex.Lock()
					c.stats.Uploads.Blobs++
					c.stats.Uploads.Bytes += h.Size
					c.statsMutex.Unlock()
					return pr, nil
				} else {
					return error("Server got blob, but reports wrong length (%v; expected %d)",
						size, h.Size)
				}
			default:
				return error("unsupported type of 'size' in received response")
			}
		}
	}

	return nil, os.NewError("Server didn't receive blob.")
}
<commit_msg>client: Use blobref.Sha1FromString<commit_after>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package client

import (
	"bytes"
	"fmt"
	"http"
	"io"
	"io/ioutil"
	"json"
	"log"
	"os"
	"strings"

	"camli/blobref"
	"camli/misc"
)

var _ = log.Printf

type UploadHandle struct {
	BlobRef  *blobref.BlobRef
	Size     int64 // or -1 if size isn't known
	Contents io.Reader
}

type PutResult struct {
	BlobRef *blobref.BlobRef
	Size    int64
	Skipped bool // already present on blobserver
}

func (pr *PutResult) SizedBlobRef() blobref.SizedBlobRef {
	return blobref.SizedBlobRef{pr.BlobRef, pr.Size}
}

type statResponse struct {
	HaveMap                    map[string]blobref.SizedBlobRef
	maxUploadSize              int64
	uploadUrl                  string
	uploadUrlExpirationSeconds int
	canLongPoll                bool
}

type ResponseFormatError os.Error

func newResFormatError(s string, arg ...interface{}) ResponseFormatError {
	return ResponseFormatError(fmt.Errorf(s, arg...))
}

// TODO-GO: if outerr is replaced by a "_", gotest(!) fails with a 6g error.
func parseStatResponse(r io.Reader) (sr *statResponse, outerr os.Error) {
	var (
		ok   bool
		err  os.Error
		s    = &statResponse{HaveMap: make(map[string]blobref.SizedBlobRef)}
		jmap = make(map[string]interface{})
	)
	if err = json.NewDecoder(io.LimitReader(r, 5<<20)).Decode(&jmap); err != nil {
		return nil, ResponseFormatError(err)
	}
	defer func() {
		if sr == nil {
			log.Printf("parseStatResponse got map: %#v", jmap)
		}
	}()

	s.uploadUrl, ok = jmap["uploadUrl"].(string)
	if !ok {
		return nil, newResFormatError("no 'uploadUrl' in stat response")
	}

	if n, ok := jmap["maxUploadSize"].(float64); ok {
		s.maxUploadSize = int64(n)
	} else {
		return nil, newResFormatError("no 'maxUploadSize' in stat response")
	}

	if n, ok := jmap["uploadUrlExpirationSeconds"].(float64); ok {
		s.uploadUrlExpirationSeconds = int(n)
	} else {
		return nil, newResFormatError("no 'uploadUrlExpirationSeconds' in stat response")
	}

	if v, ok := jmap["canLongPoll"].(bool); ok {
		s.canLongPoll = v
	}

	alreadyHave, ok := jmap["stat"].([]interface{})
	if !ok {
		return nil, newResFormatError("no 'stat' key in stat response")
	}

	for _, li := range alreadyHave {
		m, ok := li.(map[string]interface{})
		if !ok {
			return nil, newResFormatError("'stat' list value of unexpected type %T", li)
		}
		blobRefStr, ok := m["blobRef"].(string)
		if !ok {
			return nil, newResFormatError("'stat' list item has non-string 'blobRef' key")
		}
		size, ok := m["size"].(float64)
		if !ok {
			return nil, newResFormatError("'stat' list item has non-number 'size' key")
		}
		br := blobref.Parse(blobRefStr)
		if br == nil {
			return nil, newResFormatError("'stat' list item has invalid 'blobRef' key")
		}
		s.HaveMap[br.String()] = blobref.SizedBlobRef{br, int64(size)}
	}

	return s, nil
}

func NewUploadHandleFromString(data string) *UploadHandle {
	bref := blobref.Sha1FromString(data)
	r := strings.NewReader(data)
	return &UploadHandle{BlobRef: bref, Size: int64(len(data)), Contents: r}
}

func (c *Client) jsonFromResponse(requestName string, resp *http.Response) (map[string]interface{}, os.Error) {
	if resp.StatusCode != 200 {
		log.Printf("After %s request, failed to JSON from response; status code is %d", requestName, resp.StatusCode)
		io.Copy(os.Stderr, resp.Body)
		return nil, os.NewError(fmt.Sprintf("After %s request, HTTP response code is %d; no JSON to parse.", requestName, resp.StatusCode))
	}
	// TODO: LimitReader here for paranoia
	buf := new(bytes.Buffer)
	io.Copy(buf, resp.Body)
	resp.Body.Close()
	jmap := make(map[string]interface{})
	if jerr := json.Unmarshal(buf.Bytes(), &jmap); jerr != nil {
		return nil, jerr
	}
	return jmap, nil
}

func (c *Client) Stat(dest chan<- blobref.SizedBlobRef, blobs []*blobref.BlobRef, waitSeconds int) os.Error {
	if len(blobs) == 0 {
		return nil
	}

	// TODO: if len(blobs) > 1000 or something, cut this up into
	// multiple http requests, and also if the server returns a
	// 400 error, per the blob-stat-protocol.txt document.
	var buf bytes.Buffer
	fmt.Fprintf(&buf, "camliversion=1")
	for n, blob := range blobs {
		if blob == nil {
			panic("nil blob")
		}
		fmt.Fprintf(&buf, "&blob%d=%s", n+1, blob)
	}

	if waitSeconds > 0 {
		fmt.Fprintf(&buf, "&maxwaitsec=%d", waitSeconds)
	}

	req, err := http.NewRequest("POST", fmt.Sprintf("%s/camli/stat", c.server), strings.NewReader(buf.String()))
	if err != nil {
		return err
	}
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	req.ContentLength = int64(buf.Len())

	resp, err := c.httpClient.Do(req)
	if err != nil {
		return fmt.Errorf("stat HTTP error: %v", err)
	}

	if resp.StatusCode != 200 {
		return fmt.Errorf("stat response had http status %d", resp.StatusCode)
	}

	stat, err := parseStatResponse(resp.Body)
	if err != nil {
		return err
	}

	for _, sb := range stat.HaveMap {
		dest <- sb
	}
	return nil
}

func (c *Client) Upload(h *UploadHandle) (*PutResult, os.Error) {
	error := func(msg string, arg ...interface{}) (*PutResult, os.Error) {
		err := fmt.Errorf(msg, arg...)
		c.log.Print(err.String())
		return nil, err
	}

	c.statsMutex.Lock()
	c.stats.UploadRequests.Blobs++
	if h.Size != -1 {
		c.stats.UploadRequests.Bytes += h.Size
	}
	c.statsMutex.Unlock()

	blobRefString := h.BlobRef.String()

	// Pre-upload.  Check whether the blob already exists on the
	// server and if not, the URL to upload it to.
	url := fmt.Sprintf("%s/camli/stat", c.server)
	requestBody := "camliversion=1&blob1=" + blobRefString
	req := c.newRequest("POST", url)
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	req.Body = ioutil.NopCloser(strings.NewReader(requestBody))
	req.ContentLength = int64(len(requestBody))
	req.TransferEncoding = nil

	resp, err := c.httpClient.Do(req)
	if err != nil {
		return error("stat http error: %v", err)
	}

	if resp.StatusCode != 200 {
		return error("stat response had http status %d", resp.StatusCode)
	}

	stat, err := parseStatResponse(resp.Body)
	if err != nil {
		return nil, err
	}

	pr := &PutResult{BlobRef: h.BlobRef, Size: h.Size}
	if _, ok := stat.HaveMap[h.BlobRef.String()]; ok {
		pr.Skipped = true

		// Consume the buffer that was provided, just for
		// consistency. But if it's a closer, do that
		// instead. But if they didn't provide a size,
		// we consume it anyway just to get the size
		// for stats.
		closer, _ := h.Contents.(io.Closer)
		if h.Size >= 0 && closer != nil {
			closer.Close()
		} else {
			n, err := io.Copy(ioutil.Discard, h.Contents)
			if err != nil {
				return nil, err
			}
			if h.Size == -1 {
				pr.Size = n
				c.statsMutex.Lock()
				c.stats.UploadRequests.Bytes += pr.Size
				c.statsMutex.Unlock()
			}
		}
		return pr, nil
	}

	// TODO: use a proper random boundary
	boundary := "sdf8sd8f7s9df9s7df9sd7sdf9s879vs7d8v7sd8v7sd8v"

	// TODO-GO: add a multipart writer class.
	multiPartHeader := fmt.Sprintf(
		"--%s\r\nContent-Type: application/octet-stream\r\n"+
			"Content-Disposition: form-data; name=\"%s\"; filename=\"%s\"\r\n\r\n",
		boundary,
		h.BlobRef, h.BlobRef)
	multiPartFooter := "\r\n--" + boundary + "--\r\n"

	c.log.Printf("Uploading to URL: %s", stat.uploadUrl)
	req = c.newRequest("POST", stat.uploadUrl)
	req.Header.Set("Content-Type", "multipart/form-data; boundary="+boundary)

	contentsSize := int64(0)
	req.Body = ioutil.NopCloser(io.MultiReader(
		strings.NewReader(multiPartHeader),
		misc.CountingReader{h.Contents, &contentsSize},
		strings.NewReader(multiPartFooter)))

	if h.Size >= 0 {
		req.ContentLength = int64(len(multiPartHeader)) + h.Size + int64(len(multiPartFooter))
	}
	req.TransferEncoding = nil
	resp, err = c.httpClient.Do(req)
	if err != nil {
		return error("upload http error: %v", err)
	}

	if h.Size >= 0 {
		if contentsSize != h.Size {
			return error("UploadHandle declared size %d but Contents length was %d", h.Size, contentsSize)
		}
	} else {
		h.Size = contentsSize
	}

	// The only valid HTTP responses are 200 and 303.
	if resp.StatusCode != 200 && resp.StatusCode != 303 {
		return error("invalid http response %d in upload response", resp.StatusCode)
	}

	if resp.StatusCode == 303 {
		otherLocation := resp.Header.Get("Location")
		if otherLocation == "" {
			return error("303 without a Location")
		}
		baseUrl, _ := http.ParseURL(stat.uploadUrl)
		absUrl, err := baseUrl.ParseURL(otherLocation)
		if err != nil {
			return error("303 Location URL relative resolve error: %v", err)
		}
		otherLocation = absUrl.String()
		resp, err = http.Get(otherLocation)
		if err != nil {
			return error("error following 303 redirect after upload: %v", err)
		}
	}

	ures, err := c.jsonFromResponse("upload", resp)
	if err != nil {
		return error("json parse from upload error: %v", err)
	}

	errorText, ok := ures["errorText"].(string)
	if ok {
		c.log.Printf("Blob server reports error: %s", errorText)
	}

	received, ok := ures["received"].([]interface{})
	if !ok {
		return error("upload json validity error: no 'received'")
	}

	for _, rit := range received {
		it, ok := rit.(map[string]interface{})
		if !ok {
			return error("upload json validity error: 'received' is malformed")
		}
		if it["blobRef"] == blobRefString {
			switch size := it["size"].(type) {
			case nil:
				return error("upload json validity error: 'received' is missing 'size'")
			case float64:
				if int64(size) == h.Size {
					// Success!
					c.statsMutex.Lock()
					c.stats.Uploads.Blobs++
					c.stats.Uploads.Bytes += h.Size
					c.statsMutex.Unlock()
					return pr, nil
				} else {
					return error("Server got blob, but reports wrong length (%v; expected %d)",
						size, h.Size)
				}
			default:
				return error("unsupported type of 'size' in received response")
			}
		}
	}

	return nil, os.NewError("Server didn't receive blob.")
}
<|endoftext|>


<commit_before>package runner

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/iron-io/functions/api/models"
)

func getTask(url string) (*models.Task, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var task models.Task

	if err := json.Unmarshal(body, &task); err != nil {
		return nil, err
	}

	if task.ID == "" {
		return nil, errors.New("Invalid Task: ID empty")
	}
	return &task, nil
}

func getCfg(task *models.Task) *Config {
	var stdout bytes.Buffer                                           // TODO: should limit the size of this, error if gets too big. akin to: https://golang.org/pkg/io/#LimitReader
	stderr := NewFuncLogger(task.RouteName, "", *task.Image, task.ID) // TODO: missing path here, how do i get that?
	if task.Timeout == nil {
		timeout := int32(30)
		task.Timeout = &timeout
	}
	cfg := &Config{
		Image:   *task.Image,
		Timeout: time.Duration(*task.Timeout) * time.Second,
		ID:      task.ID,
		AppName: task.RouteName,
		Stdout:  &stdout,
		Stderr:  stderr,
		Env:     task.EnvVars,
	}
	return cfg
}

func deleteTask(url string, task *models.Task) error {
	// Unmarshal task to be sent over as a json
	body, err := json.Marshal(task)
	if err != nil {
		return err
	}

	// Send out Delete request to delete task from queue
	req, err := http.NewRequest(http.MethodDelete, url, bytes.NewBuffer(body))
	if err != nil {
		return err
	}
	c := &http.Client{}
	if resp, err := c.Do(req); err != nil {
		return err
	} else if resp.StatusCode != http.StatusAccepted {
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		return errors.New(string(body))
	}
	return nil
}

func runTask(task *models.Task) error {
	// Set up runner and process task
	cfg := getCfg(task)
	ctx := context.Background()
	rnr, err := New(NewMetricLogger())
	if err != nil {
		return err
	}
	_, err = rnr.Run(ctx, cfg)
	return err
}

// RunAsyncRunner pulls tasks off a queue and processes them
func RunAsyncRunner(ctx context.Context, wgAsync *sync.WaitGroup, tasksrv, port string, n int) {
	u := tasksrvURL(tasksrv, port)

	var wg sync.WaitGroup
	for i := 0; i < n; i++ {
		wg.Add(1)
		go startAsyncRunners(ctx, &wg, i, u, runTask)
	}

	wg.Wait()
	<-ctx.Done()
	wgAsync.Done()
}

func startAsyncRunners(ctx context.Context, wg *sync.WaitGroup, i int, url string, runTask func(task *models.Task) error) {
	defer wg.Done()
	for {
		select {
		case <-ctx.Done():
			return

		default:
			task, err := getTask(url)
			if err != nil {
				log.WithError(err).Error("Could not fetch task")
				time.Sleep(1 * time.Second)
				continue
			}
			log.Info("Picked up task:", task.ID)

			log.Info("Running task:", task.ID)
			// Process Task
			if err := runTask(task); err != nil {
				log.WithError(err).WithFields(log.Fields{"async runner": i, "task_id": task.ID}).Error("Cannot run task")
				continue
			}
			log.Info("Processed task:", task.ID)

			// Delete task from queue
			if err := deleteTask(url, task); err != nil {
				log.WithError(err).WithFields(log.Fields{"async runner": i, "task_id": task.ID}).Error("Cannot delete task")
				continue
			}

			log.Info("Deleted task:", task.ID)
		}
	}
}

func tasksrvURL(tasksrv, port string) string {
	parsed, err := url.Parse(tasksrv)
	if err != nil {
		log.Fatalf("cannot parse TASKSRV endpoint: %v", err)
	}

	if parsed.Scheme == "" {
		parsed.Scheme = "http"
	}

	if parsed.Path == "" || parsed.Path == "/" {
		parsed.Path = "/tasks"
	}

	if _, _, err := net.SplitHostPort(parsed.Host); err != nil {
		parsed.Host = net.JoinHostPort(parsed.Host, port)
	}

	return parsed.String()
}
<commit_msg>If no task if found don't print error (#130)<commit_after>package runner

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/iron-io/functions/api/models"
)

func getTask(url string) (*models.Task, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var task models.Task

	if err := json.Unmarshal(body, &task); err != nil {
		return nil, err
	}

	if task.ID == "" {
		return nil, nil
	}
	return &task, nil
}

func getCfg(task *models.Task) *Config {
	var stdout bytes.Buffer                                           // TODO: should limit the size of this, error if gets too big. akin to: https://golang.org/pkg/io/#LimitReader
	stderr := NewFuncLogger(task.RouteName, "", *task.Image, task.ID) // TODO: missing path here, how do i get that?
	if task.Timeout == nil {
		timeout := int32(30)
		task.Timeout = &timeout
	}
	cfg := &Config{
		Image:   *task.Image,
		Timeout: time.Duration(*task.Timeout) * time.Second,
		ID:      task.ID,
		AppName: task.RouteName,
		Stdout:  &stdout,
		Stderr:  stderr,
		Env:     task.EnvVars,
	}
	return cfg
}

func deleteTask(url string, task *models.Task) error {
	// Unmarshal task to be sent over as a json
	body, err := json.Marshal(task)
	if err != nil {
		return err
	}

	// Send out Delete request to delete task from queue
	req, err := http.NewRequest(http.MethodDelete, url, bytes.NewBuffer(body))
	if err != nil {
		return err
	}
	c := &http.Client{}
	if resp, err := c.Do(req); err != nil {
		return err
	} else if resp.StatusCode != http.StatusAccepted {
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		return errors.New(string(body))
	}
	return nil
}

func runTask(task *models.Task) error {
	// Set up runner and process task
	cfg := getCfg(task)
	ctx := context.Background()
	rnr, err := New(NewMetricLogger())
	if err != nil {
		return err
	}
	_, err = rnr.Run(ctx, cfg)
	return err
}

// RunAsyncRunner pulls tasks off a queue and processes them
func RunAsyncRunner(ctx context.Context, wgAsync *sync.WaitGroup, tasksrv, port string, n int) {
	u := tasksrvURL(tasksrv, port)

	var wg sync.WaitGroup
	for i := 0; i < n; i++ {
		wg.Add(1)
		go startAsyncRunners(ctx, &wg, i, u, runTask)
	}

	wg.Wait()
	<-ctx.Done()
	wgAsync.Done()
}

func startAsyncRunners(ctx context.Context, wg *sync.WaitGroup, i int, url string, runTask func(task *models.Task) error) {
	defer wg.Done()
	for {
		select {
		case <-ctx.Done():
			return

		default:
			task, err := getTask(url)
			if err != nil {
				log.WithError(err).Error("Could not fetch task")
				time.Sleep(1 * time.Second)
				continue
			}
			if task == nil {
				time.Sleep(1 * time.Second)
				continue
			}
			log.Info("Picked up task:", task.ID)

			log.Info("Running task:", task.ID)
			// Process Task
			if err := runTask(task); err != nil {
				log.WithError(err).WithFields(log.Fields{"async runner": i, "task_id": task.ID}).Error("Cannot run task")
				continue
			}
			log.Info("Processed task:", task.ID)

			// Delete task from queue
			if err := deleteTask(url, task); err != nil {
				log.WithError(err).WithFields(log.Fields{"async runner": i, "task_id": task.ID}).Error("Cannot delete task")
				continue
			}

			log.Info("Deleted task:", task.ID)
		}
	}
}

func tasksrvURL(tasksrv, port string) string {
	parsed, err := url.Parse(tasksrv)
	if err != nil {
		log.Fatalf("cannot parse TASKSRV endpoint: %v", err)
	}

	if parsed.Scheme == "" {
		parsed.Scheme = "http"
	}

	if parsed.Path == "" || parsed.Path == "/" {
		parsed.Path = "/tasks"
	}

	if _, _, err := net.SplitHostPort(parsed.Host); err != nil {
		parsed.Host = net.JoinHostPort(parsed.Host, port)
	}

	return parsed.String()
}
<|endoftext|>


<commit_before>// Copyright 2014 Tamás Gulácsi. All rights reserved.
// Use of this source code is governed by an Apache 2.0
// license that can be found in the LICENSE file.

package picago

import (
	"errors"
	"fmt"
	"net"
	"net/http"

	"camlistore.org/third_party/code.google.com/p/goauth2/oauth"
)

const picasaScope = "https://picasaweb.google.com/data/"

var ErrCodeNeeded = errors.New("Authorization code is needed")

// Authorize authorizes using OAuth2
// the ID and secret strings can be acquired from Google for the application
// https://developers.google.com/accounts/docs/OAuth2#basicsteps
func Authorize(ID, secret string) error {
	return errors.New("Not implemented")
}

// NewClient returns an authorized http.Client usable for requests,
// caching tokens in the given file.
func NewClient(id, secret, code, tokenCacheFilename string) (*http.Client, error) {
	return NewClientCache(id, secret, code, oauth.CacheFile(tokenCacheFilename))
}

// For redirect_uri, see https://developers.google.com/accounts/docs/OAuth2InstalledApp#choosingredirecturi .
//
// NewClientCache returns an authorized http.Client with the given oauth.Cache implementation
func NewClientCache(id, secret, code string, cache oauth.Cache) (*http.Client, error) {
	transport, err := NewTransport(id, secret, cache)
	if err != nil {
		return nil, err
	}

	// Try to pull the token from the cache; if this fails, we need to get one.
	token, err := transport.Config.TokenCache.Token()
	if err == nil {
		transport.Token = token
	} else {
		if id == "" || secret == "" {
			return nil, errors.New("token cache is empty, thus client ID and secret is needed!")
		}
		if code == "" {
			l, err := getListener()
			if err != nil {
				return nil, err
			}
			donech := make(chan struct{}, 1)
			transport.Config.RedirectURL = "http://" + l.Addr().String()
			// Get an authorization code from the data provider.
			// ("Please ask the user if I can access this resource.")
			url := transport.Config.AuthCodeURL("picago")
			fmt.Println("Visit this URL to get a code, then run again with code=YOUR_CODE\n")
			fmt.Println(url)

			srv := &http.Server{Handler: NewAuthorizeHandler(transport, donech)}
			go srv.Serve(l)
			<-donech
			l.Close()

			if transport.Token == nil {
				return nil, ErrCodeNeeded
			}
		}
		if transport.Token == nil {
			// Exchange the authorization code for an access token.
			// ("Here's the code you gave the user, now give me a token!")
			transport.Token, err = transport.Exchange(code)
			if err != nil {
				return nil, fmt.Errorf("Exchange: %v", err)
			}
		}
	}
	return &http.Client{Transport: transport}, nil
}

func NewTransport(id, secret string, cache oauth.Cache) (*oauth.Transport, error) {
	if id == "" || secret == "" {
		return nil, errors.New("Client ID and secret is needed!")
	}
	config := &oauth.Config{
		ClientId:     id,
		ClientSecret: secret,
		AuthURL:      "https://accounts.google.com/o/oauth2/auth",
		TokenURL:     "https://accounts.google.com/o/oauth2/token",
		Scope:        picasaScope,
		TokenCache:   cache,
	}
	return &oauth.Transport{Config: config}, nil
}

// NewAuthorizeHandler returns a http.HandlerFunc which will set the Token of
// the given oauth.Transport and send a struct{} on the donech on success.
func NewAuthorizeHandler(transport *oauth.Transport, donech chan<- struct{}) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		token, err := transport.Exchange(r.FormValue("code"))
		if err != nil {
			http.Error(w, fmt.Sprintf("error exchanging code: %v", err), http.StatusBadRequest)
			return
		}
		transport.Token = token
		// The Transport now has a valid Token.
		donech <- struct{}{}
	}
}

func getListener() (*net.TCPListener, error) {
	return net.ListenTCP("tcp", &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: 0})
}
<commit_msg>third_party/picago: remove unneeded oauth helpers<commit_after><|endoftext|>


<commit_before>package types

import "time"

type Report struct {
	Id             string       `json:"id" header:"ID"`
	Year           int          `json:"year" header:"YEAR"`
	Month          time.Month   `json:"month" header:"MONTH"`
	Start_time     time.Time    `json:"start_time" header:"START_TIME"`
	End_time       time.Time    `json:"end_time" header:"END_TIME"`
	Server_seconds float32      `json:"server_seconds" header:"SERVER_SECONDS"`
	Closed         bool         `json:"closed" header:"CLOSED"`
	Li             []Lines      `json:"lines" header:"LINES"`
	Account_group  AccountGroup `json:"account_group" header:"ACCOUNT_GROUP"`
}

type Lines struct {
	Id                string    `json:"_id" header:"ID"`
	Commissioned_at   time.Time `json:"commissioned_at" header:"COMMISSIONED_AT"`
	Decommissioned_at time.Time `json:"decommissioned_at" header:"DECOMMISSIONED_AT"`
	Instance_id       string    `json:"instance_id" header:"INSTANCE_ID"`
	Instance_name     string    `json:"instance_name" header:"INSTANCE_NAME"`
	Instance_fqdn     string    `json:"instance_fqdn" header:"INSTANCE_FQDN"`
	Consumption       float32   `json:"consumption" header:"CONSUMPTION"`
}

type AccountGroup struct {
	Id   string `json:"_id" header:"ID"`
	Name string `json:"name" header:"NAME"`
}
<commit_msg>admin report - hide/rename columns<commit_after>package types

import "time"

// Report holds a report header fields
type Report struct {
	Id             string       `json:"id" header:"REPORT ID"`
	Year           int          `json:"year" header:"YEAR"`
	Month          time.Month   `json:"month" header:"MONTH"`
	Start_time     time.Time    `json:"start_time" header:"START_TIME"`
	End_time       time.Time    `json:"end_time" header:"END_TIME"`
	Server_seconds float32      `json:"server_seconds" header:"SERVER_SECONDS"`
	Closed         bool         `json:"closed" header:"CLOSED"`
	Li             []Lines      `json:"lines" header:"LINES" show:"nolist"`
	Account_group  AccountGroup `json:"account_group" header:"ACCOUNT_GROUP" show:"nolist"`
}

type Lines struct {
	Id                string    `json:"_id" header:"ID"`
	Commissioned_at   time.Time `json:"commissioned_at" header:"COMMISSIONED_AT"`
	Decommissioned_at time.Time `json:"decommissioned_at" header:"DECOMMISSIONED_AT"`
	Instance_id       string    `json:"instance_id" header:"INSTANCE_ID"`
	Instance_name     string    `json:"instance_name" header:"INSTANCE_NAME"`
	Instance_fqdn     string    `json:"instance_fqdn" header:"INSTANCE_FQDN"`
	Consumption       float32   `json:"consumption" header:"CONSUMPTION"`
}

type AccountGroup struct {
	Id   string `json:"_id" header:"ID"`
	Name string `json:"name" header:"NAME"`
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !plan9,!windows

package main

/*
#include <stdint.h>
#include <signal.h>
#include <pthread.h>

volatile int32_t spinlock;

static void *thread1(void *p) {
	(void)p;
	while (spinlock == 0)
		;
	pthread_kill(pthread_self(), SIGPROF);
	spinlock = 0;
	return NULL;
}
__attribute__((constructor)) void issue9456() {
	pthread_t tid;
	pthread_create(&tid, 0, thread1, NULL);
}

void **nullptr;

void *crash(void *p) {
	*nullptr = p;
	return 0;
}

int start_crashing_thread(void) {
	pthread_t tid;
	return pthread_create(&tid, 0, crash, 0);
}
*/
import "C"

import (
	"fmt"
	"os"
	"os/exec"
	"runtime"
	"sync/atomic"
	"time"
	"unsafe"
)

func init() {
	register("CgoExternalThreadSIGPROF", CgoExternalThreadSIGPROF)
	register("CgoExternalThreadSignal", CgoExternalThreadSignal)
}

func CgoExternalThreadSIGPROF() {
	// This test intends to test that sending SIGPROF to foreign threads
	// before we make any cgo call will not abort the whole process, so
	// we cannot make any cgo call here. See https://golang.org/issue/9456.
	atomic.StoreInt32((*int32)(unsafe.Pointer(&C.spinlock)), 1)
	for atomic.LoadInt32((*int32)(unsafe.Pointer(&C.spinlock))) == 1 {
		runtime.Gosched()
	}
	println("OK")
}

func CgoExternalThreadSignal() {
	if len(os.Args) > 2 && os.Args[2] == "crash" {
		i := C.start_crashing_thread()
		if i != 0 {
			fmt.Println("pthread_create failed:", i)
			// Exit with 0 because parent expects us to crash.
			return
		}

		// We should crash immediately, but give it plenty of
		// time before failing (by exiting 0) in case we are
		// running on a slow system.
		time.Sleep(5 * time.Second)
		return
	}

	out, err := exec.Command(os.Args[0], "CgoExternalThreadSignal", "crash").CombinedOutput()
	if err == nil {
		fmt.Println("C signal did not crash as expected\n")
		fmt.Printf("%s\n", out)
		os.Exit(1)
	}

	fmt.Println("OK")
}
<commit_msg>runtime: move printing of extra newline<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !plan9,!windows

package main

/*
#include <stdint.h>
#include <signal.h>
#include <pthread.h>

volatile int32_t spinlock;

static void *thread1(void *p) {
	(void)p;
	while (spinlock == 0)
		;
	pthread_kill(pthread_self(), SIGPROF);
	spinlock = 0;
	return NULL;
}
__attribute__((constructor)) void issue9456() {
	pthread_t tid;
	pthread_create(&tid, 0, thread1, NULL);
}

void **nullptr;

void *crash(void *p) {
	*nullptr = p;
	return 0;
}

int start_crashing_thread(void) {
	pthread_t tid;
	return pthread_create(&tid, 0, crash, 0);
}
*/
import "C"

import (
	"fmt"
	"os"
	"os/exec"
	"runtime"
	"sync/atomic"
	"time"
	"unsafe"
)

func init() {
	register("CgoExternalThreadSIGPROF", CgoExternalThreadSIGPROF)
	register("CgoExternalThreadSignal", CgoExternalThreadSignal)
}

func CgoExternalThreadSIGPROF() {
	// This test intends to test that sending SIGPROF to foreign threads
	// before we make any cgo call will not abort the whole process, so
	// we cannot make any cgo call here. See https://golang.org/issue/9456.
	atomic.StoreInt32((*int32)(unsafe.Pointer(&C.spinlock)), 1)
	for atomic.LoadInt32((*int32)(unsafe.Pointer(&C.spinlock))) == 1 {
		runtime.Gosched()
	}
	println("OK")
}

func CgoExternalThreadSignal() {
	if len(os.Args) > 2 && os.Args[2] == "crash" {
		i := C.start_crashing_thread()
		if i != 0 {
			fmt.Println("pthread_create failed:", i)
			// Exit with 0 because parent expects us to crash.
			return
		}

		// We should crash immediately, but give it plenty of
		// time before failing (by exiting 0) in case we are
		// running on a slow system.
		time.Sleep(5 * time.Second)
		return
	}

	out, err := exec.Command(os.Args[0], "CgoExternalThreadSignal", "crash").CombinedOutput()
	if err == nil {
		fmt.Println("C signal did not crash as expected")
		fmt.Printf("\n%s\n", out)
		os.Exit(1)
	}

	fmt.Println("OK")
}
<|endoftext|>