<commit_before>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package api

// TODO(ericsnow) Eliminate the apiserver dependencies, if possible.

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"strconv"

	"github.com/juju/errors"
	"github.com/juju/loggo"
	"github.com/juju/utils"
	charmresource "gopkg.in/juju/charm.v6-unstable/resource"

	"github.com/juju/juju/apiserver/common"
	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/resource"
)

var logger = loggo.GetLogger("juju.resource.api")

const (
	// HTTPEndpointPattern is the URL path pattern registered with
	// the API server. This includes wildcards (starting with ":") that
	// are converted into URL query values by the pattern mux. Also see
	// apiserver/apiserver.go.
	HTTPEndpointPattern = "/services/:service/resources/:resource"

	// HTTPEndpointPath is the URL path, with substitutions, for
	// a resource request.
	HTTPEndpointPath = "/services/%s/resources/%s"
)

const (
	// ContentTypeRaw is the HTTP content-type value used for raw, unformattedcontent.
	ContentTypeRaw = "application/octet-stream"

	// ContentTypeJSON is the HTTP content-type value used for JSON content.
	ContentTypeJSON = "application/json"
)

// NewEndpointPath returns the API URL path for the identified resource.
func NewEndpointPath(service, name string) string {
	return fmt.Sprintf(HTTPEndpointPath, service, name)
}

// ExtractEndpointDetails pulls the endpoint wildcard values from
// the provided URL.
func ExtractEndpointDetails(url *url.URL) (service, name string) {
	service = url.Query().Get(":service")
	name = url.Query().Get(":resource")
	return service, name
}

// NewHTTPUploadRequest generates a new HTTP request for the given resource.
func NewHTTPUploadRequest(service, name string, r io.ReadSeeker) (*http.Request, error) {
	var st utils.SizeTracker
	sizingReader := io.TeeReader(r, &st)
	fp, err := charmresource.GenerateFingerprint(sizingReader)
	if err != nil {
		return nil, errors.Trace(err)
	}
	if _, err := r.Seek(0, os.SEEK_SET); err != nil {
		return nil, errors.Trace(err)
	}
	size := st.Size()

	method := "PUT"
	// TODO(ericsnow) What about the rest of the URL?
	urlStr := NewEndpointPath(service, name)
	req, err := http.NewRequest(method, urlStr, nil)
	if err != nil {
		return nil, errors.Trace(err)
	}

	req.Header.Set("Content-Type", ContentTypeRaw)
	req.Header.Set("Content-Sha384", fp.String())
	req.Header.Set("Content-Length", fmt.Sprint(size))
	req.ContentLength = size

	return req, nil
}

// ExtractUploadRequest pulls the required info from the HTTP request.
func ExtractUploadRequest(req *http.Request) (service, name string, size int64, _ charmresource.Fingerprint, _ error) {
	var fp charmresource.Fingerprint

	if req.Header.Get("Content-Length") == "" {
		req.Header.Set("Content-Length", fmt.Sprint(req.ContentLength))
	}

	ctype := req.Header.Get("Content-Type")
	if ctype != ContentTypeRaw {
		return "", "", 0, fp, errors.Errorf("unsupported content type %q", ctype)
	}

	service, name = ExtractEndpointDetails(req.URL)

	fingerprint := req.Header.Get("Content-Sha384") // This parallels "Content-MD5".
	sizeRaw := req.Header.Get("Content-Length")

	fp, err := charmresource.ParseFingerprint(fingerprint)
	if err != nil {
		return "", "", 0, fp, errors.Annotate(err, "invalid fingerprint")
	}

	size, err = strconv.ParseInt(sizeRaw, 10, 64)
	if err != nil {
		return "", "", 0, fp, errors.Annotate(err, "invalid size")
	}

	return service, name, size, fp, nil
}

// NewHTTPDownloadRequest creates a new HTTP download request
// for the given resource.
//
// Intended for use on the client side.
func NewHTTPDownloadRequest(resourceName string) (*http.Request, error) {
	return http.NewRequest("GET", "/resources/"+resourceName, nil)
}

// ExtractDownloadRequest pulls the download request info out of the
// given HTTP request.
//
// Intended for use on the server side.
func ExtractDownloadRequest(req *http.Request) string {
	return req.URL.Query().Get(":resource")
}

// UpdateDownloadResponse sets the appropriate headers in the response
// to an HTTP download request.
//
// Intended for use on the server side.
func UpdateDownloadResponse(resp http.ResponseWriter, resource resource.Resource) {
	resp.Header().Set("Content-Type", ContentTypeRaw)
	resp.Header().Set("Content-Length", fmt.Sprint(resource.Size))
	resp.Header().Set("Content-Sha384", resource.Fingerprint.String())
}

// ExtractDownloadResponse pulls the download size and checksum
// from the HTTP response.
func ExtractDownloadResponse(resp *http.Response) (int64, charmresource.Fingerprint, error) {
	var fp charmresource.Fingerprint

	// TODO(ericsnow) Finish!
	return 0, fp, errors.New("not finished")
}

// TODO(ericsnow) These are copied from apiserver/httpcontext.go...

// SendHTTPError sends a JSON-encoded error response
// for errors encountered during processing.
func SendHTTPError(w http.ResponseWriter, err error) {
	err1, statusCode := common.ServerErrorAndStatus(err)
	logger.Debugf("sending error: %d %v", statusCode, err1)
	SendHTTPStatusAndJSON(w, statusCode, &params.ErrorResult{
		Error: err1,
	})
}

// SendStatusAndJSON sends an HTTP status code and
// a JSON-encoded response to a client.
func SendHTTPStatusAndJSON(w http.ResponseWriter, statusCode int, response interface{}) {
	body, err := json.Marshal(response)
	if err != nil {
		logger.Errorf("cannot marshal JSON result %#v: %v", response, err)
		return
	}

	if statusCode == http.StatusUnauthorized {
		w.Header().Set("WWW-Authenticate", `Basic realm="juju"`)
	}
	w.Header().Set("Content-Type", params.ContentTypeJSON)
	w.Header().Set("Content-Length", fmt.Sprint(len(body)))
	w.WriteHeader(statusCode)
	w.Write(body)
}
<commit_msg>Expand a TODO comment.<commit_after>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package api

// TODO(ericsnow) Eliminate the apiserver dependencies, if possible.

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"strconv"

	"github.com/juju/errors"
	"github.com/juju/loggo"
	"github.com/juju/utils"
	charmresource "gopkg.in/juju/charm.v6-unstable/resource"

	"github.com/juju/juju/apiserver/common"
	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/resource"
)

var logger = loggo.GetLogger("juju.resource.api")

const (
	// HTTPEndpointPattern is the URL path pattern registered with
	// the API server. This includes wildcards (starting with ":") that
	// are converted into URL query values by the pattern mux. Also see
	// apiserver/apiserver.go.
	HTTPEndpointPattern = "/services/:service/resources/:resource"

	// HTTPEndpointPath is the URL path, with substitutions, for
	// a resource request.
	HTTPEndpointPath = "/services/%s/resources/%s"
)

const (
	// ContentTypeRaw is the HTTP content-type value used for raw, unformattedcontent.
	ContentTypeRaw = "application/octet-stream"

	// ContentTypeJSON is the HTTP content-type value used for JSON content.
	ContentTypeJSON = "application/json"
)

// NewEndpointPath returns the API URL path for the identified resource.
func NewEndpointPath(service, name string) string {
	return fmt.Sprintf(HTTPEndpointPath, service, name)
}

// ExtractEndpointDetails pulls the endpoint wildcard values from
// the provided URL.
func ExtractEndpointDetails(url *url.URL) (service, name string) {
	service = url.Query().Get(":service")
	name = url.Query().Get(":resource")
	return service, name
}

// NewHTTPUploadRequest generates a new HTTP request for the given resource.
func NewHTTPUploadRequest(service, name string, r io.ReadSeeker) (*http.Request, error) {
	var st utils.SizeTracker
	sizingReader := io.TeeReader(r, &st)
	fp, err := charmresource.GenerateFingerprint(sizingReader)
	if err != nil {
		return nil, errors.Trace(err)
	}
	if _, err := r.Seek(0, os.SEEK_SET); err != nil {
		return nil, errors.Trace(err)
	}
	size := st.Size()

	method := "PUT"
	// TODO(ericsnow) What about the rest of the URL?
	urlStr := NewEndpointPath(service, name)
	req, err := http.NewRequest(method, urlStr, nil)
	if err != nil {
		return nil, errors.Trace(err)
	}

	req.Header.Set("Content-Type", ContentTypeRaw)
	req.Header.Set("Content-Sha384", fp.String())
	req.Header.Set("Content-Length", fmt.Sprint(size))
	req.ContentLength = size

	return req, nil
}

// ExtractUploadRequest pulls the required info from the HTTP request.
func ExtractUploadRequest(req *http.Request) (service, name string, size int64, _ charmresource.Fingerprint, _ error) {
	var fp charmresource.Fingerprint

	if req.Header.Get("Content-Length") == "" {
		req.Header.Set("Content-Length", fmt.Sprint(req.ContentLength))
	}

	ctype := req.Header.Get("Content-Type")
	if ctype != ContentTypeRaw {
		return "", "", 0, fp, errors.Errorf("unsupported content type %q", ctype)
	}

	service, name = ExtractEndpointDetails(req.URL)

	fingerprint := req.Header.Get("Content-Sha384") // This parallels "Content-MD5".
	sizeRaw := req.Header.Get("Content-Length")

	fp, err := charmresource.ParseFingerprint(fingerprint)
	if err != nil {
		return "", "", 0, fp, errors.Annotate(err, "invalid fingerprint")
	}

	size, err = strconv.ParseInt(sizeRaw, 10, 64)
	if err != nil {
		return "", "", 0, fp, errors.Annotate(err, "invalid size")
	}

	return service, name, size, fp, nil
}

// NewHTTPDownloadRequest creates a new HTTP download request
// for the given resource.
//
// Intended for use on the client side.
func NewHTTPDownloadRequest(resourceName string) (*http.Request, error) {
	return http.NewRequest("GET", "/resources/"+resourceName, nil)
}

// ExtractDownloadRequest pulls the download request info out of the
// given HTTP request.
//
// Intended for use on the server side.
func ExtractDownloadRequest(req *http.Request) string {
	return req.URL.Query().Get(":resource")
}

// UpdateDownloadResponse sets the appropriate headers in the response
// to an HTTP download request.
//
// Intended for use on the server side.
func UpdateDownloadResponse(resp http.ResponseWriter, resource resource.Resource) {
	resp.Header().Set("Content-Type", ContentTypeRaw)
	resp.Header().Set("Content-Length", fmt.Sprint(resource.Size))
	resp.Header().Set("Content-Sha384", resource.Fingerprint.String())
}

// ExtractDownloadResponse pulls the download size and checksum
// from the HTTP response.
func ExtractDownloadResponse(resp *http.Response) (int64, charmresource.Fingerprint, error) {
	var fp charmresource.Fingerprint

	// TODO(ericsnow) Finish!
	// See UpdateDownloadResponse for the data to extract.
	return 0, fp, errors.New("not finished")
}

// TODO(ericsnow) These are copied from apiserver/httpcontext.go...

// SendHTTPError sends a JSON-encoded error response
// for errors encountered during processing.
func SendHTTPError(w http.ResponseWriter, err error) {
	err1, statusCode := common.ServerErrorAndStatus(err)
	logger.Debugf("sending error: %d %v", statusCode, err1)
	SendHTTPStatusAndJSON(w, statusCode, &params.ErrorResult{
		Error: err1,
	})
}

// SendStatusAndJSON sends an HTTP status code and
// a JSON-encoded response to a client.
func SendHTTPStatusAndJSON(w http.ResponseWriter, statusCode int, response interface{}) {
	body, err := json.Marshal(response)
	if err != nil {
		logger.Errorf("cannot marshal JSON result %#v: %v", response, err)
		return
	}

	if statusCode == http.StatusUnauthorized {
		w.Header().Set("WWW-Authenticate", `Basic realm="juju"`)
	}
	w.Header().Set("Content-Type", params.ContentTypeJSON)
	w.Header().Set("Content-Length", fmt.Sprint(len(body)))
	w.WriteHeader(statusCode)
	w.Write(body)
}
<|endoftext|>


<commit_before>package reverse_test

import (
	"bufio"

	"code.google.com/p/rog-go/reverse"

	"reflect"
	"strings"
	"testing"
)

var scanTests = []struct {
	text   string
	split  bufio.SplitFunc
	tokens []string
}{{
	text: `
hello one two three four five six seven
there

you one two three four five six seven` + "\r" + `
x

`,
	split: bufio.ScanLines,
	tokens: []string{
		"",
		"x",
		"you one two three four five six seven",
		"",
		"there",
		"hello one two three four five six seven",
		"",
	},
}, {
	text: `
hello one two three four five six seven
there

you one two three four five six seven` + "\r" + `
x

`,
	split: bufio.ScanWords,
	tokens: []string{
		"x",
		"seven",
		"six",
		"five",
		"four",
		"three",
		"two",
		"one",
		"you",
		"there",
		"seven",
		"six",
		"five",
		"four",
		"three",
		"two",
		"one",
		"hello",
	},
}}

func TestScan(t *testing.T) {
	for i, test := range scanTests {
		t.Logf("test %d", i)
		b := reverse.NewScanner(strings.NewReader(test.text))
		b.Split(test.split)
		var got []string
		for b.Scan() {
			got = append(got, b.Text())
		}
		if b.Err() != nil {
			t.Fatalf("error after scan: %v", b.Err())
		}
		if !reflect.DeepEqual(got, test.tokens) {
			t.Fatalf("token mismatch; got %q want %q", got, test.tokens)
		}
	}
}
<commit_msg>reverse: add comment<commit_after>package reverse_test

import (
	"bufio"

	"code.google.com/p/rog-go/reverse"

	"reflect"
	"strings"
	"testing"
)

// TODO much more comprehensive tests!

var scanTests = []struct {
	text   string
	split  bufio.SplitFunc
	tokens []string
}{{
	text: `
hello one two three four five six seven
there

you one two three four five six seven` + "\r" + `
x

`,
	split: bufio.ScanLines,
	tokens: []string{
		"",
		"x",
		"you one two three four five six seven",
		"",
		"there",
		"hello one two three four five six seven",
		"",
	},
}, {
	text: `
hello one two three four five six seven
there

you one two three four five six seven` + "\r" + `
x

`,
	split: bufio.ScanWords,
	tokens: []string{
		"x",
		"seven",
		"six",
		"five",
		"four",
		"three",
		"two",
		"one",
		"you",
		"there",
		"seven",
		"six",
		"five",
		"four",
		"three",
		"two",
		"one",
		"hello",
	},
}}

func TestScan(t *testing.T) {
	for i, test := range scanTests {
		t.Logf("test %d", i)
		b := reverse.NewScanner(strings.NewReader(test.text))
		b.Split(test.split)
		var got []string
		for b.Scan() {
			got = append(got, b.Text())
		}
		if b.Err() != nil {
			t.Fatalf("error after scan: %v", b.Err())
		}
		if !reflect.DeepEqual(got, test.tokens) {
			t.Fatalf("token mismatch; got %q want %q", got, test.tokens)
		}
	}
}
<|endoftext|>


<commit_before>package registry

import (
	"path"
	"strings"
	"time"

	"github.com/coreos/go-etcd/etcd"
	log "github.com/golang/glog"

	"github.com/coreos/coreinit/event"
)

type EventStream struct {
	etcd *etcd.Client
	stop chan bool
}

func NewEventStream(client *etcd.Client) *EventStream {
	return &EventStream{client, make(chan bool)}
}

func (self *EventStream) Stream(eventchan chan *event.Event) {
	watchMap := map[string][]func(*etcd.Response) *event.Event{
		path.Join(keyPrefix, statePrefix):   []func(*etcd.Response) *event.Event{filterEventJobStatePublished, filterEventJobStateExpired},
		path.Join(keyPrefix, jobPrefix):     []func(*etcd.Response) *event.Event{filterEventJobCreated, filterEventJobScheduled, filterEventJobCancelled},
		path.Join(keyPrefix, machinePrefix): []func(*etcd.Response) *event.Event{self.filterEventMachineUpdated, self.filterEventMachineRemoved},
		path.Join(keyPrefix, requestPrefix): []func(*etcd.Response) *event.Event{filterEventRequestCreated},
		path.Join(keyPrefix, offerPrefix):   []func(*etcd.Response) *event.Event{self.filterEventJobOffered, filterEventJobBidSubmitted},
	}

	for key, funcs := range watchMap {
		for _, f := range funcs {
			etcdchan := make(chan *etcd.Response)
			go watch(self.etcd, etcdchan, key, self.stop)
			go pipe(etcdchan, f, eventchan, self.stop)
		}
	}
}

func pipe(etcdchan chan *etcd.Response, translate func(resp *etcd.Response) *event.Event, eventchan chan *event.Event, stopchan chan bool) {
	for true {
		select {
		case <-stopchan:
			return
		case resp := <-etcdchan:
			log.V(2).Infof("Received response from etcd watcher: Action=%s ModifiedIndex=%d Key=%s", resp.Action, resp.Node.ModifiedIndex, resp.Node.Key)
			ev := translate(resp)
			if ev != nil {
				log.V(2).Infof("Translated response(ModifiedIndex=%d) to event(Type=%s)", resp.Node.ModifiedIndex, ev.Type)
				eventchan <- ev
			} else {
				log.V(2).Infof("Discarding response(ModifiedIndex=%d) from etcd watcher", resp.Node.ModifiedIndex)
			}
		}
	}
}

func watch(client *etcd.Client, etcdchan chan *etcd.Response, key string, stopchan chan bool) {
	for true {
		select {
		case <-stopchan:
			log.V(2).Infof("Gracefully closing etcd watcher: key=%s", key)
			return
		default:
			log.V(2).Infof("Creating etcd watcher: key=%s, machines=%s", key, strings.Join(client.GetCluster(), ","))
			_, err := client.Watch(key, 0, true, etcdchan, stopchan)

			if err != nil {
				log.V(2).Infof("etcd watch closed exited: key=%s, err=\"%s\"", key, err.Error())

				// Let's not slam the etcd server in the event that we know
				// an unexpected error occurred.
				time.Sleep(time.Second)
			}
		}
	}
}
<commit_msg>feat(registry): track last-seen index in registry.watch<commit_after>package registry

import (
	"path"
	"strings"
	"time"

	"github.com/coreos/go-etcd/etcd"
	log "github.com/golang/glog"

	"github.com/coreos/coreinit/event"
)

type EventStream struct {
	etcd *etcd.Client
	stop chan bool
}

func NewEventStream(client *etcd.Client) *EventStream {
	return &EventStream{client, make(chan bool)}
}

func (self *EventStream) Stream(eventchan chan *event.Event) {
	watchMap := map[string][]func(*etcd.Response) *event.Event{
		path.Join(keyPrefix, statePrefix):   []func(*etcd.Response) *event.Event{filterEventJobStatePublished, filterEventJobStateExpired},
		path.Join(keyPrefix, jobPrefix):     []func(*etcd.Response) *event.Event{filterEventJobCreated, filterEventJobScheduled, filterEventJobCancelled},
		path.Join(keyPrefix, machinePrefix): []func(*etcd.Response) *event.Event{self.filterEventMachineUpdated, self.filterEventMachineRemoved},
		path.Join(keyPrefix, requestPrefix): []func(*etcd.Response) *event.Event{filterEventRequestCreated},
		path.Join(keyPrefix, offerPrefix):   []func(*etcd.Response) *event.Event{self.filterEventJobOffered, filterEventJobBidSubmitted},
	}

	for key, funcs := range watchMap {
		for _, f := range funcs {
			etcdchan := make(chan *etcd.Response)
			go watch(self.etcd, etcdchan, key, self.stop)
			go pipe(etcdchan, f, eventchan, self.stop)
		}
	}
}

func pipe(etcdchan chan *etcd.Response, translate func(resp *etcd.Response) *event.Event, eventchan chan *event.Event, stopchan chan bool) {
	for true {
		select {
		case <-stopchan:
			return
		case resp := <-etcdchan:
			log.V(2).Infof("Received response from etcd watcher: Action=%s ModifiedIndex=%d Key=%s", resp.Action, resp.Node.ModifiedIndex, resp.Node.Key)
			ev := translate(resp)
			if ev != nil {
				log.V(2).Infof("Translated response(ModifiedIndex=%d) to event(Type=%s)", resp.Node.ModifiedIndex, ev.Type)
				eventchan <- ev
			} else {
				log.V(2).Infof("Discarding response(ModifiedIndex=%d) from etcd watcher", resp.Node.ModifiedIndex)
			}
		}
	}
}

func watch(client *etcd.Client, etcdchan chan *etcd.Response, key string, stopchan chan bool) {
	idx := uint64(0)
	for true {
		select {
		case <-stopchan:
			log.V(2).Infof("Gracefully closing etcd watch loop: key=%s", key)
			return
		default:
			log.V(2).Infof("Creating etcd watcher: key=%s, machines=%s", key, strings.Join(client.GetCluster(), ","))
			resp, err := client.Watch(key, idx, true, nil, nil)

			if err == nil {
				idx = resp.Node.ModifiedIndex + 1
				etcdchan <- resp
			} else {
				log.V(2).Infof("etcd watcher returned error: key=%s, err=\"%s\"", key, err.Error())

				// Let's not slam the etcd server in the event that we know
				// an unexpected error occurred.
				time.Sleep(time.Second)
			}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package google

import (
	"path"

	"github.com/juju/errors"
	"google.golang.org/api/compute/v1"
)

// addInstance sends a request to GCE to add a new instance to the
// connection's project, with the provided instance data and machine
// type. Each of the provided zones is attempted and the first available
// zone is where the instance is provisioned. If no zones are available
// then an error is returned. The instance that was passed in is updated
// with the new instance's data upon success. The call blocks until the
// instance is created or the request fails.
// TODO(ericsnow) Return a new inst.
func (gce *Connection) addInstance(requestedInst *compute.Instance, machineType string, zones []string) error {
	for _, zoneName := range zones {
		var waitErr error
		inst := *requestedInst
		inst.MachineType = formatMachineType(zoneName, machineType)
		err := gce.raw.AddInstance(gce.projectID, zoneName, &inst)
		if isWaitError(err) {
			waitErr = err
		} else if err != nil {
			// We are guaranteed the insert failed at the point.
			return errors.Annotate(err, "sending new instance request")
		}

		// Check if the instance was created.
		realized, err := gce.raw.GetInstance(gce.projectID, zoneName, inst.Name)
		if err != nil {
			if waitErr == nil {
				return errors.Trace(err)
			}
			// Try the next zone.
			logger.Errorf("failed to get new instance in zone %q: %v", zoneName, waitErr)
			continue
		}

		// Success!
		*requestedInst = *realized
		return nil
	}
	return errors.Errorf("not able to provision in any zone")
}

// AddInstance creates a new instance based on the spec's data and
// returns it. The instance will be created using the provided
// connection and in one of the provided zones.
func (gce *Connection) AddInstance(spec InstanceSpec, zones ...string) (*Instance, error) {
	raw := spec.raw()
	if err := gce.addInstance(raw, spec.Type, zones); err != nil {
		return nil, errors.Trace(err)
	}

	return newInstance(raw, &spec), nil
}

// Instance gets the up-to-date info about the given instance
// and returns it.
func (gce *Connection) Instance(id, zone string) (Instance, error) {
	var result Instance
	raw, err := gce.raw.GetInstance(gce.projectID, zone, id)
	if err != nil {
		return result, errors.Trace(err)
	}
	result = *newInstance(raw, nil)
	return result, nil
}

// Instances sends a request to the GCE API for a list of all instances
// (in the Connection's project) for which the name starts with the
// provided prefix. The result is also limited to those instances with
// one of the specified statuses (if any).
func (gce *Connection) Instances(prefix string, statuses ...string) ([]Instance, error) {
	rawInsts, err := gce.raw.ListInstances(gce.projectID, prefix, statuses...)
	if err != nil {
		return nil, errors.Trace(err)
	}

	var insts []Instance
	for _, rawInst := range rawInsts {
		inst := newInstance(rawInst, nil)
		insts = append(insts, *inst)
	}
	return insts, nil
}

// removeInstance sends a request to the GCE API to remove the instance
// with the provided ID (in the specified zone). The call blocks until
// the instance is removed (or the request fails).
func (gce *Connection) removeInstance(id, zone string) error {
	err := gce.raw.RemoveInstance(gce.projectID, zone, id)
	if err != nil {
		// TODO(ericsnow) Try removing the firewall anyway?
		return errors.Trace(err)
	}

	fwname := id
	err = gce.raw.RemoveFirewall(gce.projectID, fwname)
	if errors.IsNotFound(err) {
		return nil
	}
	return errors.Trace(err)
}

// RemoveInstances sends a request to the GCE API to terminate all
// instances (in the Connection's project) that match one of the
// provided IDs. If a prefix is provided, only IDs that start with the
// prefix will be considered. The call blocks until all the instances
// are removed or the request fails.
func (gce *Connection) RemoveInstances(prefix string, ids ...string) error {
	if len(ids) == 0 {
		return nil
	}

	instances, err := gce.Instances(prefix)
	if err != nil {
		return errors.Annotatef(err, "while removing instances %v", ids)
	}

	// TODO(ericsnow) Remove instances in parallel?
	var failed []string
	for _, instID := range ids {
		for _, inst := range instances {
			if inst.ID == instID {
				zoneName := path.Base(inst.InstanceSummary.ZoneName)
				if err := gce.removeInstance(instID, zoneName); err != nil {
					failed = append(failed, instID)
					logger.Errorf("while removing instance %q: %v", instID, err)
				}
				break
			}
		}
	}
	if len(failed) != 0 {
		return errors.Errorf("some instance removals failed: %v", failed)
	}
	return nil
}
<commit_msg>Be more explicit about the early-exit.<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package google

import (
	"path"

	"github.com/juju/errors"
	"google.golang.org/api/compute/v1"
)

// addInstance sends a request to GCE to add a new instance to the
// connection's project, with the provided instance data and machine
// type. Each of the provided zones is attempted and the first available
// zone is where the instance is provisioned. If no zones are available
// then an error is returned. The instance that was passed in is updated
// with the new instance's data upon success. The call blocks until the
// instance is created or the request fails.
// TODO(ericsnow) Return a new inst.
func (gce *Connection) addInstance(requestedInst *compute.Instance, machineType string, zones []string) error {
	for _, zoneName := range zones {
		var waitErr error
		inst := *requestedInst
		inst.MachineType = formatMachineType(zoneName, machineType)
		err := gce.raw.AddInstance(gce.projectID, zoneName, &inst)
		if isWaitError(err) {
			waitErr = err
		} else if err != nil {
			// We are guaranteed the insert failed at the point.
			return errors.Annotate(err, "sending new instance request")
		}

		// Check if the instance was created.
		realized, err := gce.raw.GetInstance(gce.projectID, zoneName, inst.Name)
		if err != nil {
			if waitErr == nil {
				return errors.Trace(err)
			}
			// Try the next zone.
			logger.Errorf("failed to get new instance in zone %q: %v", zoneName, waitErr)
			continue
		}

		// Success!
		*requestedInst = *realized
		return nil
	}
	return errors.Errorf("not able to provision in any zone")
}

// AddInstance creates a new instance based on the spec's data and
// returns it. The instance will be created using the provided
// connection and in one of the provided zones.
func (gce *Connection) AddInstance(spec InstanceSpec, zones ...string) (*Instance, error) {
	raw := spec.raw()
	if err := gce.addInstance(raw, spec.Type, zones); err != nil {
		return nil, errors.Trace(err)
	}

	return newInstance(raw, &spec), nil
}

// Instance gets the up-to-date info about the given instance
// and returns it.
func (gce *Connection) Instance(id, zone string) (Instance, error) {
	var result Instance
	raw, err := gce.raw.GetInstance(gce.projectID, zone, id)
	if err != nil {
		return result, errors.Trace(err)
	}
	result = *newInstance(raw, nil)
	return result, nil
}

// Instances sends a request to the GCE API for a list of all instances
// (in the Connection's project) for which the name starts with the
// provided prefix. The result is also limited to those instances with
// one of the specified statuses (if any).
func (gce *Connection) Instances(prefix string, statuses ...string) ([]Instance, error) {
	rawInsts, err := gce.raw.ListInstances(gce.projectID, prefix, statuses...)
	if err != nil {
		return nil, errors.Trace(err)
	}

	var insts []Instance
	for _, rawInst := range rawInsts {
		inst := newInstance(rawInst, nil)
		insts = append(insts, *inst)
	}
	return insts, nil
}

// removeInstance sends a request to the GCE API to remove the instance
// with the provided ID (in the specified zone). The call blocks until
// the instance is removed (or the request fails).
func (gce *Connection) removeInstance(id, zone string) error {
	err := gce.raw.RemoveInstance(gce.projectID, zone, id)
	if err != nil {
		// TODO(ericsnow) Try removing the firewall anyway?
		return errors.Trace(err)
	}

	fwname := id
	err = gce.raw.RemoveFirewall(gce.projectID, fwname)
	if errors.IsNotFound(err) {
		return nil
	}
	if err != nil {
		return errors.Trace(err)
	}
	return nil
}

// RemoveInstances sends a request to the GCE API to terminate all
// instances (in the Connection's project) that match one of the
// provided IDs. If a prefix is provided, only IDs that start with the
// prefix will be considered. The call blocks until all the instances
// are removed or the request fails.
func (gce *Connection) RemoveInstances(prefix string, ids ...string) error {
	if len(ids) == 0 {
		return nil
	}

	instances, err := gce.Instances(prefix)
	if err != nil {
		return errors.Annotatef(err, "while removing instances %v", ids)
	}

	// TODO(ericsnow) Remove instances in parallel?
	var failed []string
	for _, instID := range ids {
		for _, inst := range instances {
			if inst.ID == instID {
				zoneName := path.Base(inst.InstanceSummary.ZoneName)
				if err := gce.removeInstance(instID, zoneName); err != nil {
					failed = append(failed, instID)
					logger.Errorf("while removing instance %q: %v", instID, err)
				}
				break
			}
		}
	}
	if len(failed) != 0 {
		return errors.Errorf("some instance removals failed: %v", failed)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"

	"github.com/sashajeltuhin/ket/provision/openstack"
)

func main() {
	fmt.Println("Listening on port 8013")
	http.HandleFunc("/nodeup/etcd", openstack.NodeUp)
	http.HandleFunc("/nodeup/etcd", openstack.ProvisionAndInstall)
	err := http.ListenAndServe(":8013", nil)
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}
<commit_msg>Fixed dup routes<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"

	"github.com/sashajeltuhin/ket/provision/openstack"
)

func main() {
	fmt.Println("Listening on port 8013")
	http.HandleFunc("/nodeup", openstack.NodeUp)
	http.HandleFunc("/install", openstack.ProvisionAndInstall)
	err := http.ListenAndServe(":8013", nil)
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}
<|endoftext|>


<commit_before>package ams

import (
	"bytes"
	"encoding/json"
	"io"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/pkg/errors"
)

type requestOptions struct {
	Body   io.Reader
	Header http.Header
	Params url.Values
}

type requestOption func(*requestOptions) error

func composeOptions(opts ...requestOption) requestOption {
	return func(option *requestOptions) error {
		for _, opt := range opts {
			if err := opt(option); err != nil {
				return err
			}
		}
		return nil
	}
}

func defaultRequestOption() *requestOptions {
	option := &requestOptions{
		Header: http.Header{},
		Params: url.Values{},
	}

	option.Header.Set("x-ms-version", APIVersion)
	withOData(false)(option)

	return option
}

func defaultStorageRequestOption() *requestOptions {
	option := &requestOptions{
		Header: http.Header{},
		Params: url.Values{},
	}
	option.Header.Set("x-ms-version", StorageAPIVersion)
	option.Header.Set("Date", formatTime(time.Now()))

	return option
}

func withDataServiceVersion(option *requestOptions) error {
	option.Header.Set("DataServiceVersion", DataServiceVersion)
	option.Header.Set("MaxDataServiceVersion", MaxDataServiceVersion)
	return nil
}

func withCustomHeader(key, value string) requestOption {
	return func(option *requestOptions) error {
		option.Header.Set(key, value)
		return nil
	}
}

func withQuery(params url.Values) requestOption {
	return func(option *requestOptions) error {
		for k, vs := range params {
			for _, v := range vs {
				option.Params.Add(k, v)
			}
		}
		return nil
	}
}

func withForm(params url.Values) requestOption {
	return func(option *requestOptions) error {
		option.Body = strings.NewReader(params.Encode())
		return nil
	}
}

func withBody(body io.Reader) requestOption {
	return func(option *requestOptions) error {
		option.Body = body
		return nil
	}
}

func withJSON(data interface{}) requestOption {
	return func(option *requestOptions) error {
		encoded, err := json.Marshal(data)
		if err != nil {
			return errors.Wrap(err, "json marshal failed")
		}
		option.Body = bytes.NewReader(encoded)
		return nil
	}
}

func withBytes(b []byte) requestOption {
	return func(option *requestOptions) error {
		option.Body = bytes.NewReader(b)
		return nil
	}
}

func withContentType(mimeType string) requestOption {
	return func(option *requestOptions) error {
		option.Header.Set("Content-Type", mimeType)
		return nil
	}
}

func setAccept(mimeType string) requestOption {
	return func(option *requestOptions) error {
		option.Header.Set("Accept", mimeType)
		return nil
	}
}

func withOData(verbose bool) requestOption {
	contentType := "application/json"
	accept := "application/json"
	if verbose {
		contentType += ";odata=verbose"
		accept += ";odata=verbose"
	}
	return composeOptions(withContentType(contentType), setAccept(accept), withDataServiceVersion)
}
<commit_msg>refactor: delete unused function<commit_after>package ams

import (
	"bytes"
	"encoding/json"
	"io"
	"net/http"
	"net/url"
	"time"

	"github.com/pkg/errors"
)

type requestOptions struct {
	Body   io.Reader
	Header http.Header
	Params url.Values
}

type requestOption func(*requestOptions) error

func composeOptions(opts ...requestOption) requestOption {
	return func(option *requestOptions) error {
		for _, opt := range opts {
			if err := opt(option); err != nil {
				return err
			}
		}
		return nil
	}
}

func defaultRequestOption() *requestOptions {
	option := &requestOptions{
		Header: http.Header{},
		Params: url.Values{},
	}

	option.Header.Set("x-ms-version", APIVersion)
	withOData(false)(option)

	return option
}

func defaultStorageRequestOption() *requestOptions {
	option := &requestOptions{
		Header: http.Header{},
		Params: url.Values{},
	}
	option.Header.Set("x-ms-version", StorageAPIVersion)
	option.Header.Set("Date", formatTime(time.Now()))

	return option
}

func withDataServiceVersion(option *requestOptions) error {
	option.Header.Set("DataServiceVersion", DataServiceVersion)
	option.Header.Set("MaxDataServiceVersion", MaxDataServiceVersion)
	return nil
}

func withCustomHeader(key, value string) requestOption {
	return func(option *requestOptions) error {
		option.Header.Set(key, value)
		return nil
	}
}

func withQuery(params url.Values) requestOption {
	return func(option *requestOptions) error {
		for k, vs := range params {
			for _, v := range vs {
				option.Params.Add(k, v)
			}
		}
		return nil
	}
}

func withBody(body io.Reader) requestOption {
	return func(option *requestOptions) error {
		option.Body = body
		return nil
	}
}

func withJSON(data interface{}) requestOption {
	return func(option *requestOptions) error {
		encoded, err := json.Marshal(data)
		if err != nil {
			return errors.Wrap(err, "json marshal failed")
		}
		option.Body = bytes.NewReader(encoded)
		return nil
	}
}

func withBytes(b []byte) requestOption {
	return func(option *requestOptions) error {
		option.Body = bytes.NewReader(b)
		return nil
	}
}

func withContentType(mimeType string) requestOption {
	return func(option *requestOptions) error {
		option.Header.Set("Content-Type", mimeType)
		return nil
	}
}

func setAccept(mimeType string) requestOption {
	return func(option *requestOptions) error {
		option.Header.Set("Accept", mimeType)
		return nil
	}
}

func withOData(verbose bool) requestOption {
	contentType := "application/json"
	accept := "application/json"
	if verbose {
		contentType += ";odata=verbose"
		accept += ";odata=verbose"
	}
	return composeOptions(withContentType(contentType), setAccept(accept), withDataServiceVersion)
}
<|endoftext|>


<commit_before>/*
 *
 * Copyright 2017 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

// Package encoding defines the interface for the compressor and codec, and
// functions to register and retrieve compressors and codecs.
//
// Experimental
//
// Notice: This package is EXPERIMENTAL and may be changed or removed in a
// later release.
package encoding

import (
	"io"
	"strings"
)

// Identity specifies the optional encoding for uncompressed streams.
// It is intended for grpc internal use only.
const Identity = "identity"

// Compressor is used for compressing and decompressing when sending or
// receiving messages.
type Compressor interface {
	// Compress writes the data written to wc to w after compressing it.  If an
	// error occurs while initializing the compressor, that error is returned
	// instead.
	Compress(w io.Writer) (io.WriteCloser, error)
	// Decompress reads data from r, decompresses it, and provides the
	// uncompressed data via the returned io.Reader.  If an error occurs while
	// initializing the decompressor, that error is returned instead.
	Decompress(r io.Reader) (io.Reader, error)
	// Name is the name of the compression codec and is used to set the content
	// coding header.  The result must be static; the result cannot change
	// between calls.
	Name() string
	// If a Compressor implements
	// DecompressedSize(compressedBytes []byte) int, gRPC will call it
	// to determine the size of the buffer allocated for the result of decompression.
	// Return -1 to indicate unknown size.
	//
	// Experimental
	//
	// Notice: This API is EXPERIMENTAL and may be changed or removed in a
	// later release.
}

var registeredCompressor = make(map[string]Compressor)

// RegisterCompressor registers the compressor with gRPC by its name.  It can
// be activated when sending an RPC via grpc.UseCompressor().  It will be
// automatically accessed when receiving a message based on the content coding
// header.  Servers also use it to send a response with the same encoding as
// the request.
//
// NOTE: this function must only be called during initialization time (i.e. in
// an init() function), and is not thread-safe.  If multiple Compressors are
// registered with the same name, the one registered last will take effect.
func RegisterCompressor(c Compressor) {
	registeredCompressor[c.Name()] = c
}

// GetCompressor returns Compressor for the given compressor name.
func GetCompressor(name string) Compressor {
	return registeredCompressor[name]
}

// Codec defines the interface gRPC uses to encode and decode messages.  Note
// that implementations of this interface must be thread safe; a Codec's
// methods can be called from concurrent goroutines.
type Codec interface {
	// Marshal returns the wire format of v.
	Marshal(v interface{}) ([]byte, error)
	// Unmarshal parses the wire format into v.
	Unmarshal(data []byte, v interface{}) error
	// Name returns the name of the Codec implementation. The returned string
	// will be used as part of content type in transmission.  The result must be
	// static; the result cannot change between calls.
	Name() string
}

var registeredCodecs = make(map[string]Codec)

// RegisterCodec registers the provided Codec for use with all gRPC clients and
// servers.
//
// The Codec will be stored and looked up by result of its Name() method, which
// should match the content-subtype of the encoding handled by the Codec.  This
// is case-insensitive, and is stored and looked up as lowercase.  If the
// result of calling Name() is an empty string, RegisterCodec will panic. See
// Content-Type on
// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests for
// more details.
//
// NOTE: this function must only be called during initialization time (i.e. in
// an init() function), and is not thread-safe.  If multiple Compressors are
// registered with the same name, the one registered last will take effect.
func RegisterCodec(codec Codec) {
	if codec == nil {
		panic("cannot register a nil Codec")
	}
	if codec.Name() == "" {
		panic("cannot register Codec with empty string result for Name()")
	}
	contentSubtype := strings.ToLower(codec.Name())
	registeredCodecs[contentSubtype] = codec
}

// GetCodec gets a registered Codec by content-subtype, or nil if no Codec is
// registered for the content-subtype.
//
// The content-subtype is expected to be lowercase.
func GetCodec(contentSubtype string) Codec {
	return registeredCodecs[contentSubtype]
}
<commit_msg>documentation: fix typo in RegisterCodec godoc (#5306)<commit_after>/*
 *
 * Copyright 2017 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

// Package encoding defines the interface for the compressor and codec, and
// functions to register and retrieve compressors and codecs.
//
// Experimental
//
// Notice: This package is EXPERIMENTAL and may be changed or removed in a
// later release.
package encoding

import (
	"io"
	"strings"
)

// Identity specifies the optional encoding for uncompressed streams.
// It is intended for grpc internal use only.
const Identity = "identity"

// Compressor is used for compressing and decompressing when sending or
// receiving messages.
type Compressor interface {
	// Compress writes the data written to wc to w after compressing it.  If an
	// error occurs while initializing the compressor, that error is returned
	// instead.
	Compress(w io.Writer) (io.WriteCloser, error)
	// Decompress reads data from r, decompresses it, and provides the
	// uncompressed data via the returned io.Reader.  If an error occurs while
	// initializing the decompressor, that error is returned instead.
	Decompress(r io.Reader) (io.Reader, error)
	// Name is the name of the compression codec and is used to set the content
	// coding header.  The result must be static; the result cannot change
	// between calls.
	Name() string
	// If a Compressor implements
	// DecompressedSize(compressedBytes []byte) int, gRPC will call it
	// to determine the size of the buffer allocated for the result of decompression.
	// Return -1 to indicate unknown size.
	//
	// Experimental
	//
	// Notice: This API is EXPERIMENTAL and may be changed or removed in a
	// later release.
}

var registeredCompressor = make(map[string]Compressor)

// RegisterCompressor registers the compressor with gRPC by its name.  It can
// be activated when sending an RPC via grpc.UseCompressor().  It will be
// automatically accessed when receiving a message based on the content coding
// header.  Servers also use it to send a response with the same encoding as
// the request.
//
// NOTE: this function must only be called during initialization time (i.e. in
// an init() function), and is not thread-safe.  If multiple Compressors are
// registered with the same name, the one registered last will take effect.
func RegisterCompressor(c Compressor) {
	registeredCompressor[c.Name()] = c
}

// GetCompressor returns Compressor for the given compressor name.
func GetCompressor(name string) Compressor {
	return registeredCompressor[name]
}

// Codec defines the interface gRPC uses to encode and decode messages.  Note
// that implementations of this interface must be thread safe; a Codec's
// methods can be called from concurrent goroutines.
type Codec interface {
	// Marshal returns the wire format of v.
	Marshal(v interface{}) ([]byte, error)
	// Unmarshal parses the wire format into v.
	Unmarshal(data []byte, v interface{}) error
	// Name returns the name of the Codec implementation. The returned string
	// will be used as part of content type in transmission.  The result must be
	// static; the result cannot change between calls.
	Name() string
}

var registeredCodecs = make(map[string]Codec)

// RegisterCodec registers the provided Codec for use with all gRPC clients and
// servers.
//
// The Codec will be stored and looked up by result of its Name() method, which
// should match the content-subtype of the encoding handled by the Codec.  This
// is case-insensitive, and is stored and looked up as lowercase.  If the
// result of calling Name() is an empty string, RegisterCodec will panic. See
// Content-Type on
// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests for
// more details.
//
// NOTE: this function must only be called during initialization time (i.e. in
// an init() function), and is not thread-safe.  If multiple Codecs are
// registered with the same name, the one registered last will take effect.
func RegisterCodec(codec Codec) {
	if codec == nil {
		panic("cannot register a nil Codec")
	}
	if codec.Name() == "" {
		panic("cannot register Codec with empty string result for Name()")
	}
	contentSubtype := strings.ToLower(codec.Name())
	registeredCodecs[contentSubtype] = codec
}

// GetCodec gets a registered Codec by content-subtype, or nil if no Codec is
// registered for the content-subtype.
//
// The content-subtype is expected to be lowercase.
func GetCodec(contentSubtype string) Codec {
	return registeredCodecs[contentSubtype]
}
<|endoftext|>


<commit_before>package mesos_test

import (
	"testing"

	"github.com/mesos/mesos-go"
)

func TestResource_IsEmpty(t *testing.T) {
	for i, tc := range []struct {
		r     *mesos.Resource
		wants bool
	}{
		{nil, true},
		{new(mesos.Resource), true},
		{resource(valueScalar(0)), true},
		{resource(valueSet()), true},
		{resource(valueSet([]string{}...)), true},
		{resource(valueSet()), true},
		{resource(valueSet("")), false},
		{resource(valueRange()), true},
		{resource(valueRange(span(0, 0))), false},
	} {
		actual := tc.r.IsEmpty()
		if tc.wants != actual {
			t.Errorf("test case %d failed: wants (%t) != actual (%t)", i, tc.wants, actual)
		}
	}
}

func TestResources_PlusAll(t *testing.T) {
	for i, tc := range []struct {
		r1, r2      mesos.Resources
		wants       mesos.Resources
		wantsCPU    float64
		wantsMemory uint64
	}{
		{r1: nil, r2: nil, wants: nil},
		{r1: resources(), r2: resources(), wants: resources()},
		// simple scalars, same roles for everything
		{
			r1: resources(
				resource(name("cpus"), valueScalar(1), role("*")),
				resource(name("mem"), valueScalar(5), role("*")),
			),
			r2: resources(
				resource(name("cpus"), valueScalar(2), role("*")),
				resource(name("mem"), valueScalar(10), role("*")),
			),
			wants: resources(
				resource(name("cpus"), valueScalar(3), role("*")),
				resource(name("mem"), valueScalar(15), role("*")),
			),
			wantsCPU:    3,
			wantsMemory: 15,
		},
		// simple scalars, differing roles
		{
			r1: resources(
				resource(name("cpus"), valueScalar(1), role("role1")),
				resource(name("cpus"), valueScalar(3), role("role2")),
			),
			r2: resources(
				resource(name("cpus"), valueScalar(5), role("role1")),
			),
			wants: resources(
				resource(name("cpus"), valueScalar(6), role("role1")),
				resource(name("cpus"), valueScalar(3), role("role2")),
			),
			wantsCPU: 9,
		},
		// ranges addition yields continuous range
		{
			r1: resources(
				resource(name("ports"), valueRange(span(20000, 40000)), role("*")),
			),
			r2: resources(
				resource(name("ports"), valueRange(span(30000, 50000), span(10000, 20000)), role("*")),
			),
			wants: resources(
				resource(name("ports"), valueRange(span(10000, 50000)), role("*")),
			),
		},
		// ranges addition yields a split set of ranges
		{
			r1: resources(
				resource(name("ports"), valueRange(span(1, 10), span(5, 30), span(50, 60)), role("*")),
				resource(name("ports"), valueRange(span(1, 65), span(70, 80)), role("*")),
			),
			wants: resources(
				resource(name("ports"), valueRange(span(1, 65), span(70, 80)), role("*")),
			),
		},
		// ranges addition (composite) yields a continuous range
		{
			r1: resources(
				resource(name("ports"), valueRange(span(1, 2)), role("*")),
				resource(name("ports"), valueRange(span(3, 4)), role("*")),
			),
			r2: resources(
				resource(name("ports"), valueRange(span(7, 8)), role("*")),
				resource(name("ports"), valueRange(span(5, 6)), role("*")),
			),
			wants: resources(
				resource(name("ports"), valueRange(span(1, 8)), role("*")),
			),
		},
		// ranges addition yields a split set of ranges
		{
			r1: resources(
				resource(name("ports"), valueRange(span(1, 4), span(9, 10), span(20, 22), span(26, 30)), role("*")),
			),
			r2: resources(
				resource(name("ports"), valueRange(span(5, 8), span(23, 25)), role("*")),
			),
			wants: resources(
				resource(name("ports"), valueRange(span(1, 10), span(20, 30)), role("*")),
			),
		},
	} {
		backup := tc.r1.Clone()

		// PlusAll preserves the left operand
		actual := tc.r1.PlusAll(tc.r2)
		if !tc.wants.Equivalent(actual) {
			t.Errorf("test case %d failed: wants (%v) != actual (%v)", i, tc.wants, actual)
		}
		if !backup.Equivalent(tc.r1) {
			t.Errorf("test case %d failed: backup (%v) != r1 (%v)", i, backup, tc.r1)
		}

		// AddAll mutates the left operand
		tc.r1.AddAll(tc.r2)
		if !tc.wants.Equivalent(tc.r1) {
			t.Errorf("test case %d failed: wants (%v) != r1 (%v)", i, tc.wants, tc.r1)
		}

		cpus, ok := tc.r1.CPUs()
		if !ok && tc.wantsCPU > 0 {
			t.Errorf("test case %d failed: failed to obtain total CPU resources", i)
		} else if cpus != tc.wantsCPU {
			t.Errorf("test case %d failed: wants cpu (%v) != r1 cpu (%v)", i, tc.wantsCPU, cpus)
		}

		mem, ok := tc.r1.Memory()
		if !ok && tc.wantsMemory > 0 {
			t.Errorf("test case %d failed: failed to obtain total memory resources", i)
		} else if mem != tc.wantsMemory {
			t.Errorf("test case %d failed: wants mem (%v) != r1 mem (%v)", i, tc.wantsMemory, mem)
		}
	}
}

// functional resource modifier
type resourceOpt func(*mesos.Resource)

func resource(opt ...resourceOpt) *mesos.Resource {
	if len(opt) == 0 {
		return nil
	}
	r := &mesos.Resource{}
	for _, f := range opt {
		f(r)
	}
	return r
}

func name(x string) resourceOpt { return func(r *mesos.Resource) { r.Name = x } }
func role(x string) resourceOpt { return func(r *mesos.Resource) { r.Role = &x } }

func valueScalar(x float64) resourceOpt {
	return func(r *mesos.Resource) {
		r.Type = mesos.SCALAR.Enum()
		r.Scalar = &mesos.Value_Scalar{Value: x}
	}
}

func valueSet(x ...string) resourceOpt {
	return func(r *mesos.Resource) {
		r.Type = mesos.SET.Enum()
		r.Set = &mesos.Value_Set{Item: x}
	}
}

type rangeOpt func(*mesos.Ranges)

// "range" is a keyword, so I called this func "span": it naively appends a range to a Ranges collection
func span(bp, ep uint64) rangeOpt {
	return func(rs *mesos.Ranges) {
		*rs = append(*rs, mesos.Value_Range{Begin: bp, End: ep})
	}
}

func valueRange(p ...rangeOpt) resourceOpt {
	return func(r *mesos.Resource) {
		rs := mesos.Ranges(nil)
		for _, f := range p {
			f(&rs)
		}
		r.Type = mesos.RANGES.Enum()
		r.Ranges = r.Ranges.Add(&mesos.Value_Ranges{Range: rs})
	}
}

func resources(r ...*mesos.Resource) (result mesos.Resources) {
	for _, x := range r {
		result.Add(x)
	}
	return result
}
<commit_msg>test resources set addition<commit_after>package mesos_test

import (
	"testing"

	"github.com/mesos/mesos-go"
)

func TestResource_IsEmpty(t *testing.T) {
	for i, tc := range []struct {
		r     *mesos.Resource
		wants bool
	}{
		{nil, true},
		{new(mesos.Resource), true},
		{resource(valueScalar(0)), true},
		{resource(valueSet()), true},
		{resource(valueSet([]string{}...)), true},
		{resource(valueSet()), true},
		{resource(valueSet("")), false},
		{resource(valueRange()), true},
		{resource(valueRange(span(0, 0))), false},
	} {
		actual := tc.r.IsEmpty()
		if tc.wants != actual {
			t.Errorf("test case %d failed: wants (%t) != actual (%t)", i, tc.wants, actual)
		}
	}
}

func TestResources_PlusAll(t *testing.T) {
	for i, tc := range []struct {
		r1, r2      mesos.Resources
		wants       mesos.Resources
		wantsCPU    float64
		wantsMemory uint64
	}{
		{r1: nil, r2: nil, wants: nil},
		{r1: resources(), r2: resources(), wants: resources()},
		// simple scalars, same roles for everything
		{
			r1: resources(
				resource(name("cpus"), valueScalar(1), role("*")),
				resource(name("mem"), valueScalar(5), role("*")),
			),
			r2: resources(
				resource(name("cpus"), valueScalar(2), role("*")),
				resource(name("mem"), valueScalar(10), role("*")),
			),
			wants: resources(
				resource(name("cpus"), valueScalar(3), role("*")),
				resource(name("mem"), valueScalar(15), role("*")),
			),
			wantsCPU:    3,
			wantsMemory: 15,
		},
		// simple scalars, differing roles
		{
			r1: resources(
				resource(name("cpus"), valueScalar(1), role("role1")),
				resource(name("cpus"), valueScalar(3), role("role2")),
			),
			r2: resources(
				resource(name("cpus"), valueScalar(5), role("role1")),
			),
			wants: resources(
				resource(name("cpus"), valueScalar(6), role("role1")),
				resource(name("cpus"), valueScalar(3), role("role2")),
			),
			wantsCPU: 9,
		},
		// ranges addition yields continuous range
		{
			r1: resources(
				resource(name("ports"), valueRange(span(20000, 40000)), role("*")),
			),
			r2: resources(
				resource(name("ports"), valueRange(span(30000, 50000), span(10000, 20000)), role("*")),
			),
			wants: resources(
				resource(name("ports"), valueRange(span(10000, 50000)), role("*")),
			),
		},
		// ranges addition yields a split set of ranges
		{
			r1: resources(
				resource(name("ports"), valueRange(span(1, 10), span(5, 30), span(50, 60)), role("*")),
				resource(name("ports"), valueRange(span(1, 65), span(70, 80)), role("*")),
			),
			wants: resources(
				resource(name("ports"), valueRange(span(1, 65), span(70, 80)), role("*")),
			),
		},
		// ranges addition (composite) yields a continuous range
		{
			r1: resources(
				resource(name("ports"), valueRange(span(1, 2)), role("*")),
				resource(name("ports"), valueRange(span(3, 4)), role("*")),
			),
			r2: resources(
				resource(name("ports"), valueRange(span(7, 8)), role("*")),
				resource(name("ports"), valueRange(span(5, 6)), role("*")),
			),
			wants: resources(
				resource(name("ports"), valueRange(span(1, 8)), role("*")),
			),
		},
		// ranges addition yields a split set of ranges
		{
			r1: resources(
				resource(name("ports"), valueRange(span(1, 4), span(9, 10), span(20, 22), span(26, 30)), role("*")),
			),
			r2: resources(
				resource(name("ports"), valueRange(span(5, 8), span(23, 25)), role("*")),
			),
			wants: resources(
				resource(name("ports"), valueRange(span(1, 10), span(20, 30)), role("*")),
			),
		},
		// set addition
		{
			r1: resources(
				resource(name("disks"), valueSet("sda1", "sda2", "sda3"), role("*")),
			),
			r2: resources(
				resource(name("disks"), valueSet("sda1", "sda2", "sda3", "sda4"), role("*")),
			),
			wants: resources(
				resource(name("disks"), valueSet("sda4", "sda2", "sda1", "sda3"), role("*")),
			),
		},
	} {
		backup := tc.r1.Clone()

		// PlusAll preserves the left operand
		actual := tc.r1.PlusAll(tc.r2)
		if !tc.wants.Equivalent(actual) {
			t.Errorf("test case %d failed: wants (%v) != actual (%v)", i, tc.wants, actual)
		}
		if !backup.Equivalent(tc.r1) {
			t.Errorf("test case %d failed: backup (%v) != r1 (%v)", i, backup, tc.r1)
		}

		// AddAll mutates the left operand
		tc.r1.AddAll(tc.r2)
		if !tc.wants.Equivalent(tc.r1) {
			t.Errorf("test case %d failed: wants (%v) != r1 (%v)", i, tc.wants, tc.r1)
		}

		cpus, ok := tc.r1.CPUs()
		if !ok && tc.wantsCPU > 0 {
			t.Errorf("test case %d failed: failed to obtain total CPU resources", i)
		} else if cpus != tc.wantsCPU {
			t.Errorf("test case %d failed: wants cpu (%v) != r1 cpu (%v)", i, tc.wantsCPU, cpus)
		}

		mem, ok := tc.r1.Memory()
		if !ok && tc.wantsMemory > 0 {
			t.Errorf("test case %d failed: failed to obtain total memory resources", i)
		} else if mem != tc.wantsMemory {
			t.Errorf("test case %d failed: wants mem (%v) != r1 mem (%v)", i, tc.wantsMemory, mem)
		}
	}
}

// functional resource modifier
type resourceOpt func(*mesos.Resource)

func resource(opt ...resourceOpt) *mesos.Resource {
	if len(opt) == 0 {
		return nil
	}
	r := &mesos.Resource{}
	for _, f := range opt {
		f(r)
	}
	return r
}

func name(x string) resourceOpt { return func(r *mesos.Resource) { r.Name = x } }
func role(x string) resourceOpt { return func(r *mesos.Resource) { r.Role = &x } }

func valueScalar(x float64) resourceOpt {
	return func(r *mesos.Resource) {
		r.Type = mesos.SCALAR.Enum()
		r.Scalar = &mesos.Value_Scalar{Value: x}
	}
}

func valueSet(x ...string) resourceOpt {
	return func(r *mesos.Resource) {
		r.Type = mesos.SET.Enum()
		r.Set = &mesos.Value_Set{Item: x}
	}
}

type rangeOpt func(*mesos.Ranges)

// "range" is a keyword, so I called this func "span": it naively appends a range to a Ranges collection
func span(bp, ep uint64) rangeOpt {
	return func(rs *mesos.Ranges) {
		*rs = append(*rs, mesos.Value_Range{Begin: bp, End: ep})
	}
}

func valueRange(p ...rangeOpt) resourceOpt {
	return func(r *mesos.Resource) {
		rs := mesos.Ranges(nil)
		for _, f := range p {
			f(&rs)
		}
		r.Type = mesos.RANGES.Enum()
		r.Ranges = r.Ranges.Add(&mesos.Value_Ranges{Range: rs})
	}
}

func resources(r ...*mesos.Resource) (result mesos.Resources) {
	for _, x := range r {
		result.Add(x)
	}
	return result
}
<|endoftext|>


<commit_before>package rest

import (
	"bufio"
	"context"
	"fmt"
	"io"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"sync/atomic"
	"time"

	"github.com/couchbase/sync_gateway/base"
	"github.com/pkg/errors"
)

var (
	// ErrSGCollectInfoAlreadyRunning is returned if sgcollect_info is already running.
	ErrSGCollectInfoAlreadyRunning = errors.New("already running")
	// ErrSGCollectInfoNotRunning is returned if sgcollect_info is not running.
	ErrSGCollectInfoNotRunning = errors.New("not running")

	validateTicketPattern = regexp.MustCompile(`\d{1,7}`)

	sgPath, sgCollectPath, err = sgCollectPaths()
	sgcollectInstance          = sgCollect{
		status:        base.Uint32Ptr(sgStopped),
		sgPath:        sgPath,
		sgCollectPath: sgCollectPath,
		pathError:     err}
)

const (
	sgStopped uint32 = iota
	sgRunning

	defaultSGUploadHost = "https://uploads.couchbase.com"
)

type sgCollect struct {
	cancel        context.CancelFunc
	status        *uint32
	sgPath        string
	sgCollectPath string
	pathError     error
}

// Start will attempt to start sgcollect_info, if another is not already running.
func (sg *sgCollect) Start(zipFilename string, params sgCollectOptions) error {
	if atomic.LoadUint32(sg.status) == sgRunning {
		return ErrSGCollectInfoAlreadyRunning
	}

	// Return error if there is any failure while obtaining sgCollectPaths.
	if sg.pathError != nil {
		return err
	}

	if params.OutputDirectory == "" {
		// If no output directory specified, default to the configured LogFilePath
		if config != nil && config.Logging != nil && config.Logging.LogFilePath != "" {
			params.OutputDirectory = config.Logging.LogFilePath
			base.Debugf(base.KeyAdmin, "sgcollect_info: no output directory specified, using LogFilePath: %v", params.OutputDirectory)
		} else {
			// If LogFilePath is not set, and DefaultLogFilePath is not set via a service script, error out.
			return errors.New("no output directory or LogFilePath specified")
		}

		// Validate the path, just in case were not getting it correctly.
		if err := validateOutputDirectory(params.OutputDirectory); err != nil {
			return err
		}
	}

	zipPath := filepath.Join(params.OutputDirectory, zipFilename)

	args := params.Args()
	args = append(args, "--sync-gateway-executable", sgPath)
	args = append(args, zipPath)

	ctx, cancelFunc := context.WithCancel(context.Background())
	sg.cancel = cancelFunc
	cmd := exec.CommandContext(ctx, sgCollectPath, args...)

	// Send command stderr/stdout to pipes
	stderrPipeReader, stderrPipeWriter := io.Pipe()
	cmd.Stderr = stderrPipeWriter
	stdoutPipeReader, stdoutpipeWriter := io.Pipe()
	cmd.Stdout = stdoutpipeWriter

	if err := cmd.Start(); err != nil {
		return err
	}

	atomic.StoreUint32(sg.status, sgRunning)
	startTime := time.Now()
	base.Infof(base.KeyAdmin, "sgcollect_info started with args: %v", base.UD(args))

	// Stream sgcollect_info stderr to warn logs
	go func() {
		scanner := bufio.NewScanner(stderrPipeReader)
		for scanner.Scan() {
			base.Warnf("sgcollect_info: %v", scanner.Text())
		}
		if err := scanner.Err(); err != nil {
			base.Errorf("sgcollect_info: unexpected error: %v", err)
		}
	}()

	// Stream sgcollect_info stdout to debug logs
	go func() {
		scanner := bufio.NewScanner(stdoutPipeReader)
		for scanner.Scan() {
			base.Debugf(base.KeyAdmin, "sgcollect_info: %v", scanner.Text())
		}
		if err := scanner.Err(); err != nil {
			base.Errorf("sgcollect_info: unexpected error: %v", err)
		}
	}()

	go func() {
		// Blocks until command finishes
		err := cmd.Wait()

		atomic.StoreUint32(sg.status, sgStopped)
		duration := time.Since(startTime)

		if err != nil {
			if err.Error() == "signal: killed" {
				base.Infof(base.KeyAdmin, "sgcollect_info cancelled after %v", duration)
				return
			}

			base.Errorf("sgcollect_info failed after %v with reason: %v. Check warning level logs for more information.", duration, err)
			return
		}

		base.Infof(base.KeyAdmin, "sgcollect_info finished successfully after %v", duration)
	}()

	return nil
}

// Stop will stop sgcollect_info, if running.
func (sg *sgCollect) Stop() error {
	if atomic.LoadUint32(sg.status) == sgStopped {
		return ErrSGCollectInfoNotRunning
	}

	sg.cancel()
	atomic.StoreUint32(sg.status, sgStopped)

	return nil
}

// IsRunning returns true if sgcollect_info is running
func (sg *sgCollect) IsRunning() bool {
	return atomic.LoadUint32(sg.status) == sgRunning
}

type sgCollectOptions struct {
	RedactLevel     string `json:"redact_level,omitempty"`
	RedactSalt      string `json:"redact_salt,omitempty"`
	OutputDirectory string `json:"output_dir,omitempty"`
	Upload          bool   `json:"upload,omitempty"`
	UploadHost      string `json:"upload_host,omitempty"`
	UploadProxy     string `json:"upload_proxy,omitempty"`
	Customer        string `json:"customer,omitempty"`
	Ticket          string `json:"ticket,omitempty"`
}

// validateOutputDirectory will check that the given path exists, and is a directory.
func validateOutputDirectory(dir string) error {
	// Clean the given path first, mainly for cross-platform compatability.
	dir = filepath.Clean(dir)

	// Validate given output directory exists, and is a directory.
	// This does not check for write permission, however sgcollect_info
	// will fail with an error giving that reason, if this is the case.
	if fileInfo, err := os.Stat(dir); err != nil {
		if os.IsNotExist(err) {
			return errors.Wrap(err, "no such file or directory")
		}
		return err
	} else if !fileInfo.IsDir() {
		return errors.New("not a directory")
	}

	return nil
}

// Validate ensures the options are OK to use in sgcollect_info.
func (c *sgCollectOptions) Validate() error {
	if c.OutputDirectory != "" {
		if err := validateOutputDirectory(c.OutputDirectory); err != nil {
			return err
		}
	}

	if c.Ticket != "" {
		if !validateTicketPattern.MatchString(c.Ticket) {
			return errors.New("ticket number must be 1 to 7 digits")
		}
	}

	if c.Upload {
		// Customer number is required if uploading.
		if c.Customer == "" {
			return errors.New("customer must be set if upload is true")
		}
		// Default uploading to support bucket if upload_host is not specified.
		if c.UploadHost == "" {
			c.UploadHost = defaultSGUploadHost
		}
	} else {
		// These fields suggest the user actually wanted to upload,
		// so we'll enforce "upload: true" if any of these are set.
		if c.UploadHost != "" {
			return errors.New("upload must be set to true if upload_host is specified")
		}
		if c.Customer != "" {
			return errors.New("upload must be set to true if customer is specified")
		}
		if c.Ticket != "" {
			return errors.New("upload must be set to true if ticket is specified")
		}
	}

	return nil
}

// Args returns a set of arguments to pass to sgcollect_info.
func (c *sgCollectOptions) Args() []string {
	var args = make([]string, 0)

	if c.Upload {
		args = append(args, "--upload-host", c.UploadHost)
	}

	if c.UploadProxy != "" {
		args = append(args, "--upload-proxy", c.UploadProxy)
	}

	if c.Customer != "" {
		args = append(args, "--customer", c.Customer)
	}

	if c.Ticket != "" {
		args = append(args, "--ticket", c.Ticket)
	}

	if c.RedactLevel != "" {
		args = append(args, "--log-redaction-level", c.RedactLevel)
	}

	if c.RedactSalt != "" {
		args = append(args, "--log-redaction-salt", c.RedactSalt)
	}

	return args
}

// sgCollectPaths attempts to return the absolute paths to Sync Gateway and to sgcollect_info binaries.
func sgCollectPaths() (sgBinary, sgCollectBinary string, err error) {
	sgBinary, err = os.Executable()
	if err != nil {
		return "", "", err
	}

	sgBinary, err = filepath.Abs(sgBinary)
	if err != nil {
		return "", "", err
	}

	hasBinDir := true
	sgCollectPath := filepath.Join("tools", "sgcollect_info")

	if runtime.GOOS == "windows" {
		sgCollectPath += ".exe"
		// Windows has no bin directory for the SG executable.
		hasBinDir = false
	}

	for {
		if hasBinDir {
			sgCollectBinary = filepath.Join(filepath.Dir(filepath.Dir(sgBinary)), sgCollectPath)
		} else {
			sgCollectBinary = filepath.Join(filepath.Dir(sgBinary), sgCollectPath)
		}

		// Check sgcollect_info exists at the path we guessed.
		base.Debugf(base.KeyAdmin, "Checking sgcollect_info binary exists at: %v", sgCollectBinary)
		_, err = os.Stat(sgCollectBinary)
		if err != nil {

			// First attempt may fail if there's no bin directory, so we'll try once more without.
			if hasBinDir {
				hasBinDir = false
				continue
			}

			return "", "", err
		}

		return sgBinary, sgCollectBinary, nil
	}
}

// sgcollectFilename returns a Windows-safe filename for sgcollect_info zip files.
func sgcollectFilename() string {

	// get timestamp
	timestamp := time.Now().UTC().Format("2006-01-02t150405")

	// use a shortened product name
	name := "sg"

	// get primary IP address
	ip, err := base.FindPrimaryAddr()
	if err != nil {
		ip = net.IPv4zero
	}

	// E.g: sgcollectinfo-2018-05-10t133456-sg@203.0.113.123.zip
	filename := fmt.Sprintf("sgcollectinfo-%s-%s@%s.zip", timestamp, name, ip)

	// Strip illegal Windows filename characters
	filename = base.ReplaceAll(filename, "\\/:*?\"<>|", "")

	return filename
}
<commit_msg>CBG-700 Map sgcollect stdout / stderr to info all (#4512)<commit_after>package rest

import (
	"bufio"
	"context"
	"fmt"
	"io"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"sync/atomic"
	"time"

	"github.com/couchbase/sync_gateway/base"
	"github.com/pkg/errors"
)

var (
	// ErrSGCollectInfoAlreadyRunning is returned if sgcollect_info is already running.
	ErrSGCollectInfoAlreadyRunning = errors.New("already running")
	// ErrSGCollectInfoNotRunning is returned if sgcollect_info is not running.
	ErrSGCollectInfoNotRunning = errors.New("not running")

	validateTicketPattern = regexp.MustCompile(`\d{1,7}`)

	sgPath, sgCollectPath, err = sgCollectPaths()
	sgcollectInstance          = sgCollect{
		status:        base.Uint32Ptr(sgStopped),
		sgPath:        sgPath,
		sgCollectPath: sgCollectPath,
		pathError:     err}
)

const (
	sgStopped uint32 = iota
	sgRunning

	defaultSGUploadHost = "https://uploads.couchbase.com"
)

type sgCollect struct {
	cancel        context.CancelFunc
	status        *uint32
	sgPath        string
	sgCollectPath string
	pathError     error
}

// Start will attempt to start sgcollect_info, if another is not already running.
func (sg *sgCollect) Start(zipFilename string, params sgCollectOptions) error {
	if atomic.LoadUint32(sg.status) == sgRunning {
		return ErrSGCollectInfoAlreadyRunning
	}

	// Return error if there is any failure while obtaining sgCollectPaths.
	if sg.pathError != nil {
		return err
	}

	if params.OutputDirectory == "" {
		// If no output directory specified, default to the configured LogFilePath
		if config != nil && config.Logging != nil && config.Logging.LogFilePath != "" {
			params.OutputDirectory = config.Logging.LogFilePath
			base.Debugf(base.KeyAdmin, "sgcollect_info: no output directory specified, using LogFilePath: %v", params.OutputDirectory)
		} else {
			// If LogFilePath is not set, and DefaultLogFilePath is not set via a service script, error out.
			return errors.New("no output directory or LogFilePath specified")
		}

		// Validate the path, just in case were not getting it correctly.
		if err := validateOutputDirectory(params.OutputDirectory); err != nil {
			return err
		}
	}

	zipPath := filepath.Join(params.OutputDirectory, zipFilename)

	args := params.Args()
	args = append(args, "--sync-gateway-executable", sgPath)
	args = append(args, zipPath)

	ctx, cancelFunc := context.WithCancel(context.Background())
	sg.cancel = cancelFunc
	cmd := exec.CommandContext(ctx, sgCollectPath, args...)

	// Send command stderr/stdout to pipes
	stderrPipeReader, stderrPipeWriter := io.Pipe()
	cmd.Stderr = stderrPipeWriter
	stdoutPipeReader, stdoutpipeWriter := io.Pipe()
	cmd.Stdout = stdoutpipeWriter

	if err := cmd.Start(); err != nil {
		return err
	}

	atomic.StoreUint32(sg.status, sgRunning)
	startTime := time.Now()
	base.Infof(base.KeyAdmin, "sgcollect_info started with args: %v", base.UD(args))

	// Stream sgcollect_info stderr to warn logs
	go func() {
		scanner := bufio.NewScanner(stderrPipeReader)
		for scanner.Scan() {
			base.Infof(base.KeyAll, "sgcollect_info: %v", scanner.Text())
		}
		if err := scanner.Err(); err != nil {
			base.Errorf("sgcollect_info: unexpected error: %v", err)
		}
	}()

	// Stream sgcollect_info stdout to debug logs
	go func() {
		scanner := bufio.NewScanner(stdoutPipeReader)
		for scanner.Scan() {
			base.Infof(base.KeyAll, "sgcollect_info: %v", scanner.Text())
		}
		if err := scanner.Err(); err != nil {
			base.Errorf("sgcollect_info: unexpected error: %v", err)
		}
	}()

	go func() {
		// Blocks until command finishes
		err := cmd.Wait()

		atomic.StoreUint32(sg.status, sgStopped)
		duration := time.Since(startTime)

		if err != nil {
			if err.Error() == "signal: killed" {
				base.Infof(base.KeyAdmin, "sgcollect_info cancelled after %v", duration)
				return
			}

			base.Errorf("sgcollect_info failed after %v with reason: %v. Check warning level logs for more information.", duration, err)
			return
		}

		base.Infof(base.KeyAdmin, "sgcollect_info finished successfully after %v", duration)
	}()

	return nil
}

// Stop will stop sgcollect_info, if running.
func (sg *sgCollect) Stop() error {
	if atomic.LoadUint32(sg.status) == sgStopped {
		return ErrSGCollectInfoNotRunning
	}

	sg.cancel()
	atomic.StoreUint32(sg.status, sgStopped)

	return nil
}

// IsRunning returns true if sgcollect_info is running
func (sg *sgCollect) IsRunning() bool {
	return atomic.LoadUint32(sg.status) == sgRunning
}

type sgCollectOptions struct {
	RedactLevel     string `json:"redact_level,omitempty"`
	RedactSalt      string `json:"redact_salt,omitempty"`
	OutputDirectory string `json:"output_dir,omitempty"`
	Upload          bool   `json:"upload,omitempty"`
	UploadHost      string `json:"upload_host,omitempty"`
	UploadProxy     string `json:"upload_proxy,omitempty"`
	Customer        string `json:"customer,omitempty"`
	Ticket          string `json:"ticket,omitempty"`
}

// validateOutputDirectory will check that the given path exists, and is a directory.
func validateOutputDirectory(dir string) error {
	// Clean the given path first, mainly for cross-platform compatability.
	dir = filepath.Clean(dir)

	// Validate given output directory exists, and is a directory.
	// This does not check for write permission, however sgcollect_info
	// will fail with an error giving that reason, if this is the case.
	if fileInfo, err := os.Stat(dir); err != nil {
		if os.IsNotExist(err) {
			return errors.Wrap(err, "no such file or directory")
		}
		return err
	} else if !fileInfo.IsDir() {
		return errors.New("not a directory")
	}

	return nil
}

// Validate ensures the options are OK to use in sgcollect_info.
func (c *sgCollectOptions) Validate() error {
	if c.OutputDirectory != "" {
		if err := validateOutputDirectory(c.OutputDirectory); err != nil {
			return err
		}
	}

	if c.Ticket != "" {
		if !validateTicketPattern.MatchString(c.Ticket) {
			return errors.New("ticket number must be 1 to 7 digits")
		}
	}

	if c.Upload {
		// Customer number is required if uploading.
		if c.Customer == "" {
			return errors.New("customer must be set if upload is true")
		}
		// Default uploading to support bucket if upload_host is not specified.
		if c.UploadHost == "" {
			c.UploadHost = defaultSGUploadHost
		}
	} else {
		// These fields suggest the user actually wanted to upload,
		// so we'll enforce "upload: true" if any of these are set.
		if c.UploadHost != "" {
			return errors.New("upload must be set to true if upload_host is specified")
		}
		if c.Customer != "" {
			return errors.New("upload must be set to true if customer is specified")
		}
		if c.Ticket != "" {
			return errors.New("upload must be set to true if ticket is specified")
		}
	}

	return nil
}

// Args returns a set of arguments to pass to sgcollect_info.
func (c *sgCollectOptions) Args() []string {
	var args = make([]string, 0)

	if c.Upload {
		args = append(args, "--upload-host", c.UploadHost)
	}

	if c.UploadProxy != "" {
		args = append(args, "--upload-proxy", c.UploadProxy)
	}

	if c.Customer != "" {
		args = append(args, "--customer", c.Customer)
	}

	if c.Ticket != "" {
		args = append(args, "--ticket", c.Ticket)
	}

	if c.RedactLevel != "" {
		args = append(args, "--log-redaction-level", c.RedactLevel)
	}

	if c.RedactSalt != "" {
		args = append(args, "--log-redaction-salt", c.RedactSalt)
	}

	return args
}

// sgCollectPaths attempts to return the absolute paths to Sync Gateway and to sgcollect_info binaries.
func sgCollectPaths() (sgBinary, sgCollectBinary string, err error) {
	sgBinary, err = os.Executable()
	if err != nil {
		return "", "", err
	}

	sgBinary, err = filepath.Abs(sgBinary)
	if err != nil {
		return "", "", err
	}

	hasBinDir := true
	sgCollectPath := filepath.Join("tools", "sgcollect_info")

	if runtime.GOOS == "windows" {
		sgCollectPath += ".exe"
		// Windows has no bin directory for the SG executable.
		hasBinDir = false
	}

	for {
		if hasBinDir {
			sgCollectBinary = filepath.Join(filepath.Dir(filepath.Dir(sgBinary)), sgCollectPath)
		} else {
			sgCollectBinary = filepath.Join(filepath.Dir(sgBinary), sgCollectPath)
		}

		// Check sgcollect_info exists at the path we guessed.
		base.Debugf(base.KeyAdmin, "Checking sgcollect_info binary exists at: %v", sgCollectBinary)
		_, err = os.Stat(sgCollectBinary)
		if err != nil {

			// First attempt may fail if there's no bin directory, so we'll try once more without.
			if hasBinDir {
				hasBinDir = false
				continue
			}

			return "", "", err
		}

		return sgBinary, sgCollectBinary, nil
	}
}

// sgcollectFilename returns a Windows-safe filename for sgcollect_info zip files.
func sgcollectFilename() string {

	// get timestamp
	timestamp := time.Now().UTC().Format("2006-01-02t150405")

	// use a shortened product name
	name := "sg"

	// get primary IP address
	ip, err := base.FindPrimaryAddr()
	if err != nil {
		ip = net.IPv4zero
	}

	// E.g: sgcollectinfo-2018-05-10t133456-sg@203.0.113.123.zip
	filename := fmt.Sprintf("sgcollectinfo-%s-%s@%s.zip", timestamp, name, ip)

	// Strip illegal Windows filename characters
	filename = base.ReplaceAll(filename, "\\/:*?\"<>|", "")

	return filename
}
<|endoftext|>


<commit_before>package dht

import (
  "container/list"
)

// ID for IpfsDHT should be a byte slice, to allow for simpler operations
// (xor). DHT ids are based on the peer.IDs.
//
// NOTE: peer.IDs are biased because they are (a) multihashes (first bytes
// biased), and (b) first bits are zeroes when using the S/Kademlia PoW.
// Thus, may need to re-hash keys (uniform dist). TODO(jbenet)
type ID []byte

// Bucket holds a list of peers.
type Bucket []*list.List

// RoutingTable defines the routing table.
type RoutingTable struct {

  // kBuckets define all the fingers to other nodes.
  Buckets []Bucket
}

func (id ID) commonPrefixLen() int {
  for i := 0; i < len(id); i++ {
    for j := 0; j < 8; j++ {
      if (id[i] >> uint8(7 - j)) & 0x1 != 0 {
        return i * 8 + j;
      }
    }
  }
  return len(id) * 8 - 1;
}

func xor(a, b ID) ID {

  // ids may actually be of different sizes.
  var ba ID
  var bb ID
  if len(a) >= len(b) {
    ba = a
    bb = b
  } else {
    ba = b
    bb = a
  }

  c := make(ID, len(ba))
  for i := 0; i < len(ba); i++ {
    if len(bb) > i {
      c[i] = ba[i] ^ bb[i]
    } else {
      c[i] = ba[i] ^ 0
    }
  }
  return c
}
<commit_msg>equalize sizes<commit_after>package dht

import (
	"bytes"
	"container/list"
)

// ID for IpfsDHT should be a byte slice, to allow for simpler operations
// (xor). DHT ids are based on the peer.IDs.
//
// NOTE: peer.IDs are biased because they are multihashes (first bytes
// biased). Thus, may need to re-hash keys (uniform dist). TODO(jbenet)
type ID []byte

// Bucket holds a list of peers.
type Bucket []*list.List

// RoutingTable defines the routing table.
type RoutingTable struct {

	// kBuckets define all the fingers to other nodes.
	Buckets []Bucket
}

func (id ID) Equal(other ID) bool {
	return bytes.Equal(id, other)
}

func (id ID) Less(other interface{}) bool {
	a, b := equalizeSizes(id, other.(ID))
	for i := 0; i < len(a); i++ {
		if a[i] != b[i] {
			return a[i] < b[i]
		}
	}
	return len(a) < len(b)
}

func (id ID) commonPrefixLen() int {
	for i := 0; i < len(id); i++ {
		for j := 0; j < 8; j++ {
			if (id[i]>>uint8(7-j))&0x1 != 0 {
				return i*8 + j
			}
		}
	}
	return len(id)*8 - 1
}

func xor(a, b ID) ID {
	a, b = equalizeSizes(a, b)

	c := make(ID, len(a))
	for i := 0; i < len(a); i++ {
		c[i] = a[i] ^ b[i]
	}
	return c
}

func equalizeSizes(a, b ID) (ID, ID) {
	la := len(a)
	lb := len(b)

	if la < lb {
		na := make([]byte, lb)
		copy(na, a)
		a = na
	} else if lb < la {
		nb := make([]byte, la)
		copy(nb, b)
		b = nb
	}

	return a, b
}
<|endoftext|>


<commit_before>package matchers

import (
	"fmt"
	"strings"

	"github.com/cloudfoundry/cli/cf/terminal"
	"github.com/onsi/gomega"
)

type SliceMatcher struct {
	expected      [][]string
	failedAtIndex int
}

func ContainSubstrings(substrings ...[]string) gomega.OmegaMatcher {
	return &SliceMatcher{expected: substrings}
}

//func (matcher *SliceMatcher) Match(actual interface{}) (success bool, err error) {
//	actualStrings, ok := actual.([]string)
//	if !ok {
//		return false, nil
//	}
//
//	matcher.failedAtIndex = 0
//	for _, actualValue := range actualStrings {
//		allStringsFound := true
//		for _, expectedValue := range matcher.expected[matcher.failedAtIndex] {
//			allStringsFound = allStringsFound && strings.Contains(terminal.Decolorize(actualValue), expectedValue)
//		}
//
//		if allStringsFound {
//			matcher.failedAtIndex++
//			if matcher.failedAtIndex == len(matcher.expected) {
//				matcher.failedAtIndex--
//				return true, nil
//			}
//		}
//	}
//
//	return false, nil
//}

func (matcher *SliceMatcher) Match(actual interface{}) (success bool, err error) {
	actualStrings, ok := actual.([]string)
	if !ok {
		return false, nil
	}

	allStringsMatched := make([]bool, len(matcher.expected))

	for index, expectedArray := range matcher.expected {
		for _, actualValue := range actualStrings {

			allStringsFound := true

			for _, expectedValue := range expectedArray {
				allStringsFound = allStringsFound && strings.Contains(terminal.Decolorize(actualValue), expectedValue)
			}

			if allStringsFound {
				allStringsMatched[index] = true
				break
			}
		}
	}

	for index, value := range allStringsMatched {
		if !value {
			matcher.failedAtIndex = index
			return false, nil
		}
	}

	return true, nil
}

func (matcher *SliceMatcher) FailureMessage(actual interface{}) string {
	actualStrings, ok := actual.([]string)
	if !ok {
		return fmt.Sprintf("Expected actual to be a slice of strings, but it's actually a %T", actual)
	}

	return fmt.Sprintf("expected to find \"%s\" in actual:\n'%s'\n", matcher.expected[matcher.failedAtIndex], strings.Join(actualStrings, "\n"))
}

func (matcher *SliceMatcher) NegatedFailureMessage(actual interface{}) string {
	actualStrings, ok := actual.([]string)
	if !ok {
		return fmt.Sprintf("Expected actual to be a slice of strings, but it's actually a %T", actual)
	}
	return fmt.Sprintf("expected to not find \"%s\" in actual:\n'%s'\n", matcher.expected[matcher.failedAtIndex], strings.Join(actualStrings, "\n"))
}
<commit_msg>remove commented code<commit_after>package matchers

import (
	"fmt"
	"strings"

	"github.com/cloudfoundry/cli/cf/terminal"
	"github.com/onsi/gomega"
)

type SliceMatcher struct {
	expected      [][]string
	failedAtIndex int
}

func ContainSubstrings(substrings ...[]string) gomega.OmegaMatcher {
	return &SliceMatcher{expected: substrings}
}

func (matcher *SliceMatcher) Match(actual interface{}) (success bool, err error) {
	actualStrings, ok := actual.([]string)
	if !ok {
		return false, nil
	}

	allStringsMatched := make([]bool, len(matcher.expected))

	for index, expectedArray := range matcher.expected {
		for _, actualValue := range actualStrings {

			allStringsFound := true

			for _, expectedValue := range expectedArray {
				allStringsFound = allStringsFound && strings.Contains(terminal.Decolorize(actualValue), expectedValue)
			}

			if allStringsFound {
				allStringsMatched[index] = true
				break
			}
		}
	}

	for index, value := range allStringsMatched {
		if !value {
			matcher.failedAtIndex = index
			return false, nil
		}
	}

	return true, nil
}

func (matcher *SliceMatcher) FailureMessage(actual interface{}) string {
	actualStrings, ok := actual.([]string)
	if !ok {
		return fmt.Sprintf("Expected actual to be a slice of strings, but it's actually a %T", actual)
	}

	return fmt.Sprintf("expected to find \"%s\" in actual:\n'%s'\n", matcher.expected[matcher.failedAtIndex], strings.Join(actualStrings, "\n"))
}

func (matcher *SliceMatcher) NegatedFailureMessage(actual interface{}) string {
	actualStrings, ok := actual.([]string)
	if !ok {
		return fmt.Sprintf("Expected actual to be a slice of strings, but it's actually a %T", actual)
	}
	return fmt.Sprintf("expected to not find \"%s\" in actual:\n'%s'\n", matcher.expected[matcher.failedAtIndex], strings.Join(actualStrings, "\n"))
}
<|endoftext|>