<commit_before>package gitmediaclient

import (
	".."
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/cheggaaa/pb"
	"io"
	"io/ioutil"
	"mime"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
)

const (
	gitMediaType      = "application/vnd.git-media"
	gitMediaMetaType  = gitMediaType + "+json; charset=utf-8"
	gitMediaHeader    = "--git-media."
	gitBoundaryLength = 61
)

func Options(filehash string) error {
	oid := filepath.Base(filehash)
	_, err := os.Stat(filehash)
	if err != nil {
		return err
	}

	req, creds, err := clientRequest("OPTIONS", oid)
	if err != nil {
		return err
	}

	_, err = doRequest(req, creds)
	if err != nil {
		return err
	}

	return nil
}

func Put(filehash, filename string) error {
	if filename == "" {
		filename = filehash
	}

	oid := filepath.Base(filehash)
	stat, err := os.Stat(filehash)
	if err != nil {
		return err
	}

	file, err := os.Open(filehash)
	if err != nil {
		return err
	}

	req, creds, err := clientRequest("PUT", oid)
	if err != nil {
		return err
	}

	bar := pb.StartNew(int(stat.Size()))
	bar.SetUnits(pb.U_BYTES)
	bar.Start()

	req.Header.Set("Content-Type", gitMediaType)
	req.Header.Set("Accept", gitMediaMetaType)
	req.Body = ioutil.NopCloser(bar.NewProxyReader(file))
	req.ContentLength = stat.Size()

	fmt.Printf("Sending %s\n", filename)

	_, err = doRequest(req, creds)
	if err != nil {
		return err
	}

	return nil
}

func Get(filename string) (io.ReadCloser, error) {
	oid := filepath.Base(filename)
	if stat, err := os.Stat(filename); err != nil || stat == nil {
		req, creds, err := clientRequest("GET", oid)
		if err != nil {
			return nil, err
		}

		req.Header.Set("Accept", gitMediaType)
		res, err := doRequest(req, creds)

		if err != nil {
			return nil, err
		}

		contentType := res.Header.Get("Content-Type")
		if contentType == "" {
			return nil, errors.New("Invalid Content-Type")
		}

		if ok, err := validateMediaHeader(contentType, res.Body); !ok {
			return nil, err
		}

		return res.Body, nil
	}

	return os.Open(filename)
}

func validateMediaHeader(contentType string, reader io.Reader) (bool, error) {
	mediaType, params, err := mime.ParseMediaType(contentType)
	if err != nil {
		return false, errors.New("Invalid Media Type")
	}

	if mediaType != gitMediaType {
		return false, errors.New("Invalid Media Type")
	}

	givenHeader, ok := params["header"]
	if !ok {
		return false, errors.New("Invalid header")
	}

	fullGivenHeader := "--" + givenHeader + "\n"

	header := make([]byte, len(fullGivenHeader))
	_, err = io.ReadAtLeast(reader, header, len(fullGivenHeader))
	if err != nil {
		return false, err
	}

	if string(header) != fullGivenHeader {
		return false, errors.New("Invalid header")
	}

	return true, nil
}

func doRequest(req *http.Request, creds Creds) (*http.Response, error) {
	res, err := http.DefaultClient.Do(req)

	if err == nil {
		if res.StatusCode > 299 {
			execCreds(creds, "reject")

			apierr := &Error{}
			dec := json.NewDecoder(res.Body)
			if err := dec.Decode(apierr); err != nil {
				return res, err
			}

			return res, apierr
		}
	} else {
		execCreds(creds, "approve")
	}

	return res, err
}

func clientRequest(method, oid string) (*http.Request, Creds, error) {
	u := ObjectUrl(oid)
	req, err := http.NewRequest(method, u.String(), nil)
	if err == nil {
		creds, err := credentials(u)
		if err != nil {
			return req, nil, err
		}

		token := fmt.Sprintf("%s:%s", creds["username"], creds["password"])
		auth := "Basic " + base64.URLEncoding.EncodeToString([]byte(token))
		req.Header.Set("Authorization", auth)
		return req, creds, nil
	}

	return req, nil, err
}

func ObjectUrl(oid string) *url.URL {
	c := gitmedia.Config
	u, _ := url.Parse(c.Endpoint())
	u.Path = filepath.Join(u.Path, "/objects/"+oid)
	return u
}

type Error struct {
	Message   string `json:"message"`
	RequestId string `json:"request_id,omitempty"`
}

func (e *Error) Error() string {
	return e.Message
}
<commit_msg>アーア アアアア アーアー<commit_after>package gitmediaclient

import (
	".."
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/cheggaaa/pb"
	"io"
	"io/ioutil"
	"mime"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
)

const (
	gitMediaType     = "application/vnd.git-media"
	gitMediaMetaType = gitMediaType + "+json; charset=utf-8"
	gitMediaHeader   = "--git-media."
)

func Options(filehash string) error {
	oid := filepath.Base(filehash)
	_, err := os.Stat(filehash)
	if err != nil {
		return err
	}

	req, creds, err := clientRequest("OPTIONS", oid)
	if err != nil {
		return err
	}

	_, err = doRequest(req, creds)
	if err != nil {
		return err
	}

	return nil
}

func Put(filehash, filename string) error {
	if filename == "" {
		filename = filehash
	}

	oid := filepath.Base(filehash)
	stat, err := os.Stat(filehash)
	if err != nil {
		return err
	}

	file, err := os.Open(filehash)
	if err != nil {
		return err
	}

	req, creds, err := clientRequest("PUT", oid)
	if err != nil {
		return err
	}

	bar := pb.StartNew(int(stat.Size()))
	bar.SetUnits(pb.U_BYTES)
	bar.Start()

	req.Header.Set("Content-Type", gitMediaType)
	req.Header.Set("Accept", gitMediaMetaType)
	req.Body = ioutil.NopCloser(bar.NewProxyReader(file))
	req.ContentLength = stat.Size()

	fmt.Printf("Sending %s\n", filename)

	_, err = doRequest(req, creds)
	if err != nil {
		return err
	}

	return nil
}

func Get(filename string) (io.ReadCloser, error) {
	oid := filepath.Base(filename)
	if stat, err := os.Stat(filename); err != nil || stat == nil {
		req, creds, err := clientRequest("GET", oid)
		if err != nil {
			return nil, err
		}

		req.Header.Set("Accept", gitMediaType)
		res, err := doRequest(req, creds)

		if err != nil {
			return nil, err
		}

		contentType := res.Header.Get("Content-Type")
		if contentType == "" {
			return nil, errors.New("Invalid Content-Type")
		}

		if ok, err := validateMediaHeader(contentType, res.Body); !ok {
			return nil, err
		}

		return res.Body, nil
	}

	return os.Open(filename)
}

func validateMediaHeader(contentType string, reader io.Reader) (bool, error) {
	mediaType, params, err := mime.ParseMediaType(contentType)
	if err != nil {
		return false, errors.New("Invalid Media Type")
	}

	if mediaType != gitMediaType {
		return false, errors.New("Invalid Media Type")
	}

	givenHeader, ok := params["header"]
	if !ok {
		return false, errors.New("Invalid header")
	}

	fullGivenHeader := "--" + givenHeader + "\n"

	header := make([]byte, len(fullGivenHeader))
	_, err = io.ReadAtLeast(reader, header, len(fullGivenHeader))
	if err != nil {
		return false, err
	}

	if string(header) != fullGivenHeader {
		return false, errors.New("Invalid header")
	}

	return true, nil
}

func doRequest(req *http.Request, creds Creds) (*http.Response, error) {
	res, err := http.DefaultClient.Do(req)

	if err == nil {
		if res.StatusCode > 299 {
			execCreds(creds, "reject")

			apierr := &Error{}
			dec := json.NewDecoder(res.Body)
			if err := dec.Decode(apierr); err != nil {
				return res, err
			}

			return res, apierr
		}
	} else {
		execCreds(creds, "approve")
	}

	return res, err
}

func clientRequest(method, oid string) (*http.Request, Creds, error) {
	u := ObjectUrl(oid)
	req, err := http.NewRequest(method, u.String(), nil)
	if err == nil {
		creds, err := credentials(u)
		if err != nil {
			return req, nil, err
		}

		token := fmt.Sprintf("%s:%s", creds["username"], creds["password"])
		auth := "Basic " + base64.URLEncoding.EncodeToString([]byte(token))
		req.Header.Set("Authorization", auth)
		return req, creds, nil
	}

	return req, nil, err
}

func ObjectUrl(oid string) *url.URL {
	c := gitmedia.Config
	u, _ := url.Parse(c.Endpoint())
	u.Path = filepath.Join(u.Path, "/objects/"+oid)
	return u
}

type Error struct {
	Message   string `json:"message"`
	RequestId string `json:"request_id,omitempty"`
}

func (e *Error) Error() string {
	return e.Message
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package client

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"sync"
	"time"

	"github.com/coreos/etcd/Godeps/_workspace/src/golang.org/x/net/context"
)

var (
	ErrNoEndpoints      = errors.New("client: no endpoints available")
	ErrTooManyRedirects = errors.New("client: too many redirects")
)

var DefaultRequestTimeout = 5 * time.Second

var DefaultTransport CancelableTransport = &http.Transport{
	Proxy: http.ProxyFromEnvironment,
	Dial: (&net.Dialer{
		Timeout:   30 * time.Second,
		KeepAlive: 30 * time.Second,
	}).Dial,
	TLSHandshakeTimeout: 10 * time.Second,
}

type Config struct {
	// Endpoints defines a set of URLs (schemes, hosts and ports only)
	// that can be used to communicate with a logical etcd cluster. For
	// example, a three-node cluster could be provided like so:
	//
	// 	Endpoints: []string{
	//		"http://node1.example.com:4001",
	//		"http://node2.example.com:2379",
	//		"http://node3.example.com:4001",
	//	}
	//
	// If multiple endpoints are provided, the Client will attempt to
	// use them all in the event that one or more of them are unusable.
	//
	// If Client.Sync is ever called, the Client may cache an alternate
	// set of endpoints to continue operation.
	Endpoints []string

	// Transport is used by the Client to drive HTTP requests. If not
	// provided, DefaultTransport will be used.
	Transport CancelableTransport

	// CheckRedirect specifies the policy for handling HTTP redirects.
	// If CheckRedirect is not nil, the Client calls it before
	// following an HTTP redirect. The sole argument is the number of
	// requests that have alrady been made. If CheckRedirect returns
	// an error, Client.Do will not make any further requests and return
	// the error back it to the caller.
	//
	// If CheckRedirect is nil, the Client uses its default policy,
	// which is to stop after 10 consecutive requests.
	CheckRedirect CheckRedirectFunc
}

func (cfg *Config) transport() CancelableTransport {
	if cfg.Transport == nil {
		return DefaultTransport
	}
	return cfg.Transport
}

func (cfg *Config) checkRedirect() CheckRedirectFunc {
	if cfg.CheckRedirect == nil {
		return DefaultCheckRedirect
	}
	return cfg.CheckRedirect
}

// CancelableTransport mimics net/http.Transport, but requires that
// the object also support request cancellation.
type CancelableTransport interface {
	http.RoundTripper
	CancelRequest(req *http.Request)
}

type CheckRedirectFunc func(via int) error

// DefaultCheckRedirect follows up to 10 redirects, but no more.
var DefaultCheckRedirect CheckRedirectFunc = func(via int) error {
	if via > 10 {
		return ErrTooManyRedirects
	}
	return nil
}

type Client interface {
	// Sync updates the internal cache of the etcd cluster's membership.
	Sync(context.Context) error

	// Endpoints returns a copy of the current set of API endpoints used
	// by Client to resolve HTTP requests. If Sync has ever been called,
	// this may differ from the initial Endpoints provided in the Config.
	Endpoints() []string

	httpClient
}

func New(cfg Config) (Client, error) {
	c := &httpClusterClient{clientFactory: newHTTPClientFactory(cfg.transport(), cfg.checkRedirect())}
	if err := c.reset(cfg.Endpoints); err != nil {
		return nil, err
	}
	return c, nil
}

type httpClient interface {
	Do(context.Context, httpAction) (*http.Response, []byte, error)
}

func newHTTPClientFactory(tr CancelableTransport, cr CheckRedirectFunc) httpClientFactory {
	return func(ep url.URL) httpClient {
		return &redirectFollowingHTTPClient{
			checkRedirect: cr,
			client: &simpleHTTPClient{
				transport: tr,
				endpoint:  ep,
			},
		}
	}
}

type httpClientFactory func(url.URL) httpClient

type httpAction interface {
	HTTPRequest(url.URL) *http.Request
}

type httpClusterClient struct {
	clientFactory httpClientFactory
	endpoints     []url.URL
	sync.RWMutex
}

func (c *httpClusterClient) reset(eps []string) error {
	if len(eps) == 0 {
		return ErrNoEndpoints
	}

	neps := make([]url.URL, len(eps))
	for i, ep := range eps {
		u, err := url.Parse(ep)
		if err != nil {
			return err
		}
		neps[i] = *u
	}

	c.endpoints = neps

	return nil
}

func (c *httpClusterClient) Do(ctx context.Context, act httpAction) (resp *http.Response, body []byte, err error) {
	c.RLock()
	leps := len(c.endpoints)
	eps := make([]url.URL, leps)
	n := copy(eps, c.endpoints)
	c.RUnlock()

	if leps == 0 {
		err = ErrNoEndpoints
		return
	}

	if leps != n {
		err = errors.New("unable to pick endpoint: copy failed")
		return
	}

	for _, ep := range eps {
		hc := c.clientFactory(ep)
		resp, body, err = hc.Do(ctx, act)
		if err != nil {
			if err == context.DeadlineExceeded || err == context.Canceled {
				return nil, nil, err
			}
			continue
		}
		if resp.StatusCode/100 == 5 {
			continue
		}
		break
	}

	return
}

func (c *httpClusterClient) Endpoints() []string {
	c.RLock()
	defer c.RUnlock()

	eps := make([]string, len(c.endpoints))
	for i, ep := range c.endpoints {
		eps[i] = ep.String()
	}

	return eps
}

func (c *httpClusterClient) Sync(ctx context.Context) error {
	c.Lock()
	defer c.Unlock()

	mAPI := NewMembersAPI(c)
	ms, err := mAPI.List(ctx)
	if err != nil {
		return err
	}

	eps := make([]string, 0)
	for _, m := range ms {
		eps = append(eps, m.ClientURLs...)
	}

	return c.reset(eps)
}

type roundTripResponse struct {
	resp *http.Response
	err  error
}

type simpleHTTPClient struct {
	transport CancelableTransport
	endpoint  url.URL
}

func (c *simpleHTTPClient) Do(ctx context.Context, act httpAction) (*http.Response, []byte, error) {
	req := act.HTTPRequest(c.endpoint)

	rtchan := make(chan roundTripResponse, 1)
	go func() {
		resp, err := c.transport.RoundTrip(req)
		rtchan <- roundTripResponse{resp: resp, err: err}
		close(rtchan)
	}()

	var resp *http.Response
	var err error

	select {
	case rtresp := <-rtchan:
		resp, err = rtresp.resp, rtresp.err
	case <-ctx.Done():
		c.transport.CancelRequest(req)
		// wait for request to actually exit before continuing
		<-rtchan
		err = ctx.Err()
	}

	// always check for resp nil-ness to deal with possible
	// race conditions between channels above
	defer func() {
		if resp != nil {
			resp.Body.Close()
		}
	}()

	if err != nil {
		return nil, nil, err
	}

	body, err := ioutil.ReadAll(resp.Body)
	return resp, body, err
}

type redirectFollowingHTTPClient struct {
	client        httpClient
	checkRedirect CheckRedirectFunc
}

func (r *redirectFollowingHTTPClient) Do(ctx context.Context, act httpAction) (*http.Response, []byte, error) {
	for i := 0; ; i++ {
		if i > 0 {
			if err := r.checkRedirect(i); err != nil {
				return nil, nil, err
			}
		}
		resp, body, err := r.client.Do(ctx, act)
		if err != nil {
			return nil, nil, err
		}
		if resp.StatusCode/100 == 3 {
			hdr := resp.Header.Get("Location")
			if hdr == "" {
				return nil, nil, fmt.Errorf("Location header not set")
			}
			loc, err := url.Parse(hdr)
			if err != nil {
				return nil, nil, fmt.Errorf("Location header not valid URL: %s", hdr)
			}
			act = &redirectedHTTPAction{
				action:   act,
				location: *loc,
			}
			continue
		}
		return resp, body, nil
	}
}

type redirectedHTTPAction struct {
	action   httpAction
	location url.URL
}

func (r *redirectedHTTPAction) HTTPRequest(ep url.URL) *http.Request {
	orig := r.action.HTTPRequest(ep)
	orig.URL = &r.location
	return orig
}
<commit_msg>client: move lock so MembersAPI.List doesn't deadlock<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package client

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"sync"
	"time"

	"github.com/coreos/etcd/Godeps/_workspace/src/golang.org/x/net/context"
)

var (
	ErrNoEndpoints      = errors.New("client: no endpoints available")
	ErrTooManyRedirects = errors.New("client: too many redirects")
)

var DefaultRequestTimeout = 5 * time.Second

var DefaultTransport CancelableTransport = &http.Transport{
	Proxy: http.ProxyFromEnvironment,
	Dial: (&net.Dialer{
		Timeout:   30 * time.Second,
		KeepAlive: 30 * time.Second,
	}).Dial,
	TLSHandshakeTimeout: 10 * time.Second,
}

type Config struct {
	// Endpoints defines a set of URLs (schemes, hosts and ports only)
	// that can be used to communicate with a logical etcd cluster. For
	// example, a three-node cluster could be provided like so:
	//
	// 	Endpoints: []string{
	//		"http://node1.example.com:4001",
	//		"http://node2.example.com:2379",
	//		"http://node3.example.com:4001",
	//	}
	//
	// If multiple endpoints are provided, the Client will attempt to
	// use them all in the event that one or more of them are unusable.
	//
	// If Client.Sync is ever called, the Client may cache an alternate
	// set of endpoints to continue operation.
	Endpoints []string

	// Transport is used by the Client to drive HTTP requests. If not
	// provided, DefaultTransport will be used.
	Transport CancelableTransport

	// CheckRedirect specifies the policy for handling HTTP redirects.
	// If CheckRedirect is not nil, the Client calls it before
	// following an HTTP redirect. The sole argument is the number of
	// requests that have alrady been made. If CheckRedirect returns
	// an error, Client.Do will not make any further requests and return
	// the error back it to the caller.
	//
	// If CheckRedirect is nil, the Client uses its default policy,
	// which is to stop after 10 consecutive requests.
	CheckRedirect CheckRedirectFunc
}

func (cfg *Config) transport() CancelableTransport {
	if cfg.Transport == nil {
		return DefaultTransport
	}
	return cfg.Transport
}

func (cfg *Config) checkRedirect() CheckRedirectFunc {
	if cfg.CheckRedirect == nil {
		return DefaultCheckRedirect
	}
	return cfg.CheckRedirect
}

// CancelableTransport mimics net/http.Transport, but requires that
// the object also support request cancellation.
type CancelableTransport interface {
	http.RoundTripper
	CancelRequest(req *http.Request)
}

type CheckRedirectFunc func(via int) error

// DefaultCheckRedirect follows up to 10 redirects, but no more.
var DefaultCheckRedirect CheckRedirectFunc = func(via int) error {
	if via > 10 {
		return ErrTooManyRedirects
	}
	return nil
}

type Client interface {
	// Sync updates the internal cache of the etcd cluster's membership.
	Sync(context.Context) error

	// Endpoints returns a copy of the current set of API endpoints used
	// by Client to resolve HTTP requests. If Sync has ever been called,
	// this may differ from the initial Endpoints provided in the Config.
	Endpoints() []string

	httpClient
}

func New(cfg Config) (Client, error) {
	c := &httpClusterClient{clientFactory: newHTTPClientFactory(cfg.transport(), cfg.checkRedirect())}
	if err := c.reset(cfg.Endpoints); err != nil {
		return nil, err
	}
	return c, nil
}

type httpClient interface {
	Do(context.Context, httpAction) (*http.Response, []byte, error)
}

func newHTTPClientFactory(tr CancelableTransport, cr CheckRedirectFunc) httpClientFactory {
	return func(ep url.URL) httpClient {
		return &redirectFollowingHTTPClient{
			checkRedirect: cr,
			client: &simpleHTTPClient{
				transport: tr,
				endpoint:  ep,
			},
		}
	}
}

type httpClientFactory func(url.URL) httpClient

type httpAction interface {
	HTTPRequest(url.URL) *http.Request
}

type httpClusterClient struct {
	clientFactory httpClientFactory
	endpoints     []url.URL
	sync.RWMutex
}

func (c *httpClusterClient) reset(eps []string) error {
	if len(eps) == 0 {
		return ErrNoEndpoints
	}

	neps := make([]url.URL, len(eps))
	for i, ep := range eps {
		u, err := url.Parse(ep)
		if err != nil {
			return err
		}
		neps[i] = *u
	}

	c.endpoints = neps

	return nil
}

func (c *httpClusterClient) Do(ctx context.Context, act httpAction) (resp *http.Response, body []byte, err error) {
	c.RLock()
	leps := len(c.endpoints)
	eps := make([]url.URL, leps)
	n := copy(eps, c.endpoints)
	c.RUnlock()

	if leps == 0 {
		err = ErrNoEndpoints
		return
	}

	if leps != n {
		err = errors.New("unable to pick endpoint: copy failed")
		return
	}

	for _, ep := range eps {
		hc := c.clientFactory(ep)
		resp, body, err = hc.Do(ctx, act)
		if err != nil {
			if err == context.DeadlineExceeded || err == context.Canceled {
				return nil, nil, err
			}
			continue
		}
		if resp.StatusCode/100 == 5 {
			continue
		}
		break
	}

	return
}

func (c *httpClusterClient) Endpoints() []string {
	c.RLock()
	defer c.RUnlock()

	eps := make([]string, len(c.endpoints))
	for i, ep := range c.endpoints {
		eps[i] = ep.String()
	}

	return eps
}

func (c *httpClusterClient) Sync(ctx context.Context) error {
	mAPI := NewMembersAPI(c)
	ms, err := mAPI.List(ctx)
	if err != nil {
		return err
	}

	c.Lock()
	defer c.Unlock()

	eps := make([]string, 0)
	for _, m := range ms {
		eps = append(eps, m.ClientURLs...)
	}

	return c.reset(eps)
}

type roundTripResponse struct {
	resp *http.Response
	err  error
}

type simpleHTTPClient struct {
	transport CancelableTransport
	endpoint  url.URL
}

func (c *simpleHTTPClient) Do(ctx context.Context, act httpAction) (*http.Response, []byte, error) {
	req := act.HTTPRequest(c.endpoint)

	rtchan := make(chan roundTripResponse, 1)
	go func() {
		resp, err := c.transport.RoundTrip(req)
		rtchan <- roundTripResponse{resp: resp, err: err}
		close(rtchan)
	}()

	var resp *http.Response
	var err error

	select {
	case rtresp := <-rtchan:
		resp, err = rtresp.resp, rtresp.err
	case <-ctx.Done():
		c.transport.CancelRequest(req)
		// wait for request to actually exit before continuing
		<-rtchan
		err = ctx.Err()
	}

	// always check for resp nil-ness to deal with possible
	// race conditions between channels above
	defer func() {
		if resp != nil {
			resp.Body.Close()
		}
	}()

	if err != nil {
		return nil, nil, err
	}

	body, err := ioutil.ReadAll(resp.Body)
	return resp, body, err
}

type redirectFollowingHTTPClient struct {
	client        httpClient
	checkRedirect CheckRedirectFunc
}

func (r *redirectFollowingHTTPClient) Do(ctx context.Context, act httpAction) (*http.Response, []byte, error) {
	for i := 0; ; i++ {
		if i > 0 {
			if err := r.checkRedirect(i); err != nil {
				return nil, nil, err
			}
		}
		resp, body, err := r.client.Do(ctx, act)
		if err != nil {
			return nil, nil, err
		}
		if resp.StatusCode/100 == 3 {
			hdr := resp.Header.Get("Location")
			if hdr == "" {
				return nil, nil, fmt.Errorf("Location header not set")
			}
			loc, err := url.Parse(hdr)
			if err != nil {
				return nil, nil, fmt.Errorf("Location header not valid URL: %s", hdr)
			}
			act = &redirectedHTTPAction{
				action:   act,
				location: *loc,
			}
			continue
		}
		return resp, body, nil
	}
}

type redirectedHTTPAction struct {
	action   httpAction
	location url.URL
}

func (r *redirectedHTTPAction) HTTPRequest(ep url.URL) *http.Request {
	orig := r.action.HTTPRequest(ep)
	orig.URL = &r.location
	return orig
}
<|endoftext|>


<commit_before>package git

import (
	"os"
	"os/exec"
	"path/filepath"
)

// does the given directory contain a '.git' directory?
func IsRepo(path string) bool {
	_, err := os.Stat(filepath.Join(path, ".git"))
	return err == nil
}

func Clone(repo, workingDir string) error {
	return exec.Command("git", "clone", repo, workingDir).Run()
}

func Pull(workingDir string) error {
	return exec.Command("git", "-C", workingDir, "pull").Run()
}

func MasterSha(repo string) (string, error) {
	bytes, err := exec.Command("git", "ls-remote", repo, "master").Output()

	if err != nil {
		return "", err
	}

	return string(bytes), nil
}

// performs Pull or Clone as necessary
func PullOrClone(repo, workingDir string) error {
	if IsRepo(workingDir) {
		return Pull(workingDir)
	} else {
		return Clone(repo, workingDir)
	}
}
<commit_msg>don't use git -C to change working directory: requires modern git. set cmd.Dir instead<commit_after>package git

import (
	"os"
	"os/exec"
	"path/filepath"
)

// does the given directory contain a '.git' directory?
func IsRepo(path string) bool {
	_, err := os.Stat(filepath.Join(path, ".git"))
	return err == nil
}

func Clone(repo, workingDir string) error {
	return exec.Command("git", "clone", repo, workingDir).Run()
}

func Pull(workingDir string) error {
	cmd := exec.Command("git", "pull")
	cmd.Dir = workingDir
	return cmd.Run()
}

func MasterSha(repo string) (string, error) {
	bytes, err := exec.Command("git", "ls-remote", repo, "master").Output()

	if err != nil {
		return "", err
	}

	return string(bytes), nil
}

// performs Pull or Clone as necessary
func PullOrClone(repo, workingDir string) error {
	if IsRepo(workingDir) {
		return Pull(workingDir)
	} else {
		return Clone(repo, workingDir)
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 Francisco Souza. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package git

// #cgo pkg-config: libgit2
// #include <git2.h>
import "C"

import (
	"errors"
	"unsafe"
)

// Commit represents a git commit.
type Commit struct {
	commit *C.struct_git_commit
}

// Free is used to deallocate a commit object.
func (c *Commit) Free() {
	C.git_commit_free(c.commit)
}

// Id returns the hash of the commit.
func (c *Commit) Id() string {
	oid := C.git_commit_id(c.commit)
	defer C.free(unsafe.Pointer(oid))
	id := C.git_oid_allocfmt(oid)
	defer C.free(unsafe.Pointer(id))
	return C.GoString(id)
}

// Repository is the basic type of the git package, it represents a git
// repository.
type Repository struct {
	repository *C.struct_git_repository
}

// InitRepository inits a new repository.
//
// If the path does not exist, it will be created.
//
// Returns an instance of Repository, or an error in case of failure.
func InitRepository(path string, bare bool) (*Repository, error) {
	var cbare C.unsigned = 0
	if bare {
		cbare = 1
	}
	repo := &Repository{}
	cpath := C.CString(path)
	defer C.free(unsafe.Pointer(cpath))
	if C.git_repository_init(&repo.repository, cpath, cbare) != C.GIT_OK {
		return nil, lastErr()
	}
	return repo, nil
}

// OpenRepository opens a repository by its path.
func OpenRepository(path string) (*Repository, error) {
	repo := &Repository{}
	cpath := C.CString(path)
	defer C.free(unsafe.Pointer(cpath))
	if C.git_repository_open(&repo.repository, cpath) != C.GIT_OK {
		return nil, lastErr()
	}
	return repo, nil
}

// Config returns a Config instance, representing the configuration of the
// repository.
func (r *Repository) Config() (*Config, error) {
	conf := &Config{}
	if C.git_repository_config(&conf.config, r.repository) != C.GIT_OK {
		return nil, lastErr()
	}
	return conf, nil
}

// Free is used to deallocate the repository. It should be called to finish the
// repository. It's a good practice to use it with the defer statement:
//
//     repo, err := git.GetRepository("/path/to/repository")
//     // check error
//     defer repo.Free()
//     // use repo
func (r *Repository) Free() {
	C.git_repository_free(r.repository)
}

// Head returns the commit at the head of the repository.
func (r *Repository) Head() (*Commit, error) {
	var reference *C.struct_git_reference
	if C.git_repository_head(&reference, r.repository) != C.GIT_OK {
		return nil, lastErr()
	}
	defer C.git_reference_free(reference)
	c := new(Commit)
	if C.git_commit_lookup(&c.commit, r.repository, C.git_reference_oid(reference)) != C.GIT_OK {
		return nil, lastErr()
	}
	return c, nil
}

// Config represents the configuration of a git repository.
//
// You can use it retrieve or define configurations of the repository.
type Config struct {
	config *C.struct_git_config
}

// Free is used to deallocate the Config instance. It should be called to
// finish the instance. You can use it with the defer statement:
//
//     // get repository instance
//     config, err := repo.Config()
//     // check error
//     defer config.Free()
func (c *Config) Free() {
	C.git_config_free(c.config)
}

// GetBool is used to get boolean config values.
//
// The dot notation is used for configuration parameters. Example:
//
//     v, err := config.GetBool("core.ignorecase")
//     // check errors and use v
func (c *Config) GetBool(name string) (bool, error) {
	var v C.int
	cname := C.CString(name)
	defer C.free(unsafe.Pointer(cname))
	if C.git_config_get_bool(&v, c.config, cname) != C.GIT_OK {
		return false, lastErr()
	}
	return v == 1, nil
}

// SetBool is used to add a boolean setting to the configuration file.
//
// The format of the configuration parameter is the same as in GetBool. If the
// configuration parameter is not declared in the config file, it will be
// created. Example of use:
//
//     err := config.SetBool("core.ignorecase", true)
//     if err != nil {
//         panic(err)
//     }
func (c *Config) SetBool(name string, value bool) error {
	var v C.int = 0
	if value {
		v = 1
	}
	cname := C.CString(name)
	defer C.free(unsafe.Pointer(cname))
	if C.git_config_set_bool(c.config, cname, v) != C.GIT_OK {
		return lastErr()
	}
	return nil
}

// GetString is used to get string config values.
//
// The format of the configuration parameter is the same as in GetBool.
func (c *Config) GetString(name string) (string, error) {
	var v *C.char
	cname := C.CString(name)
	defer C.free(unsafe.Pointer(cname))
	if C.git_config_get_string(&v, c.config, cname) != C.GIT_OK {
		return "", lastErr()
	}
	return C.GoString(v), nil
}

// SetString is used to add a string setting to the config file.
//
// The format of the configuration parameter is the same as in GetBool. If the
// parameter is not declared in the config file, it will be created.
func (c *Config) SetString(name, value string) error {
	cname := C.CString(name)
	defer C.free(unsafe.Pointer(cname))
	cvalue := C.CString(value)
	defer C.free(unsafe.Pointer(cvalue))
	if C.git_config_set_string(c.config, cname, cvalue) != C.GIT_OK {
		return lastErr()
	}
	return nil
}

// GetInt64 is used to get int64 config values.
//
// The format of the configuration parameter is the same as in GetBool.
func (c *Config) GetInt64(name string) (int64, error) {
	var v C.int64_t
	cname := C.CString(name)
	defer C.free(unsafe.Pointer(cname))
	if C.git_config_get_int64(&v, c.config, cname) != C.GIT_OK {
		return 0, lastErr()
	}
	return int64(v), nil
}

// SetInt64 is used to add a int64 setting to the config file.
//
// The format of the configuration parameter is the same as in GetBool. If the
// parameter is not declared in the config file, it will be created.
func (c *Config) SetInt64(name string, value int64) error {
	cname := C.CString(name)
	defer C.free(unsafe.Pointer(cname))
	if C.git_config_set_int64(c.config, cname, C.int64_t(value)) != C.GIT_OK {
		return lastErr()
	}
	return nil
}

func lastErr() error {
	err := C.giterr_last()
	return errors.New(C.GoString(err.message))
}
<commit_msg>git: improve docs for OpenRepository function<commit_after>// Copyright 2012 Francisco Souza. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package git

// #cgo pkg-config: libgit2
// #include <git2.h>
import "C"

import (
	"errors"
	"unsafe"
)

// Commit represents a git commit.
type Commit struct {
	commit *C.struct_git_commit
}

// Free is used to deallocate a commit object.
func (c *Commit) Free() {
	C.git_commit_free(c.commit)
}

// Id returns the hash of the commit.
func (c *Commit) Id() string {
	oid := C.git_commit_id(c.commit)
	defer C.free(unsafe.Pointer(oid))
	id := C.git_oid_allocfmt(oid)
	defer C.free(unsafe.Pointer(id))
	return C.GoString(id)
}

// Repository is the basic type of the git package, it represents a git
// repository.
type Repository struct {
	repository *C.struct_git_repository
}

// InitRepository inits a new repository.
//
// If the path does not exist, it will be created.
//
// Returns an instance of Repository, or an error in case of failure.
func InitRepository(path string, bare bool) (*Repository, error) {
	var cbare C.unsigned = 0
	if bare {
		cbare = 1
	}
	repo := &Repository{}
	cpath := C.CString(path)
	defer C.free(unsafe.Pointer(cpath))
	if C.git_repository_init(&repo.repository, cpath, cbare) != C.GIT_OK {
		return nil, lastErr()
	}
	return repo, nil
}

// OpenRepository opens a repository by its path.
//
// Returns an error in case of failure (e.g.: if the path does not exist; if
// it exists but is not a git repository or if the path exists, is a git
// repository, but the user does not have access to it).
func OpenRepository(path string) (*Repository, error) {
	repo := &Repository{}
	cpath := C.CString(path)
	defer C.free(unsafe.Pointer(cpath))
	if C.git_repository_open(&repo.repository, cpath) != C.GIT_OK {
		return nil, lastErr()
	}
	return repo, nil
}

// Config returns a Config instance, representing the configuration of the
// repository.
func (r *Repository) Config() (*Config, error) {
	conf := &Config{}
	if C.git_repository_config(&conf.config, r.repository) != C.GIT_OK {
		return nil, lastErr()
	}
	return conf, nil
}

// Free is used to deallocate the repository. It should be called to finish the
// repository. It's a good practice to use it with the defer statement:
//
//     repo, err := git.GetRepository("/path/to/repository")
//     // check error
//     defer repo.Free()
//     // use repo
func (r *Repository) Free() {
	C.git_repository_free(r.repository)
}

// Head returns the commit at the head of the repository.
func (r *Repository) Head() (*Commit, error) {
	var reference *C.struct_git_reference
	if C.git_repository_head(&reference, r.repository) != C.GIT_OK {
		return nil, lastErr()
	}
	defer C.git_reference_free(reference)
	c := new(Commit)
	if C.git_commit_lookup(&c.commit, r.repository, C.git_reference_oid(reference)) != C.GIT_OK {
		return nil, lastErr()
	}
	return c, nil
}

// Config represents the configuration of a git repository.
//
// You can use it retrieve or define configurations of the repository.
type Config struct {
	config *C.struct_git_config
}

// Free is used to deallocate the Config instance. It should be called to
// finish the instance. You can use it with the defer statement:
//
//     // get repository instance
//     config, err := repo.Config()
//     // check error
//     defer config.Free()
func (c *Config) Free() {
	C.git_config_free(c.config)
}

// GetBool is used to get boolean config values.
//
// The dot notation is used for configuration parameters. Example:
//
//     v, err := config.GetBool("core.ignorecase")
//     // check errors and use v
func (c *Config) GetBool(name string) (bool, error) {
	var v C.int
	cname := C.CString(name)
	defer C.free(unsafe.Pointer(cname))
	if C.git_config_get_bool(&v, c.config, cname) != C.GIT_OK {
		return false, lastErr()
	}
	return v == 1, nil
}

// SetBool is used to add a boolean setting to the configuration file.
//
// The format of the configuration parameter is the same as in GetBool. If the
// configuration parameter is not declared in the config file, it will be
// created. Example of use:
//
//     err := config.SetBool("core.ignorecase", true)
//     if err != nil {
//         panic(err)
//     }
func (c *Config) SetBool(name string, value bool) error {
	var v C.int = 0
	if value {
		v = 1
	}
	cname := C.CString(name)
	defer C.free(unsafe.Pointer(cname))
	if C.git_config_set_bool(c.config, cname, v) != C.GIT_OK {
		return lastErr()
	}
	return nil
}

// GetString is used to get string config values.
//
// The format of the configuration parameter is the same as in GetBool.
func (c *Config) GetString(name string) (string, error) {
	var v *C.char
	cname := C.CString(name)
	defer C.free(unsafe.Pointer(cname))
	if C.git_config_get_string(&v, c.config, cname) != C.GIT_OK {
		return "", lastErr()
	}
	return C.GoString(v), nil
}

// SetString is used to add a string setting to the config file.
//
// The format of the configuration parameter is the same as in GetBool. If the
// parameter is not declared in the config file, it will be created.
func (c *Config) SetString(name, value string) error {
	cname := C.CString(name)
	defer C.free(unsafe.Pointer(cname))
	cvalue := C.CString(value)
	defer C.free(unsafe.Pointer(cvalue))
	if C.git_config_set_string(c.config, cname, cvalue) != C.GIT_OK {
		return lastErr()
	}
	return nil
}

// GetInt64 is used to get int64 config values.
//
// The format of the configuration parameter is the same as in GetBool.
func (c *Config) GetInt64(name string) (int64, error) {
	var v C.int64_t
	cname := C.CString(name)
	defer C.free(unsafe.Pointer(cname))
	if C.git_config_get_int64(&v, c.config, cname) != C.GIT_OK {
		return 0, lastErr()
	}
	return int64(v), nil
}

// SetInt64 is used to add a int64 setting to the config file.
//
// The format of the configuration parameter is the same as in GetBool. If the
// parameter is not declared in the config file, it will be created.
func (c *Config) SetInt64(name string, value int64) error {
	cname := C.CString(name)
	defer C.free(unsafe.Pointer(cname))
	if C.git_config_set_int64(c.config, cname, C.int64_t(value)) != C.GIT_OK {
		return lastErr()
	}
	return nil
}

func lastErr() error {
	err := C.giterr_last()
	return errors.New(C.GoString(err.message))
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"context"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io/ioutil"
	"math"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	kingpin "gopkg.in/alecthomas/kingpin.v2"

	"github.com/ShowMax/go-fqdn"
	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	"github.com/pkg/errors"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/prometheus/common/promlog"
	"github.com/prometheus/common/promlog/flag"
	"github.com/robustperception/pushprox/util"
)

var (
	myFqdn      = kingpin.Flag("fqdn", "FQDN to register with").Default(fqdn.Get()).String()
	proxyURL    = kingpin.Flag("proxy-url", "Push proxy to talk to.").Required().String()
	caCertFile  = kingpin.Flag("tls.cacert", "<file> CA certificate to verify peer against").String()
	tlsCert     = kingpin.Flag("tls.cert", "<cert> Client certificate file").String()
	tlsKey      = kingpin.Flag("tls.key", "<key> Private key file").String()
	metricsAddr = kingpin.Flag("metrics-addr", "Serve Prometheus metrics at this address").Default(":9369").String()
)

var (
	scrapeErrorCounter = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "pushprox_client_scrape_errors_total",
			Help: "Number of scrape errors",
		},
	)
	pushErrorCounter = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "pushprox_client_push_errors_total",
			Help: "Number of push errors",
		},
	)
	pollErrorCounter = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "pushprox_client_poll_errors_total",
			Help: "Number of poll errors",
		},
	)
)

func init() {
	prometheus.MustRegister(pushErrorCounter, pollErrorCounter, scrapeErrorCounter)
}

// Coordinator for scrape requests and responses
type Coordinator struct {
	logger log.Logger
}

func (c *Coordinator) handleErr(request *http.Request, client *http.Client, err error) {
	level.Error(c.logger).Log("err", err)
	scrapeErrorCounter.Inc()
	resp := &http.Response{
		StatusCode: http.StatusInternalServerError,
		Body:       ioutil.NopCloser(strings.NewReader(err.Error())),
		Header:     http.Header{},
	}
	if err = c.doPush(resp, request, client); err != nil {
		pushErrorCounter.Inc()
		level.Warn(c.logger).Log("msg", "Failed to push failed scrape response:", "err", err)
		return
	}
	level.Info(c.logger).Log("msg", "Pushed failed scrape response")
}

func (c *Coordinator) doScrape(request *http.Request, client *http.Client) {
	logger := log.With(c.logger, "scrape_id", request.Header.Get("id"))
	timeout, err := util.GetHeaderTimeout(request.Header)
	if err != nil {
		c.handleErr(request, client, err)
		return
	}
	ctx, cancel := context.WithTimeout(request.Context(), timeout)
	defer cancel()
	request = request.WithContext(ctx)
	// We cannot handle https requests at the proxy, as we would only
	// see a CONNECT, so use a URL parameter to trigger it.
	params := request.URL.Query()
	if params.Get("_scheme") == "https" {
		request.URL.Scheme = "https"
		params.Del("_scheme")
		request.URL.RawQuery = params.Encode()
	}

	fqdnURL, err := url.Parse(*myFqdn)
	if err != nil {
		c.handleErr(request, client, err)
		return
	}
	if request.URL.Hostname() != fqdnURL.Hostname() {
		c.handleErr(request, client, errors.New("scrape target doesn't match client fqdn"))
		return
	}

	scrapeResp, err := client.Do(request)
	if err != nil {
		msg := fmt.Sprintf("failed to scrape %s", request.URL.String())
		c.handleErr(request, client, errors.Wrap(err, msg))
		return
	}
	level.Info(logger).Log("msg", "Retrieved scrape response")
	if err = c.doPush(scrapeResp, request, client); err != nil {
		pushErrorCounter.Inc()
		level.Warn(logger).Log("msg", "Failed to push scrape response:", "err", err)
		return
	}
	level.Info(logger).Log("msg", "Pushed scrape result")
}

// Report the result of the scrape back up to the proxy.
func (c *Coordinator) doPush(resp *http.Response, origRequest *http.Request, client *http.Client) error {
	resp.Header.Set("id", origRequest.Header.Get("id")) // Link the request and response
	// Remaining scrape deadline.
	deadline, _ := origRequest.Context().Deadline()
	resp.Header.Set("X-Prometheus-Scrape-Timeout", fmt.Sprintf("%f", float64(time.Until(deadline))/1e9))

	base, err := url.Parse(*proxyURL)
	if err != nil {
		return err
	}
	u, err := url.Parse("push")
	if err != nil {
		return err
	}
	url := base.ResolveReference(u)

	buf := &bytes.Buffer{}
	resp.Write(buf)
	request := &http.Request{
		Method:        "POST",
		URL:           url,
		Body:          ioutil.NopCloser(buf),
		ContentLength: int64(buf.Len()),
	}
	request = request.WithContext(origRequest.Context())
	if _, err = client.Do(request); err != nil {
		return err
	}
	return nil
}

func loop(c Coordinator, client *http.Client) error {
	base, err := url.Parse(*proxyURL)
	if err != nil {
		level.Error(c.logger).Log("msg", "Error parsing url:", "err", err)
		return errors.Wrap(err, "error parsing url")
	}
	u, err := url.Parse("poll")
	if err != nil {
		level.Error(c.logger).Log("msg", "Error parsing url:", "err", err)
		return errors.Wrap(err, "error parsing url poll")
	}
	url := base.ResolveReference(u)
	resp, err := client.Post(url.String(), "", strings.NewReader(*myFqdn))
	if err != nil {
		level.Error(c.logger).Log("msg", "Error polling:", "err", err)
		return errors.Wrap(err, "error polling")
	}
	defer resp.Body.Close()

	request, err := http.ReadRequest(bufio.NewReader(resp.Body))
	if err != nil {
		level.Error(c.logger).Log("msg", "Error reading request:", "err", err)
		return errors.Wrap(err, "error reading request")
	}
	level.Info(c.logger).Log("msg", "Got scrape request", "scrape_id", request.Header.Get("id"), "url", request.URL)

	request.RequestURI = ""

	go c.doScrape(request, client)

	return nil
}

// decorrelated Jitter increases the maximum jitter based on the last random value.
type decorrelatedJitter struct {
	duration float64 // sleep time
	min      float64 // min sleep time
	cap      float64 // max sleep time
}

func newJitter() decorrelatedJitter {
	rand.Seed(time.Now().UnixNano())
	return decorrelatedJitter{
		duration: 1,
		min:      1,
		cap:      10,
	}
}

func (d *decorrelatedJitter) calc() {
	d.duration = math.Min(d.cap, d.min+rand.Float64()*(d.duration*3-d.min))
}

func (d *decorrelatedJitter) sleep() {
	time.Sleep(time.Duration(d.duration))
}

func main() {
	promlogConfig := promlog.Config{}
	flag.AddFlags(kingpin.CommandLine, &promlogConfig)
	kingpin.HelpFlag.Short('h')
	kingpin.Parse()
	logger := promlog.New(&promlogConfig)
	coordinator := Coordinator{logger: logger}
	if *proxyURL == "" {
		level.Error(coordinator.logger).Log("msg", "--proxy-url flag must be specified.")
		os.Exit(1)
	}
	// Make sure proxyURL ends with a single '/'
	*proxyURL = strings.TrimRight(*proxyURL, "/") + "/"
	level.Info(coordinator.logger).Log("msg", "URL and FQDN info", "proxy_url", *proxyURL, "fqdn", *myFqdn)

	tlsConfig := &tls.Config{}
	if *tlsCert != "" {
		cert, err := tls.LoadX509KeyPair(*tlsCert, *tlsKey)
		if err != nil {
			level.Error(coordinator.logger).Log("msg", "Certificate or Key is invalid", "err", err)
			os.Exit(1)
		}

		// Setup HTTPS client
		tlsConfig.Certificates = []tls.Certificate{cert}

		tlsConfig.BuildNameToCertificate()
	}

	if *caCertFile != "" {
		caCert, err := ioutil.ReadFile(*caCertFile)
		if err != nil {
			level.Error(coordinator.logger).Log("msg", "Not able to read cacert file", "err", err)
			os.Exit(1)
		}
		caCertPool := x509.NewCertPool()
		if ok := caCertPool.AppendCertsFromPEM(caCert); !ok {
			level.Error(coordinator.logger).Log("msg", "Failed to use cacert file as ca certificate")
			os.Exit(1)
		}

		tlsConfig.RootCAs = caCertPool
	}

	if *metricsAddr != "" {
		go func() {
			if err := http.ListenAndServe(*metricsAddr, promhttp.Handler()); err != nil {
				level.Warn(coordinator.logger).Log("msg", "ListenAndServe", "err", err)
			}
		}()
	}

	transport := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		DialContext: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
			DualStack: true,
		}).DialContext,
		MaxIdleConns:          100,
		IdleConnTimeout:       90 * time.Second,
		TLSHandshakeTimeout:   10 * time.Second,
		ExpectContinueTimeout: 1 * time.Second,
		TLSClientConfig:       tlsConfig,
	}

	jitter := newJitter()
	client := &http.Client{Transport: transport}
	for {
		err := loop(coordinator, client)
		if err != nil {
			pollErrorCounter.Inc()
			jitter.sleep()
			continue
		}
	}
}
<commit_msg>Fix: Strict mode to enforce FQDN (#65)<commit_after>package main

import (
	"bufio"
	"bytes"
	"context"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io/ioutil"
	"math"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	kingpin "gopkg.in/alecthomas/kingpin.v2"

	"github.com/ShowMax/go-fqdn"
	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	"github.com/pkg/errors"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/prometheus/common/promlog"
	"github.com/prometheus/common/promlog/flag"
	"github.com/robustperception/pushprox/util"
)

var (
	myFqdn      = kingpin.Flag("fqdn", "FQDN to register with").Default(fqdn.Get()).String()
	proxyURL    = kingpin.Flag("proxy-url", "Push proxy to talk to.").Required().String()
	caCertFile  = kingpin.Flag("tls.cacert", "<file> CA certificate to verify peer against").String()
	tlsCert     = kingpin.Flag("tls.cert", "<cert> Client certificate file").String()
	tlsKey      = kingpin.Flag("tls.key", "<key> Private key file").String()
	metricsAddr = kingpin.Flag("metrics-addr", "Serve Prometheus metrics at this address").Default(":9369").String()
)

var (
	scrapeErrorCounter = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "pushprox_client_scrape_errors_total",
			Help: "Number of scrape errors",
		},
	)
	pushErrorCounter = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "pushprox_client_push_errors_total",
			Help: "Number of push errors",
		},
	)
	pollErrorCounter = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "pushprox_client_poll_errors_total",
			Help: "Number of poll errors",
		},
	)
)

func init() {
	prometheus.MustRegister(pushErrorCounter, pollErrorCounter, scrapeErrorCounter)
}

// Coordinator for scrape requests and responses
type Coordinator struct {
	logger log.Logger
}

func (c *Coordinator) handleErr(request *http.Request, client *http.Client, err error) {
	level.Error(c.logger).Log("err", err)
	scrapeErrorCounter.Inc()
	resp := &http.Response{
		StatusCode: http.StatusInternalServerError,
		Body:       ioutil.NopCloser(strings.NewReader(err.Error())),
		Header:     http.Header{},
	}
	if err = c.doPush(resp, request, client); err != nil {
		pushErrorCounter.Inc()
		level.Warn(c.logger).Log("msg", "Failed to push failed scrape response:", "err", err)
		return
	}
	level.Info(c.logger).Log("msg", "Pushed failed scrape response")
}

func (c *Coordinator) doScrape(request *http.Request, client *http.Client) {
	logger := log.With(c.logger, "scrape_id", request.Header.Get("id"))
	timeout, err := util.GetHeaderTimeout(request.Header)
	if err != nil {
		c.handleErr(request, client, err)
		return
	}
	ctx, cancel := context.WithTimeout(request.Context(), timeout)
	defer cancel()
	request = request.WithContext(ctx)
	// We cannot handle https requests at the proxy, as we would only
	// see a CONNECT, so use a URL parameter to trigger it.
	params := request.URL.Query()
	if params.Get("_scheme") == "https" {
		request.URL.Scheme = "https"
		params.Del("_scheme")
		request.URL.RawQuery = params.Encode()
	}

	if request.URL.Hostname() != *myFqdn {
		c.handleErr(request, client, errors.New("scrape target doesn't match client fqdn"))
		return
	}

	scrapeResp, err := client.Do(request)
	if err != nil {
		msg := fmt.Sprintf("failed to scrape %s", request.URL.String())
		c.handleErr(request, client, errors.Wrap(err, msg))
		return
	}
	level.Info(logger).Log("msg", "Retrieved scrape response")
	if err = c.doPush(scrapeResp, request, client); err != nil {
		pushErrorCounter.Inc()
		level.Warn(logger).Log("msg", "Failed to push scrape response:", "err", err)
		return
	}
	level.Info(logger).Log("msg", "Pushed scrape result")
}

// Report the result of the scrape back up to the proxy.
func (c *Coordinator) doPush(resp *http.Response, origRequest *http.Request, client *http.Client) error {
	resp.Header.Set("id", origRequest.Header.Get("id")) // Link the request and response
	// Remaining scrape deadline.
	deadline, _ := origRequest.Context().Deadline()
	resp.Header.Set("X-Prometheus-Scrape-Timeout", fmt.Sprintf("%f", float64(time.Until(deadline))/1e9))

	base, err := url.Parse(*proxyURL)
	if err != nil {
		return err
	}
	u, err := url.Parse("push")
	if err != nil {
		return err
	}
	url := base.ResolveReference(u)

	buf := &bytes.Buffer{}
	resp.Write(buf)
	request := &http.Request{
		Method:        "POST",
		URL:           url,
		Body:          ioutil.NopCloser(buf),
		ContentLength: int64(buf.Len()),
	}
	request = request.WithContext(origRequest.Context())
	if _, err = client.Do(request); err != nil {
		return err
	}
	return nil
}

func loop(c Coordinator, client *http.Client) error {
	base, err := url.Parse(*proxyURL)
	if err != nil {
		level.Error(c.logger).Log("msg", "Error parsing url:", "err", err)
		return errors.Wrap(err, "error parsing url")
	}
	u, err := url.Parse("poll")
	if err != nil {
		level.Error(c.logger).Log("msg", "Error parsing url:", "err", err)
		return errors.Wrap(err, "error parsing url poll")
	}
	url := base.ResolveReference(u)
	resp, err := client.Post(url.String(), "", strings.NewReader(*myFqdn))
	if err != nil {
		level.Error(c.logger).Log("msg", "Error polling:", "err", err)
		return errors.Wrap(err, "error polling")
	}
	defer resp.Body.Close()

	request, err := http.ReadRequest(bufio.NewReader(resp.Body))
	if err != nil {
		level.Error(c.logger).Log("msg", "Error reading request:", "err", err)
		return errors.Wrap(err, "error reading request")
	}
	level.Info(c.logger).Log("msg", "Got scrape request", "scrape_id", request.Header.Get("id"), "url", request.URL)

	request.RequestURI = ""

	go c.doScrape(request, client)

	return nil
}

// decorrelated Jitter increases the maximum jitter based on the last random value.
type decorrelatedJitter struct {
	duration float64 // sleep time
	min      float64 // min sleep time
	cap      float64 // max sleep time
}

func newJitter() decorrelatedJitter {
	rand.Seed(time.Now().UnixNano())
	return decorrelatedJitter{
		duration: 1,
		min:      1,
		cap:      10,
	}
}

func (d *decorrelatedJitter) calc() {
	d.duration = math.Min(d.cap, d.min+rand.Float64()*(d.duration*3-d.min))
}

func (d *decorrelatedJitter) sleep() {
	time.Sleep(time.Duration(d.duration))
}

func main() {
	promlogConfig := promlog.Config{}
	flag.AddFlags(kingpin.CommandLine, &promlogConfig)
	kingpin.HelpFlag.Short('h')
	kingpin.Parse()
	logger := promlog.New(&promlogConfig)
	coordinator := Coordinator{logger: logger}

	if *proxyURL == "" {
		level.Error(coordinator.logger).Log("msg", "--proxy-url flag must be specified.")
		os.Exit(1)
	}
	// Make sure proxyURL ends with a single '/'
	*proxyURL = strings.TrimRight(*proxyURL, "/") + "/"
	level.Info(coordinator.logger).Log("msg", "URL and FQDN info", "proxy_url", *proxyURL, "fqdn", *myFqdn)

	tlsConfig := &tls.Config{}
	if *tlsCert != "" {
		cert, err := tls.LoadX509KeyPair(*tlsCert, *tlsKey)
		if err != nil {
			level.Error(coordinator.logger).Log("msg", "Certificate or Key is invalid", "err", err)
			os.Exit(1)
		}

		// Setup HTTPS client
		tlsConfig.Certificates = []tls.Certificate{cert}

		tlsConfig.BuildNameToCertificate()
	}

	if *caCertFile != "" {
		caCert, err := ioutil.ReadFile(*caCertFile)
		if err != nil {
			level.Error(coordinator.logger).Log("msg", "Not able to read cacert file", "err", err)
			os.Exit(1)
		}
		caCertPool := x509.NewCertPool()
		if ok := caCertPool.AppendCertsFromPEM(caCert); !ok {
			level.Error(coordinator.logger).Log("msg", "Failed to use cacert file as ca certificate")
			os.Exit(1)
		}

		tlsConfig.RootCAs = caCertPool
	}

	if *metricsAddr != "" {
		go func() {
			if err := http.ListenAndServe(*metricsAddr, promhttp.Handler()); err != nil {
				level.Warn(coordinator.logger).Log("msg", "ListenAndServe", "err", err)
			}
		}()
	}

	transport := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		DialContext: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
			DualStack: true,
		}).DialContext,
		MaxIdleConns:          100,
		IdleConnTimeout:       90 * time.Second,
		TLSHandshakeTimeout:   10 * time.Second,
		ExpectContinueTimeout: 1 * time.Second,
		TLSClientConfig:       tlsConfig,
	}

	jitter := newJitter()
	client := &http.Client{Transport: transport}
	for {
		err := loop(coordinator, client)
		if err != nil {
			pollErrorCounter.Inc()
			jitter.sleep()
			continue
		}
	}
}
<|endoftext|>


<commit_before>package runkeeper

import (
	"log"
	"time"

	runkeeper "github.com/svdberg/syncmysport-runkeeper/Godeps/_workspace/src/github.com/c9s/go-runkeeper"
	dm "github.com/svdberg/syncmysport-runkeeper/datamodel"
)

const API = "API"

func ConvertToActivity(rkActivity *runkeeper.FitnessActivity) *dm.Activity {
	returnActivity := dm.CreateActivity()
	if rkActivity.Type == "Other" {
		returnActivity.Type = "Activity"
	} else {
		returnActivity.Type = rkActivity.Type
	}

	//RK time is 'Local', convert to UTC
	sourceLocation := time.FixedZone("RKSourceLocation", rkActivity.UtcOffset*60*60)
	correctedTime := time.Time(rkActivity.StartTime).In(sourceLocation)
	log.Printf("RK Local date: %s, start date: %s, unix: %d, offset: %d", time.Time(rkActivity.StartTime), correctedTime, time.Time(rkActivity.StartTime).Unix(), rkActivity.UtcOffset)
	returnActivity.StartTime = int(time.Time(correctedTime).Unix())
	returnActivity.UtcOffSet = rkActivity.UtcOffset
	returnActivity.Duration = int(rkActivity.Duration)
	returnActivity.Name = rkActivity.Notes
	returnActivity.Notes = rkActivity.Notes
	returnActivity.Private = false
	returnActivity.Stationary = rkActivity.HasMap
	returnActivity.AverageHeartRate = rkActivity.AverageHeartRate
	returnActivity.Calories = rkActivity.TotalCalories
	returnActivity.Distance = rkActivity.TotalDistance
	returnActivity.GPS = convertFromPath(rkActivity.Path)
	returnActivity.HeartRate = convertFromHR(rkActivity.HeartRate)

	// log.Printf("INPUT: %s, OUTPUT: %s", rkActivity, returnActivity)
	return returnActivity
}

func ConvertToRkActivity(activity *dm.Activity) *runkeeper.FitnessActivityNew {
	rkActivity := runkeeper.CreateNewFitnessActivity(activity.Name, float64(activity.Duration))

	rkActivity.Type = activity.Type
	//runkeeper only nows the following types:
	//Running, Cycling, Mountain Biking, Walking,
	//Hiking, Downhill Skiing, Cross-Country Skiing,
	//Snowboarding, Skating, Swimming, Wheelchair, Rowing, Elliptical, Other
	//
	//check if Type is one of these, otherwise Other.
	rkKnownTypes := map[string]string{
		"Running":              "Running",
		"Cycling":              "Cycling",
		"Mountain Biking":      "Mountain Biking",
		"Walking":              "Walking",
		"Hiking":               "Hiking",
		"Downhill Skiing":      "Downhill Skiing",
		"Cross-Country Skiing": "Cross-Country Skiing",
		"Snowboarding":         "Snowboarding",
		"Skating":              "Skating",
		"Swimming":             "Swimming",
		"Wheelchair":           "Wheelchair",
		"Rowing":               "Rowing",
		"Elliptical":           "Elliptical",
		"Other":                "Other"}

	_, ok := rkKnownTypes[activity.Type]
	if !ok {
		rkActivity.Type = "Other"
	}

	//runkeeper times are in local timezones, so covert back to the local time
	rkLocalLocation := time.FixedZone("rkZone", activity.UtcOffSet*60*60)
	rkActivity.StartTime = runkeeper.Time(time.Unix(int64(activity.StartTime), 0).In(rkLocalLocation))
	log.Printf("SMS time: %s, converted to RK time: %s for offset: %d", activity.StartTime, rkActivity.StartTime, activity.UtcOffSet)
	rkActivity.Notes = activity.Name
	rkActivity.TotalDistance = activity.Distance
	rkActivity.AverageHeartRate = activity.AverageHeartRate
	rkActivity.TotalCalories = activity.Calories
	rkActivity.Source = activity.Source
	rkActivity.EntryMode = API

	rkActivity.Path = convertToPath(activity.GPS)
	rkActivity.HeartRate = convertToHR(activity.HeartRate)
	return rkActivity
}

func convertToPath(gps []dm.GPS) []runkeeper.Path {
	rkPath := make([]runkeeper.Path, len(gps))
	for i, gp := range gps {
		rkPath[i] = runkeeper.Path{gp.Altitude, gp.Longitude, "gps", gp.Latitude, gp.Timestamp}
	}
	return rkPath
}

func convertFromPath(path []runkeeper.Path) []dm.GPS {
	dmPath := make([]dm.GPS, len(path))
	for i, rp := range path {
		dmPath[i] = dm.GPS{rp.Timestamp, rp.Altitude, rp.Longitude, rp.Latitude}
	}
	return dmPath
}

func convertToHR(hr []dm.HeartRate) []runkeeper.HeartRate {
	rkHr := make([]runkeeper.HeartRate, len(hr))
	for i, h := range hr {
		rkHr[i] = runkeeper.HeartRate{h.Timestamp, h.Heartrate}
	}
	return rkHr
}

func convertFromHR(rkHr []runkeeper.HeartRate) []dm.HeartRate {
	dmHr := make([]dm.HeartRate, len(rkHr))
	for i, h := range rkHr {
		dmHr[i] = dm.HeartRate{h.TimeStamp, h.HearRateNr}
	}
	return dmHr
}
<commit_msg>Fix time logic error for RK<commit_after>package runkeeper

import (
	"log"
	"time"

	runkeeper "github.com/svdberg/syncmysport-runkeeper/Godeps/_workspace/src/github.com/c9s/go-runkeeper"
	dm "github.com/svdberg/syncmysport-runkeeper/datamodel"
)

const API = "API"

func ConvertToActivity(rkActivity *runkeeper.FitnessActivity) *dm.Activity {
	returnActivity := dm.CreateActivity()
	if rkActivity.Type == "Other" {
		returnActivity.Type = "Activity"
	} else {
		returnActivity.Type = rkActivity.Type
	}

	//RK time is 'Local', convert to UTC
	//TODO. This might be wrong and perhaps should be the negative zone (e.g. -1 * ..)
	negativeOffset := -1 * rkActivity.UtcOffset * 60 * 60
	sourceLocation := time.FixedZone("RKSourceLocation", negativeOffset)
	correctedTime := time.Time(rkActivity.StartTime).In(sourceLocation)
	log.Printf("RK Local date: %s, start date: %s, unix: %d, offset: %d", time.Time(rkActivity.StartTime), correctedTime, time.Time(rkActivity.StartTime).Unix(), rkActivity.UtcOffset)
	returnActivity.StartTime = int(time.Time(correctedTime).Unix())
	returnActivity.UtcOffSet = rkActivity.UtcOffset
	returnActivity.Duration = int(rkActivity.Duration)
	returnActivity.Name = rkActivity.Notes
	returnActivity.Notes = rkActivity.Notes
	returnActivity.Private = false
	returnActivity.Stationary = rkActivity.HasMap
	returnActivity.AverageHeartRate = rkActivity.AverageHeartRate
	returnActivity.Calories = rkActivity.TotalCalories
	returnActivity.Distance = rkActivity.TotalDistance
	returnActivity.GPS = convertFromPath(rkActivity.Path)
	returnActivity.HeartRate = convertFromHR(rkActivity.HeartRate)

	// log.Printf("INPUT: %s, OUTPUT: %s", rkActivity, returnActivity)
	return returnActivity
}

func ConvertToRkActivity(activity *dm.Activity) *runkeeper.FitnessActivityNew {
	rkActivity := runkeeper.CreateNewFitnessActivity(activity.Name, float64(activity.Duration))

	rkActivity.Type = activity.Type
	//runkeeper only nows the following types:
	//Running, Cycling, Mountain Biking, Walking,
	//Hiking, Downhill Skiing, Cross-Country Skiing,
	//Snowboarding, Skating, Swimming, Wheelchair, Rowing, Elliptical, Other
	//
	//check if Type is one of these, otherwise Other.
	rkKnownTypes := map[string]string{
		"Running":              "Running",
		"Cycling":              "Cycling",
		"Mountain Biking":      "Mountain Biking",
		"Walking":              "Walking",
		"Hiking":               "Hiking",
		"Downhill Skiing":      "Downhill Skiing",
		"Cross-Country Skiing": "Cross-Country Skiing",
		"Snowboarding":         "Snowboarding",
		"Skating":              "Skating",
		"Swimming":             "Swimming",
		"Wheelchair":           "Wheelchair",
		"Rowing":               "Rowing",
		"Elliptical":           "Elliptical",
		"Other":                "Other"}

	_, ok := rkKnownTypes[activity.Type]
	if !ok {
		rkActivity.Type = "Other"
	}

	//runkeeper times are in local timezones, so covert back to the local time
	rkLocalLocation := time.FixedZone("rkZone", activity.UtcOffSet*60*60)
	rkActivity.StartTime = runkeeper.Time(time.Unix(int64(activity.StartTime), 0).In(rkLocalLocation))
	log.Printf("SMS time: %s, converted to RK time: %s for offset: %d", activity.StartTime, rkActivity.StartTime, activity.UtcOffSet)
	rkActivity.Notes = activity.Name
	rkActivity.TotalDistance = activity.Distance
	rkActivity.AverageHeartRate = activity.AverageHeartRate
	rkActivity.TotalCalories = activity.Calories
	rkActivity.Source = activity.Source
	rkActivity.EntryMode = API

	rkActivity.Path = convertToPath(activity.GPS)
	rkActivity.HeartRate = convertToHR(activity.HeartRate)
	return rkActivity
}

func convertToPath(gps []dm.GPS) []runkeeper.Path {
	rkPath := make([]runkeeper.Path, len(gps))
	for i, gp := range gps {
		rkPath[i] = runkeeper.Path{gp.Altitude, gp.Longitude, "gps", gp.Latitude, gp.Timestamp}
	}
	return rkPath
}

func convertFromPath(path []runkeeper.Path) []dm.GPS {
	dmPath := make([]dm.GPS, len(path))
	for i, rp := range path {
		dmPath[i] = dm.GPS{rp.Timestamp, rp.Altitude, rp.Longitude, rp.Latitude}
	}
	return dmPath
}

func convertToHR(hr []dm.HeartRate) []runkeeper.HeartRate {
	rkHr := make([]runkeeper.HeartRate, len(hr))
	for i, h := range hr {
		rkHr[i] = runkeeper.HeartRate{h.Timestamp, h.Heartrate}
	}
	return rkHr
}

func convertFromHR(rkHr []runkeeper.HeartRate) []dm.HeartRate {
	dmHr := make([]dm.HeartRate, len(rkHr))
	for i, h := range rkHr {
		dmHr[i] = dm.HeartRate{h.TimeStamp, h.HearRateNr}
	}
	return dmHr
}
<|endoftext|>


<commit_before>package irmago

import (
	"crypto/rand"
	"crypto/sha256"
	"encoding/base64"
	"errors"
	"fmt"
	"math/big"
	"strconv"

	"strings"

	"github.com/mhe/gabi"
)

// KeysharePinRequestor is used to asking the user for his PIN.
type KeysharePinRequestor interface {
	AskPin(remainingAttempts int, callback func(proceed bool, pin string))
}

type keyshareSessionHandler interface {
	KeyshareDone(message interface{})
	KeyshareCancelled()
	KeyshareBlocked(duration int)
	KeyshareError(err error)
}

type keyshareSession struct {
	session        Session
	builders       gabi.ProofBuilderList
	transports     map[SchemeManagerIdentifier]*HTTPTransport
	sessionHandler keyshareSessionHandler
	pinRequestor   KeysharePinRequestor
	keyshareServer *keyshareServer
}

type keyshareServer struct {
	URL        string              `json:"url"`
	Username   string              `json:"username"`
	Nonce      []byte              `json:"nonce"`
	PrivateKey *paillierPrivateKey `json:"keyPair"`
	token      string
}

type keyshareRegistration struct {
	Username  string             `json:"username"`
	Pin       string             `json:"pin"`
	PublicKey *paillierPublicKey `json:"publicKey"`
}

type keyshareAuthorization struct {
	Status     string   `json:"status"`
	Candidates []string `json:"candidates"`
}

type keysharePinMessage struct {
	Username string `json:"id"`
	Pin      string `json:"pin"`
}

type keysharePinStatus struct {
	Status  string `json:"status"`
	Message string `json:"message"`
}

type publicKeyIdentifier struct {
	Issuer  string `json:"issuer"`
	Counter uint   `json:"counter"`
}

func (pki *publicKeyIdentifier) UnmarshalText(text []byte) error {
	str := string(text)
	index := strings.LastIndex(str, "-")
	if index == -1 {
		return errors.New("Invalid publicKeyIdentifier")
	}
	counter, err := strconv.Atoi(str[index+1:])
	if err != nil {
		return err
	}
	*pki = publicKeyIdentifier{Issuer: str[:index], Counter: uint(counter)}
	return nil
}

func (pki *publicKeyIdentifier) MarshalText() (text []byte, err error) {
	return []byte(fmt.Sprintf("%s-%d", pki.Issuer, pki.Counter)), nil
}

type proofPCommitmentMap struct {
	Commitments map[publicKeyIdentifier]*gabi.ProofPCommitment `json:"c"`
}

// KeyshareHandler is used for asking the user for his email address and PIN,
// for registering at a keyshare server.
type KeyshareHandler interface {
	StartRegistration(manager *SchemeManager, registrationCallback func(email, pin string))
}

const (
	kssUsernameHeader = "IRMA_Username"
	kssAuthHeader     = "IRMA_Authorization"
	kssAuthorized     = "authorized"
	kssTokenExpired   = "expired"
	kssPinSuccess     = "success"
	kssPinFailure     = "failure"
	kssPinError       = "error"
)

func newKeyshareServer(privatekey *paillierPrivateKey, url, email string) (ks *keyshareServer, err error) {
	ks = &keyshareServer{
		Nonce:      make([]byte, 32),
		URL:        url,
		Username:   email,
		PrivateKey: privatekey,
	}
	_, err = rand.Read(ks.Nonce)
	return
}

func (ks *keyshareServer) HashedPin(pin string) string {
	hash := sha256.Sum256(append(ks.Nonce, []byte(pin)...))
	return base64.RawStdEncoding.EncodeToString(hash[:])
}

// startKeyshareSession starts and completes the entire keyshare protocol with all involved keyshare servers
// for the specified session, merging the keyshare proofs into the specified ProofBuilder's.
// The user's pin is retrieved using the KeysharePinRequestor, repeatedly, until either it is correct; or the
// user cancels; or one of the keyshare servers blocks us.
// Error, blocked or success of the keyshare session is reported back to the keyshareSessionHandler.
func startKeyshareSession(
	session Session,
	builders gabi.ProofBuilderList,
	sessionHandler keyshareSessionHandler,
	pin KeysharePinRequestor,
) {
	ksscount := 0
	for _, managerID := range session.SchemeManagers() {
		if MetaStore.SchemeManagers[managerID].Distributed() {
			ksscount++
			if _, registered := Manager.keyshareServers[managerID]; !registered {
				err := errors.New("Not registered to keyshare server of scheme manager " + managerID.String())
				sessionHandler.KeyshareError(err)
				return
			}
		}
	}
	if _, issuing := session.(*IssuanceRequest); issuing && ksscount > 1 {
		err := errors.New("Issuance session involving more than one keyshare servers are not supported")
		sessionHandler.KeyshareError(err)
		return
	}

	ks := &keyshareSession{
		session:        session,
		builders:       builders,
		sessionHandler: sessionHandler,
		transports:     map[SchemeManagerIdentifier]*HTTPTransport{},
		pinRequestor:   pin,
	}

	askPin := false

	for _, managerID := range session.SchemeManagers() {
		if !MetaStore.SchemeManagers[managerID].Distributed() {
			continue
		}

		ks.keyshareServer = Manager.keyshareServers[managerID]
		transport := NewHTTPTransport(ks.keyshareServer.URL)
		transport.SetHeader(kssUsernameHeader, ks.keyshareServer.Username)
		transport.SetHeader(kssAuthHeader, ks.keyshareServer.token)
		ks.transports[managerID] = transport

		authstatus := &keyshareAuthorization{}
		err := transport.Post("users/isAuthorized", authstatus, "")
		if err != nil {
			ks.sessionHandler.KeyshareError(err)
			return
		}
		switch authstatus.Status {
		case kssAuthorized: // nop
		case kssTokenExpired:
			askPin = true
		default:
			ks.sessionHandler.KeyshareError(errors.New("Keyshare server returned unrecognized authorization status"))
			return
		}
	}

	if askPin {
		ks.VerifyPin(-1)
	} else {
		ks.GetCommitments()
	}
}

// Ask for a pin, repeatedly if necessary, and either continue the keyshare protocol
// with authorization, or stop the keyshare protocol and inform of failure.
func (ks *keyshareSession) VerifyPin(attempts int) {
	ks.pinRequestor.AskPin(attempts, func(proceed bool, pin string) {
		success, attemptsRemaining, blocked, err := ks.verifyPinAttempt(pin)
		if err != nil {
			ks.sessionHandler.KeyshareError(err)
			return
		}
		if !proceed {
			ks.sessionHandler.KeyshareCancelled()
		}
		if blocked != 0 {
			ks.sessionHandler.KeyshareBlocked(blocked)
			return
		}
		if success {
			ks.GetCommitments()
			return
		}
		// Not successful but no error and not yet blocked: try again
		ks.VerifyPin(attemptsRemaining)
	})
}

// Verify the specified pin at each of the keyshare servers involved in the specified session.
// - If the pin did not verify at one of the keyshare servers but there are attempts remaining,
// the amount of remaining attempts is returned as the second return value.
// - If the pin did not verify at one of the keyshare servers and there are no attempts remaining,
// the amount of time for which we are blocked at the keyshare server is returned as the third
// parameter.
// - If this or anything else (specified in err) goes wrong, success will be false.
// If all is ok, success will be true.
func (ks *keyshareSession) verifyPinAttempt(pin string) (success bool, tries int, blocked int, err error) {
	for _, managerID := range ks.session.SchemeManagers() {
		if !MetaStore.SchemeManagers[managerID].Distributed() {
			continue
		}

		kss := Manager.keyshareServers[managerID]
		transport := ks.transports[managerID]
		pinmsg := keysharePinMessage{Username: kss.Username, Pin: kss.HashedPin(pin)}
		pinresult := &keysharePinStatus{}
		err = transport.Post("users/verify/pin", pinresult, pinmsg)
		if err != nil {
			return
		}

		switch pinresult.Status {
		case kssPinSuccess:
			kss.token = pinresult.Message
			transport.SetHeader(kssAuthHeader, kss.token)
		case kssPinFailure:
			tries, err = strconv.Atoi(pinresult.Message)
			if err != nil {
				return
			}
			return
		case kssPinError:
			blocked, err = strconv.Atoi(pinresult.Message)
			if err != nil {
				return
			}
			return
		default:
			err = errors.New("Keyshare server returned unrecognized PIN status")
			return
		}
	}

	success = true
	return
}

// GetCommitments gets the commitments (first message in Schnorr zero-knowledge protocol)
// of all keyshare servers of their part of the private key, and merges these commitments
// in our own proof builders.
func (ks *keyshareSession) GetCommitments() {
	pkids := map[SchemeManagerIdentifier][]*publicKeyIdentifier{}
	commitments := map[publicKeyIdentifier]*gabi.ProofPCommitment{}

	// For each scheme manager, build a list of public keys under this manager
	// that we will use in the keyshare protocol with the keyshare server of this manager
	for _, builder := range ks.builders {
		pk := builder.PublicKey()
		managerID := NewIssuerIdentifier(pk.Issuer).SchemeManagerIdentifier()
		if !MetaStore.SchemeManagers[managerID].Distributed() {
			continue
		}
		if _, contains := pkids[managerID]; !contains {
			pkids[managerID] = []*publicKeyIdentifier{}
		}
		pkids[managerID] = append(pkids[managerID], &publicKeyIdentifier{Issuer: pk.Issuer, Counter: pk.Counter})
	}

	// Now inform each keyshare server of with respect to which public keys
	// we want them to send us commitments
	for _, managerID := range ks.session.SchemeManagers() {
		if !MetaStore.SchemeManagers[managerID].Distributed() {
			continue
		}

		transport := ks.transports[managerID]
		comms := &proofPCommitmentMap{}
		err := transport.Post("prove/getCommitments", comms, pkids[managerID])
		if err != nil {
			ks.sessionHandler.KeyshareError(err)
			return
		}
		for pki, c := range comms.Commitments {
			commitments[pki] = c
		}
	}

	// Merge in the commitments
	for _, builder := range ks.builders {
		pk := builder.PublicKey()
		pki := publicKeyIdentifier{Issuer: pk.Issuer, Counter: pk.Counter}
		comm, distributed := commitments[pki]
		if !distributed {
			continue
		}
		builder.MergeProofPCommitment(comm)
	}

	ks.GetProofPs()
}

// GetProofPs uses the combined commitments of all keyshare servers and ourself
// to calculate the challenge, which is sent to the keyshare servers in order to
// receive their responses (2nd and 3rd message in Schnorr zero-knowledge protocol).
func (ks *keyshareSession) GetProofPs() {
	_, issig := ks.session.(*SignatureRequest)
	_, issuing := ks.session.(*IssuanceRequest)
	challenge := ks.builders.Challenge(ks.session.GetContext(), ks.session.GetNonce(), issig)
	kssChallenge := challenge

	// In disclosure or signature sessions the challenge is Paillier encrypted.
	if !issuing {
		bytes, err := ks.keyshareServer.PrivateKey.Encrypt(challenge.Bytes())
		if err != nil {
			ks.sessionHandler.KeyshareError(err)
		}
		kssChallenge = new(big.Int).SetBytes(bytes)
	}

	// Post the challenge, obtaining JWT's containing the ProofP's
	responses := map[SchemeManagerIdentifier]string{}
	for _, managerID := range ks.session.SchemeManagers() {
		transport, distributed := ks.transports[managerID]
		if !distributed {
			continue
		}
		var jwt string
		err := transport.Post("prove/getResponse", &jwt, kssChallenge)
		if err != nil {
			ks.sessionHandler.KeyshareError(err)
			return
		}
		responses[managerID] = jwt
	}

	ks.Finish(challenge, responses)
}

// Finish the keyshare protocol: in case of issuance, put the keyshare jwt in the
// IssueCommitmentMessage; in case of disclosure and signing, parse each keyshare jwt,
// merge in the received ProofP's, and finish.
func (ks *keyshareSession) Finish(challenge *big.Int, responses map[SchemeManagerIdentifier]string) {
	switch ks.session.(type) {
	case *DisclosureRequest: // Can't use fallthrough in a type switch in go
		ks.finishDisclosureOrSigning(challenge, responses)
	case *SignatureRequest: // So we have to do this in a separate method
		ks.finishDisclosureOrSigning(challenge, responses)
	case *IssuanceRequest:
		// Calculate IssueCommitmentMessage, without merging in any of the received ProofP's:
		// instead, include the keyshare server's JWT in the IssueCommitmentMessage for the
		// issuance server to verify
		list, err := ks.builders.BuildDistributedProofList(challenge, nil)
		if err != nil {
			ks.sessionHandler.KeyshareError(err)
			return
		}
		message := gabi.IssueCommitmentMessage{Proofs: list, Nonce2: ks.session.(*IssuanceRequest).state.nonce2}
		for _, response := range responses {
			message.ProofPjwt = response
			break
		}
		ks.sessionHandler.KeyshareDone(message)
	}
}

func (ks *keyshareSession) finishDisclosureOrSigning(challenge *big.Int, responses map[SchemeManagerIdentifier]string) {
	proofPs := make([]*gabi.ProofP, len(ks.builders))
	for i, builder := range ks.builders {
		// Parse each received JWT
		managerID := NewIssuerIdentifier(builder.PublicKey().Issuer).SchemeManagerIdentifier()
		if !MetaStore.SchemeManagers[managerID].Distributed() {
			continue
		}
		msg := struct {
			ProofP *gabi.ProofP
		}{}
		_, err := jwtDecode(responses[managerID], &msg)
		if err != nil {
			ks.sessionHandler.KeyshareError(err)
			return
		}

		// Decrypt the responses and populate a slice of ProofP's
		proofPs[i] = msg.ProofP
		bytes, err := ks.keyshareServer.PrivateKey.Decrypt(proofPs[i].SResponse.Bytes())
		if err != nil {
			ks.sessionHandler.KeyshareError(err)
			return
		}
		proofPs[i].SResponse = new(big.Int).SetBytes(bytes)
	}

	// Create merged proofs and finish protocol
	list, err := ks.builders.BuildDistributedProofList(challenge, proofPs)
	if err != nil {
		ks.sessionHandler.KeyshareError(err)
		return
	}
	ks.sessionHandler.KeyshareDone(list)
}
<commit_msg>Send multiple ProofP JWTs in issuance<commit_after>package irmago

import (
	"crypto/rand"
	"crypto/sha256"
	"encoding/base64"
	"errors"
	"fmt"
	"math/big"
	"strconv"

	"strings"

	"github.com/mhe/gabi"
)

// KeysharePinRequestor is used to asking the user for his PIN.
type KeysharePinRequestor interface {
	AskPin(remainingAttempts int, callback func(proceed bool, pin string))
}

type keyshareSessionHandler interface {
	KeyshareDone(message interface{})
	KeyshareCancelled()
	KeyshareBlocked(duration int)
	KeyshareError(err error)
}

type keyshareSession struct {
	session        Session
	builders       gabi.ProofBuilderList
	transports     map[SchemeManagerIdentifier]*HTTPTransport
	sessionHandler keyshareSessionHandler
	pinRequestor   KeysharePinRequestor
	keyshareServer *keyshareServer
}

type keyshareServer struct {
	URL        string              `json:"url"`
	Username   string              `json:"username"`
	Nonce      []byte              `json:"nonce"`
	PrivateKey *paillierPrivateKey `json:"keyPair"`
	token      string
}

type keyshareRegistration struct {
	Username  string             `json:"username"`
	Pin       string             `json:"pin"`
	PublicKey *paillierPublicKey `json:"publicKey"`
}

type keyshareAuthorization struct {
	Status     string   `json:"status"`
	Candidates []string `json:"candidates"`
}

type keysharePinMessage struct {
	Username string `json:"id"`
	Pin      string `json:"pin"`
}

type keysharePinStatus struct {
	Status  string `json:"status"`
	Message string `json:"message"`
}

type publicKeyIdentifier struct {
	Issuer  string `json:"issuer"`
	Counter uint   `json:"counter"`
}

func (pki *publicKeyIdentifier) UnmarshalText(text []byte) error {
	str := string(text)
	index := strings.LastIndex(str, "-")
	if index == -1 {
		return errors.New("Invalid publicKeyIdentifier")
	}
	counter, err := strconv.Atoi(str[index+1:])
	if err != nil {
		return err
	}
	*pki = publicKeyIdentifier{Issuer: str[:index], Counter: uint(counter)}
	return nil
}

func (pki *publicKeyIdentifier) MarshalText() (text []byte, err error) {
	return []byte(fmt.Sprintf("%s-%d", pki.Issuer, pki.Counter)), nil
}

type proofPCommitmentMap struct {
	Commitments map[publicKeyIdentifier]*gabi.ProofPCommitment `json:"c"`
}

// KeyshareHandler is used for asking the user for his email address and PIN,
// for registering at a keyshare server.
type KeyshareHandler interface {
	StartRegistration(manager *SchemeManager, registrationCallback func(email, pin string))
}

const (
	kssUsernameHeader = "IRMA_Username"
	kssAuthHeader     = "IRMA_Authorization"
	kssAuthorized     = "authorized"
	kssTokenExpired   = "expired"
	kssPinSuccess     = "success"
	kssPinFailure     = "failure"
	kssPinError       = "error"
)

func newKeyshareServer(privatekey *paillierPrivateKey, url, email string) (ks *keyshareServer, err error) {
	ks = &keyshareServer{
		Nonce:      make([]byte, 32),
		URL:        url,
		Username:   email,
		PrivateKey: privatekey,
	}
	_, err = rand.Read(ks.Nonce)
	return
}

func (ks *keyshareServer) HashedPin(pin string) string {
	hash := sha256.Sum256(append(ks.Nonce, []byte(pin)...))
	return base64.RawStdEncoding.EncodeToString(hash[:])
}

// startKeyshareSession starts and completes the entire keyshare protocol with all involved keyshare servers
// for the specified session, merging the keyshare proofs into the specified ProofBuilder's.
// The user's pin is retrieved using the KeysharePinRequestor, repeatedly, until either it is correct; or the
// user cancels; or one of the keyshare servers blocks us.
// Error, blocked or success of the keyshare session is reported back to the keyshareSessionHandler.
func startKeyshareSession(
	session Session,
	builders gabi.ProofBuilderList,
	sessionHandler keyshareSessionHandler,
	pin KeysharePinRequestor,
) {
	ksscount := 0
	for _, managerID := range session.SchemeManagers() {
		if MetaStore.SchemeManagers[managerID].Distributed() {
			ksscount++
			if _, registered := Manager.keyshareServers[managerID]; !registered {
				err := errors.New("Not registered to keyshare server of scheme manager " + managerID.String())
				sessionHandler.KeyshareError(err)
				return
			}
		}
	}
	if _, issuing := session.(*IssuanceRequest); issuing && ksscount > 1 {
		err := errors.New("Issuance session involving more than one keyshare servers are not supported")
		sessionHandler.KeyshareError(err)
		return
	}

	ks := &keyshareSession{
		session:        session,
		builders:       builders,
		sessionHandler: sessionHandler,
		transports:     map[SchemeManagerIdentifier]*HTTPTransport{},
		pinRequestor:   pin,
	}

	askPin := false

	for _, managerID := range session.SchemeManagers() {
		if !MetaStore.SchemeManagers[managerID].Distributed() {
			continue
		}

		ks.keyshareServer = Manager.keyshareServers[managerID]
		transport := NewHTTPTransport(ks.keyshareServer.URL)
		transport.SetHeader(kssUsernameHeader, ks.keyshareServer.Username)
		transport.SetHeader(kssAuthHeader, ks.keyshareServer.token)
		ks.transports[managerID] = transport

		authstatus := &keyshareAuthorization{}
		err := transport.Post("users/isAuthorized", authstatus, "")
		if err != nil {
			ks.sessionHandler.KeyshareError(err)
			return
		}
		switch authstatus.Status {
		case kssAuthorized: // nop
		case kssTokenExpired:
			askPin = true
		default:
			ks.sessionHandler.KeyshareError(errors.New("Keyshare server returned unrecognized authorization status"))
			return
		}
	}

	if askPin {
		ks.VerifyPin(-1)
	} else {
		ks.GetCommitments()
	}
}

// Ask for a pin, repeatedly if necessary, and either continue the keyshare protocol
// with authorization, or stop the keyshare protocol and inform of failure.
func (ks *keyshareSession) VerifyPin(attempts int) {
	ks.pinRequestor.AskPin(attempts, func(proceed bool, pin string) {
		success, attemptsRemaining, blocked, err := ks.verifyPinAttempt(pin)
		if err != nil {
			ks.sessionHandler.KeyshareError(err)
			return
		}
		if !proceed {
			ks.sessionHandler.KeyshareCancelled()
		}
		if blocked != 0 {
			ks.sessionHandler.KeyshareBlocked(blocked)
			return
		}
		if success {
			ks.GetCommitments()
			return
		}
		// Not successful but no error and not yet blocked: try again
		ks.VerifyPin(attemptsRemaining)
	})
}

// Verify the specified pin at each of the keyshare servers involved in the specified session.
// - If the pin did not verify at one of the keyshare servers but there are attempts remaining,
// the amount of remaining attempts is returned as the second return value.
// - If the pin did not verify at one of the keyshare servers and there are no attempts remaining,
// the amount of time for which we are blocked at the keyshare server is returned as the third
// parameter.
// - If this or anything else (specified in err) goes wrong, success will be false.
// If all is ok, success will be true.
func (ks *keyshareSession) verifyPinAttempt(pin string) (success bool, tries int, blocked int, err error) {
	for _, managerID := range ks.session.SchemeManagers() {
		if !MetaStore.SchemeManagers[managerID].Distributed() {
			continue
		}

		kss := Manager.keyshareServers[managerID]
		transport := ks.transports[managerID]
		pinmsg := keysharePinMessage{Username: kss.Username, Pin: kss.HashedPin(pin)}
		pinresult := &keysharePinStatus{}
		err = transport.Post("users/verify/pin", pinresult, pinmsg)
		if err != nil {
			return
		}

		switch pinresult.Status {
		case kssPinSuccess:
			kss.token = pinresult.Message
			transport.SetHeader(kssAuthHeader, kss.token)
		case kssPinFailure:
			tries, err = strconv.Atoi(pinresult.Message)
			if err != nil {
				return
			}
			return
		case kssPinError:
			blocked, err = strconv.Atoi(pinresult.Message)
			if err != nil {
				return
			}
			return
		default:
			err = errors.New("Keyshare server returned unrecognized PIN status")
			return
		}
	}

	success = true
	return
}

// GetCommitments gets the commitments (first message in Schnorr zero-knowledge protocol)
// of all keyshare servers of their part of the private key, and merges these commitments
// in our own proof builders.
func (ks *keyshareSession) GetCommitments() {
	pkids := map[SchemeManagerIdentifier][]*publicKeyIdentifier{}
	commitments := map[publicKeyIdentifier]*gabi.ProofPCommitment{}

	// For each scheme manager, build a list of public keys under this manager
	// that we will use in the keyshare protocol with the keyshare server of this manager
	for _, builder := range ks.builders {
		pk := builder.PublicKey()
		managerID := NewIssuerIdentifier(pk.Issuer).SchemeManagerIdentifier()
		if !MetaStore.SchemeManagers[managerID].Distributed() {
			continue
		}
		if _, contains := pkids[managerID]; !contains {
			pkids[managerID] = []*publicKeyIdentifier{}
		}
		pkids[managerID] = append(pkids[managerID], &publicKeyIdentifier{Issuer: pk.Issuer, Counter: pk.Counter})
	}

	// Now inform each keyshare server of with respect to which public keys
	// we want them to send us commitments
	for _, managerID := range ks.session.SchemeManagers() {
		if !MetaStore.SchemeManagers[managerID].Distributed() {
			continue
		}

		transport := ks.transports[managerID]
		comms := &proofPCommitmentMap{}
		err := transport.Post("prove/getCommitments", comms, pkids[managerID])
		if err != nil {
			ks.sessionHandler.KeyshareError(err)
			return
		}
		for pki, c := range comms.Commitments {
			commitments[pki] = c
		}
	}

	// Merge in the commitments
	for _, builder := range ks.builders {
		pk := builder.PublicKey()
		pki := publicKeyIdentifier{Issuer: pk.Issuer, Counter: pk.Counter}
		comm, distributed := commitments[pki]
		if !distributed {
			continue
		}
		builder.MergeProofPCommitment(comm)
	}

	ks.GetProofPs()
}

// GetProofPs uses the combined commitments of all keyshare servers and ourself
// to calculate the challenge, which is sent to the keyshare servers in order to
// receive their responses (2nd and 3rd message in Schnorr zero-knowledge protocol).
func (ks *keyshareSession) GetProofPs() {
	_, issig := ks.session.(*SignatureRequest)
	_, issuing := ks.session.(*IssuanceRequest)
	challenge := ks.builders.Challenge(ks.session.GetContext(), ks.session.GetNonce(), issig)
	kssChallenge := challenge

	// In disclosure or signature sessions the challenge is Paillier encrypted.
	if !issuing {
		bytes, err := ks.keyshareServer.PrivateKey.Encrypt(challenge.Bytes())
		if err != nil {
			ks.sessionHandler.KeyshareError(err)
		}
		kssChallenge = new(big.Int).SetBytes(bytes)
	}

	// Post the challenge, obtaining JWT's containing the ProofP's
	responses := map[SchemeManagerIdentifier]string{}
	for _, managerID := range ks.session.SchemeManagers() {
		transport, distributed := ks.transports[managerID]
		if !distributed {
			continue
		}
		var jwt string
		err := transport.Post("prove/getResponse", &jwt, kssChallenge)
		if err != nil {
			ks.sessionHandler.KeyshareError(err)
			return
		}
		responses[managerID] = jwt
	}

	ks.Finish(challenge, responses)
}

// Finish the keyshare protocol: in case of issuance, put the keyshare jwt in the
// IssueCommitmentMessage; in case of disclosure and signing, parse each keyshare jwt,
// merge in the received ProofP's, and finish.
func (ks *keyshareSession) Finish(challenge *big.Int, responses map[SchemeManagerIdentifier]string) {
	switch ks.session.(type) {
	case *DisclosureRequest: // Can't use fallthrough in a type switch in go
		ks.finishDisclosureOrSigning(challenge, responses)
	case *SignatureRequest: // So we have to do this in a separate method
		ks.finishDisclosureOrSigning(challenge, responses)
	case *IssuanceRequest:
		// Calculate IssueCommitmentMessage, without merging in any of the received ProofP's:
		// instead, include the keyshare server's JWT in the IssueCommitmentMessage for the
		// issuance server to verify
		list, err := ks.builders.BuildDistributedProofList(challenge, nil)
		if err != nil {
			ks.sessionHandler.KeyshareError(err)
			return
		}
		message := gabi.IssueCommitmentMessage{Proofs: list, Nonce2: ks.session.(*IssuanceRequest).state.nonce2}
		message.ProofPjwts = map[string]string{}
		for manager, response := range responses {
			message.ProofPjwts[manager.String()] = response
		}
		ks.sessionHandler.KeyshareDone(message)
	}
}

func (ks *keyshareSession) finishDisclosureOrSigning(challenge *big.Int, responses map[SchemeManagerIdentifier]string) {
	proofPs := make([]*gabi.ProofP, len(ks.builders))
	for i, builder := range ks.builders {
		// Parse each received JWT
		managerID := NewIssuerIdentifier(builder.PublicKey().Issuer).SchemeManagerIdentifier()
		if !MetaStore.SchemeManagers[managerID].Distributed() {
			continue
		}
		msg := struct {
			ProofP *gabi.ProofP
		}{}
		_, err := jwtDecode(responses[managerID], &msg)
		if err != nil {
			ks.sessionHandler.KeyshareError(err)
			return
		}

		// Decrypt the responses and populate a slice of ProofP's
		proofPs[i] = msg.ProofP
		bytes, err := ks.keyshareServer.PrivateKey.Decrypt(proofPs[i].SResponse.Bytes())
		if err != nil {
			ks.sessionHandler.KeyshareError(err)
			return
		}
		proofPs[i].SResponse = new(big.Int).SetBytes(bytes)
	}

	// Create merged proofs and finish protocol
	list, err := ks.builders.BuildDistributedProofList(challenge, proofPs)
	if err != nil {
		ks.sessionHandler.KeyshareError(err)
		return
	}
	ks.sessionHandler.KeyshareDone(list)
}
<|endoftext|>


<commit_before>// +build linux

package test

import (
	"testing"
)

func TestIntegration__date(t *testing.T) {
	cmd := Command("date", "-u", "--date", "@0").Trim()
	cmd.EqualT(t, "Thu Jan  1 00:00:00 UTC 1970")
	cmd.SuccessT(t)
}

func TestIntegration__unknown(t *testing.T) {
	cmd := CertManage("other").Trim()
	cmd.FailedT(t)
}

func TestIntegration__list(t *testing.T) {
	cmd := CertManage("list", "-count").Trim()
	cmd.CmpIntF(t, func(i int) bool { return i > 1 })
	cmd.SuccessT(t)
}

func TestIntegration__backup(t *testing.T) {
	cmd := CertManage("backup").Trim()
	if inCI() {
		// Travis-CI current must have something (openssl?) installed which adds
		// extra directories/certs
		// For now let's just mark this as pending in CI
		// TODO: https://github.com/adamdecaf/cert-manage/issues/105
		cmd.PendingT(t, "something wonky with 'ca-certificates.backup/.mozilla' dir")
	} else {
		cmd.EqualT(t, "Backup completed successfully")
	}
	cmd.SuccessT(t)
}

// TODO(adam): Need to run -whitelist and -restore
<commit_msg>test/linux: drop travis-ci pending<commit_after>// +build linux

package test

import (
	"testing"
)

func TestIntegration__date(t *testing.T) {
	cmd := Command("date", "-u", "--date", "@0").Trim()
	cmd.EqualT(t, "Thu Jan  1 00:00:00 UTC 1970")
	cmd.SuccessT(t)
}

func TestIntegration__unknown(t *testing.T) {
	cmd := CertManage("other").Trim()
	cmd.FailedT(t)
}

func TestIntegration__list(t *testing.T) {
	cmd := CertManage("list", "-count").Trim()
	cmd.CmpIntF(t, func(i int) bool { return i > 1 })
	cmd.SuccessT(t)
}

func TestIntegration__backup(t *testing.T) {
	cmd := CertManage("backup").Trim()
	cmd.EqualT(t, "Backup completed successfully")
	cmd.SuccessT(t)
}

// TODO(adam): Need to run -whitelist and -restore
<|endoftext|>


<commit_before>// This is a dumbed down version of the Tomb package by Gustavo Niemeyer.
//
// Copyright (c) 2011 - Gustavo Niemeyer <gustavo@niemeyer.net>
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//     * Redistributions of source code must retain the above copyright notice,
//       this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above copyright notice,
//       this list of conditions and the following disclaimer in the documentation
//       and/or other materials provided with the distribution.
//     * Neither the name of the copyright holder nor the names of its
//       contributors may be used to endorse or promote products derived from
//       this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package sdbot

import (
	"sync"
)

// Killable is a struct used to control the termination of goroutines.
type Killable struct {
	m     sync.Mutex
	dying chan struct{}
	dead  chan struct{}
}

func (k *Killable) init() {
	k.m.Lock()
	if k.dead == nil {
		k.dying = make(chan struct{})
		k.dead = make(chan struct{})
	}
	k.m.Unlock()
}

// Dying tells us if the Killable has been requested to terminate.
func (k *Killable) Dying() <-chan struct{} {
	k.init()
	return k.dying
}

// Wait waits until the goroutine has been terminated.
func (k *Killable) Wait() {
	k.init()
	<-k.dead
}

// Done requests termination and closes the dead channel.
func (k *Killable) Done() {
	k.Kill()
	close(k.dead)
}

// Kill requests termination but does not close the dead channel.
func (k *Killable) Kill() {
	k.init()
	k.m.Lock()
	defer k.m.Unlock()
	select {
	case <-k.dying:
	default:
		close(k.dying)
	}
}
<commit_msg>Remove killable.<commit_after><|endoftext|>


<commit_before>package goriak

import (
	"testing"
)

func TestHllBasics(t *testing.T) {
	res, err := Bucket("hll-test", "hlls").
		UpdateHyperLogLog().
		Add([]byte{1}).
		Add([]byte{2}).
		Add([]byte{3}).
		Add([]byte{4}).
		ReturnBody(true).
		Run(con())
	if err != nil {
		t.Error(err)
	}

	if res.Cardinality != 4 {
		t.Error("unexpected cardinality")
	}

	if res.NotFound == true {
		t.Error("not found was true")
	}

	// Get same
	res2, err := Bucket("hll-test", "hlls").
		GetHyperLogLog(res.Key).
		Run(con())

	if res2.Cardinality != 4 {
		t.Error("unexpected cardinality")
	}

	if res.NotFound == true {
		t.Error("not found was true")
	}

	if res2.Key != res.Key {
		t.Error("unexpected key")
	}

	// Add one more
	res3, err := Bucket("hll-test", "hlls").
		UpdateHyperLogLog().
		Add([]byte{5}).
		Key(res.Key).
		ReturnBody(true).
		Run(con())
	if err != nil {
		t.Error(err)
	}

	if res3.Cardinality != 5 {
		t.Error("unexpected cardinality")
	}

	if res3.NotFound == true {
		t.Error("not found was true")
	}

	if res3.Key != res.Key {
		t.Error("unexpected key")
	}
}

func TestHllWithoutReturn(t *testing.T) {
	res, err := Bucket("hll-test", "hlls").
		UpdateHyperLogLog().
		Add([]byte{1}).
		Run(con())

	if err != nil {
		t.Error(err)
	}

	if res != nil {
		t.Error("res was not nil")
	}
}
<commit_msg>#51 - Test for HLL AddMultiple<commit_after>package goriak

import (
	"testing"
)

func TestHllBasics(t *testing.T) {
	res, err := Bucket("hll-test", "hlls").
		UpdateHyperLogLog().
		Add([]byte{1}).
		Add([]byte{2}).
		Add([]byte{3}).
		Add([]byte{4}).
		ReturnBody(true).
		Run(con())
	if err != nil {
		t.Error(err)
	}

	if res.Cardinality != 4 {
		t.Error("unexpected cardinality")
	}

	if res.NotFound == true {
		t.Error("not found was true")
	}

	// Get same
	res2, err := Bucket("hll-test", "hlls").
		GetHyperLogLog(res.Key).
		Run(con())

	if res2.Cardinality != 4 {
		t.Error("unexpected cardinality")
	}

	if res.NotFound == true {
		t.Error("not found was true")
	}

	if res2.Key != res.Key {
		t.Error("unexpected key")
	}

	// Add one more
	res3, err := Bucket("hll-test", "hlls").
		UpdateHyperLogLog().
		Add([]byte{5}).
		Key(res.Key).
		ReturnBody(true).
		Run(con())
	if err != nil {
		t.Error(err)
	}

	if res3.Cardinality != 5 {
		t.Error("unexpected cardinality")
	}

	if res3.NotFound == true {
		t.Error("not found was true")
	}

	if res3.Key != res.Key {
		t.Error("unexpected key")
	}
}

func TestHllWithoutReturn(t *testing.T) {
	res, err := Bucket("hll-test", "hlls").
		UpdateHyperLogLog().
		Add([]byte{1}).
		Run(con())

	if err != nil {
		t.Error(err)
	}

	if res != nil {
		t.Error("res was not nil")
	}
}

func TestHllAddMultiple(t *testing.T) {
	res, err := Bucket("hll-test", "hlls").
		UpdateHyperLogLog().
		AddMultiple([]byte{1}, []byte{2}, []byte{3}).
		AddMultiple([]byte{1}, []byte{2}, []byte{3}).
		ReturnBody(true).
		Run(con())

	if err != nil {
		t.Error(err)
	}

	if res == nil {
		t.Error("res was nil")
	}

	if res.Cardinality != 3 {
		t.Error("unexpected val")
	}
}
<|endoftext|>