<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package p256 implements a verifiable random function using curve p256.
package p256

// Discrete Log based VRF from Appendix A of CONIKS:
// http://www.jbonneau.com/doc/MBBFF15-coniks.pdf
// based on "Unique Ring Signatures, a Practical Construction"
// http://fc13.ifca.ai/proc/5-1.pdf

import (
	"bytes"
	"crypto/ecdsa"
	"crypto/elliptic"
	"crypto/hmac"
	"crypto/rand"
	"crypto/sha256"
	"crypto/sha512"
	"crypto/x509"
	"encoding/binary"
	"encoding/pem"
	"errors"
	"math/big"
)

var (
	curve  = elliptic.P256()
	params = curve.Params()

	// ErrPointNotOnCurve occurs when a public key is not on the curve.
	ErrPointNotOnCurve = errors.New("point is not on the P256 curve")
	// ErrWrongKeyType occurs when a key is not an ECDSA key.
	ErrWrongKeyType = errors.New("not an ECDSA key")
	// ErrNoPEMFound occurs when attempting to parse a non PEM data structure.
	ErrNoPEMFound = errors.New("no PEM block found")
	// ErrInvalidVRF occurs when the VRF does not validate.
	ErrInvalidVRF = errors.New("invalid VRF proof")
)

// PublicKey holds a public VRF key.
type PublicKey struct {
	*ecdsa.PublicKey
}

// PrivateKey holds a private VRF key.
type PrivateKey struct {
	*ecdsa.PrivateKey
}

// GenerateKey generates a fresh keypair for this VRF
func GenerateKey() (*PrivateKey, *PublicKey) {
	key, err := ecdsa.GenerateKey(curve, rand.Reader)
	if err != nil {
		return nil, nil
	}

	return &PrivateKey{key}, &PublicKey{&key.PublicKey}
}

// H1 hashes m to a curve point
func H1(m []byte) (x, y *big.Int) {
	h := sha512.New()
	var i uint32
	byteLen := (params.BitSize + 7) >> 3
	buf := make([]byte, 4)
	for x == nil && i < 100 {
		// TODO: Use a NIST specified DRBG.
		binary.BigEndian.PutUint32(buf[:], i)
		h.Reset()
		h.Write(buf)
		h.Write(m)
		r := []byte{2} // Set point encoding to "compressed".
		r = h.Sum(r)
		x, y = Unmarshal(curve, r[:byteLen+1])
		i++
	}
	return
}

var one = big.NewInt(1)

// H2 hashes to an integer [1,N-1]
func H2(m []byte) *big.Int {
	// NIST SP 800-90A § A.5.1: Simple discard method.
	byteLen := (params.BitSize + 7) >> 3
	h := sha512.New()
	buf := make([]byte, 4)
	for i := uint32(0); ; i++ {
		// TODO: Use a NIST specified DRBG.
		binary.BigEndian.PutUint32(buf[:], i)
		h.Reset()
		h.Write(buf)
		h.Write(m)
		b := h.Sum(nil)
		k := new(big.Int).SetBytes(b[:byteLen])
		if k.Cmp(new(big.Int).Sub(params.N, one)) == -1 {
			return k.Add(k, one)
		}
	}
}

// Evaluate returns the verifiable unpredictable function evaluated at m
func (k PrivateKey) Evaluate(m []byte) (vrf, proof []byte) {
	// Prover chooses r <-- [1,N-1]
	r, _, _, err := elliptic.GenerateKey(curve, rand.Reader)
	if err != nil {
		return nil, nil
	}
	ri := new(big.Int).SetBytes(r)

	// H = H1(m)
	hx, hy := H1(m)

	// G is the base point
	// s = H2(m, [r]G, [r]H)
	gRx, gRy := params.ScalarBaseMult(r)
	hRx, hRy := params.ScalarMult(hx, hy, r)
	var b bytes.Buffer
	b.Write(m)
	b.Write(elliptic.Marshal(curve, gRx, gRy))
	b.Write(elliptic.Marshal(curve, hRx, hRy))
	s := H2(b.Bytes())

	// t = r−s*k mod N
	t := new(big.Int).Sub(ri, new(big.Int).Mul(s, k.D))
	t.Mod(t, params.N)

	// Write s and t to a proof blob. Also write leading zeros before s and t
	// if needed.
	var buf bytes.Buffer
	buf.Write(make([]byte, 32-len(s.Bytes())))
	buf.Write(s.Bytes())
	buf.Write(make([]byte, 32-len(t.Bytes())))
	buf.Write(t.Bytes())

	// VRF_k(m) = [k]H
	vrfx, vrfy := params.ScalarMult(hx, hy, k.D.Bytes())
	return elliptic.Marshal(curve, vrfx, vrfy), buf.Bytes()
}

// Verify asserts that vrf is the hash of proof and the proof is correct
func (pk *PublicKey) Verify(m, vrf, proof []byte) error {
	// verifier checks that s == H2(m, [t]G + [s]([k]G), [t]H1(m) + [s]VRF_k(m))
	vrfx, vrfy := elliptic.Unmarshal(curve, vrf)
	if vrfx == nil {
		return ErrInvalidVRF
	}
	if len(proof) != 64 {
		return ErrInvalidVRF
	}

	// Parse proof into s and t.
	s := proof[0:32]
	t := proof[32:64]

	// [t]G + [s]([k]G) = [t+ks]G
	gTx, gTy := params.ScalarBaseMult(t)
	pkSx, pkSy := params.ScalarMult(pk.X, pk.Y, s)
	gTKSx, gTKSy := params.Add(gTx, gTy, pkSx, pkSy)

	// H = H1(m)
	// [t]H + [s]VRF = [t+ks]H
	hx, hy := H1(m)
	hTx, hTy := params.ScalarMult(hx, hy, t)
	vSx, vSy := params.ScalarMult(vrfx, vrfy, s)
	h1TKSx, h1TKSy := params.Add(hTx, hTy, vSx, vSy)

	// H2(m, [t]G + [s]([k]G), [t]H + [s]VRF)
	// = H2(m, [t+ks]G, [t+ks]H)
	// = H2(m, [r]G, [r]H)
	var b bytes.Buffer
	b.Write(m)
	b.Write(elliptic.Marshal(curve, gTKSx, gTKSy))
	b.Write(elliptic.Marshal(curve, h1TKSx, h1TKSy))
	h2 := H2(b.Bytes())

	// Left pad h2 with zeros if needed. This will ensure that h2 is padded
	// the same way s is.
	var buf bytes.Buffer
	buf.Write(make([]byte, 32-len(h2.Bytes())))
	buf.Write(h2.Bytes())

	if !hmac.Equal(s, buf.Bytes()) {
		return ErrInvalidVRF
	}
	return nil
}

// Index computes the index for a given vrf output.
func (k *PrivateKey) Index(vrf []byte) [32]byte {
	return sha256.Sum256(vrf)
}

// Index computes the index for a given vrf output.
func (pk *PublicKey) Index(vrf []byte) [32]byte {
	return sha256.Sum256(vrf)
}

// NewVRFSigner creates a signer object from a private key.
func NewVRFSigner(key *ecdsa.PrivateKey) (*PrivateKey, error) {
	if *(key.Params()) != *curve.Params() {
		return nil, ErrPointNotOnCurve
	}
	if !curve.IsOnCurve(key.X, key.Y) {
		return nil, ErrPointNotOnCurve
	}
	return &PrivateKey{key}, nil
}

// NewVRFVerifier creates a verifier object from a public key.
func NewVRFVerifier(pubkey *ecdsa.PublicKey) (*PublicKey, error) {
	if *(pubkey.Params()) != *curve.Params() {
		return nil, ErrPointNotOnCurve
	}
	if !curve.IsOnCurve(pubkey.X, pubkey.Y) {
		return nil, ErrPointNotOnCurve
	}
	return &PublicKey{pubkey}, nil
}

// NewVRFSignerFromPEM creates a vrf private key from a PEM data structure.
func NewVRFSignerFromPEM(b []byte) (*PrivateKey, error) {
	p, _ := pem.Decode(b)
	if p == nil {
		return nil, ErrNoPEMFound
	}
	k, err := x509.ParseECPrivateKey(p.Bytes)
	if err != nil {
		return nil, err
	}
	return NewVRFSigner(k)
}

// NewVRFVerifierFromPEM creates a vrf public key from a PEM data structure.
func NewVRFVerifierFromPEM(b []byte) (*PublicKey, error) {
	p, _ := pem.Decode(b)
	if p == nil {
		return nil, ErrNoPEMFound
	}
	k, err := x509.ParsePKIXPublicKey(p.Bytes)
	if err != nil {
		return nil, err
	}
	pk, ok := k.(*ecdsa.PublicKey)
	if !ok {
		return nil, ErrWrongKeyType
	}
	return NewVRFVerifier(pk)
}
<commit_msg>Allow reading keys from raw bytes in VRF<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package p256 implements a verifiable random function using curve p256.
package p256

// Discrete Log based VRF from Appendix A of CONIKS:
// http://www.jbonneau.com/doc/MBBFF15-coniks.pdf
// based on "Unique Ring Signatures, a Practical Construction"
// http://fc13.ifca.ai/proc/5-1.pdf

import (
	"bytes"
	"crypto/ecdsa"
	"crypto/elliptic"
	"crypto/hmac"
	"crypto/rand"
	"crypto/sha256"
	"crypto/sha512"
	"crypto/x509"
	"encoding/binary"
	"encoding/pem"
	"errors"
	"math/big"
)

var (
	curve  = elliptic.P256()
	params = curve.Params()

	// ErrPointNotOnCurve occurs when a public key is not on the curve.
	ErrPointNotOnCurve = errors.New("point is not on the P256 curve")
	// ErrWrongKeyType occurs when a key is not an ECDSA key.
	ErrWrongKeyType = errors.New("not an ECDSA key")
	// ErrNoPEMFound occurs when attempting to parse a non PEM data structure.
	ErrNoPEMFound = errors.New("no PEM block found")
	// ErrInvalidVRF occurs when the VRF does not validate.
	ErrInvalidVRF = errors.New("invalid VRF proof")
)

// PublicKey holds a public VRF key.
type PublicKey struct {
	*ecdsa.PublicKey
}

// PrivateKey holds a private VRF key.
type PrivateKey struct {
	*ecdsa.PrivateKey
}

// GenerateKey generates a fresh keypair for this VRF
func GenerateKey() (*PrivateKey, *PublicKey) {
	key, err := ecdsa.GenerateKey(curve, rand.Reader)
	if err != nil {
		return nil, nil
	}

	return &PrivateKey{key}, &PublicKey{&key.PublicKey}
}

// H1 hashes m to a curve point
func H1(m []byte) (x, y *big.Int) {
	h := sha512.New()
	var i uint32
	byteLen := (params.BitSize + 7) >> 3
	buf := make([]byte, 4)
	for x == nil && i < 100 {
		// TODO: Use a NIST specified DRBG.
		binary.BigEndian.PutUint32(buf[:], i)
		h.Reset()
		h.Write(buf)
		h.Write(m)
		r := []byte{2} // Set point encoding to "compressed".
		r = h.Sum(r)
		x, y = Unmarshal(curve, r[:byteLen+1])
		i++
	}
	return
}

var one = big.NewInt(1)

// H2 hashes to an integer [1,N-1]
func H2(m []byte) *big.Int {
	// NIST SP 800-90A § A.5.1: Simple discard method.
	byteLen := (params.BitSize + 7) >> 3
	h := sha512.New()
	buf := make([]byte, 4)
	for i := uint32(0); ; i++ {
		// TODO: Use a NIST specified DRBG.
		binary.BigEndian.PutUint32(buf[:], i)
		h.Reset()
		h.Write(buf)
		h.Write(m)
		b := h.Sum(nil)
		k := new(big.Int).SetBytes(b[:byteLen])
		if k.Cmp(new(big.Int).Sub(params.N, one)) == -1 {
			return k.Add(k, one)
		}
	}
}

// Evaluate returns the verifiable unpredictable function evaluated at m
func (k PrivateKey) Evaluate(m []byte) (vrf, proof []byte) {
	// Prover chooses r <-- [1,N-1]
	r, _, _, err := elliptic.GenerateKey(curve, rand.Reader)
	if err != nil {
		return nil, nil
	}
	ri := new(big.Int).SetBytes(r)

	// H = H1(m)
	hx, hy := H1(m)

	// G is the base point
	// s = H2(m, [r]G, [r]H)
	gRx, gRy := params.ScalarBaseMult(r)
	hRx, hRy := params.ScalarMult(hx, hy, r)
	var b bytes.Buffer
	b.Write(m)
	b.Write(elliptic.Marshal(curve, gRx, gRy))
	b.Write(elliptic.Marshal(curve, hRx, hRy))
	s := H2(b.Bytes())

	// t = r−s*k mod N
	t := new(big.Int).Sub(ri, new(big.Int).Mul(s, k.D))
	t.Mod(t, params.N)

	// Write s and t to a proof blob. Also write leading zeros before s and t
	// if needed.
	var buf bytes.Buffer
	buf.Write(make([]byte, 32-len(s.Bytes())))
	buf.Write(s.Bytes())
	buf.Write(make([]byte, 32-len(t.Bytes())))
	buf.Write(t.Bytes())

	// VRF_k(m) = [k]H
	vrfx, vrfy := params.ScalarMult(hx, hy, k.D.Bytes())
	return elliptic.Marshal(curve, vrfx, vrfy), buf.Bytes()
}

// Verify asserts that vrf is the hash of proof and the proof is correct
func (pk *PublicKey) Verify(m, vrf, proof []byte) error {
	// verifier checks that s == H2(m, [t]G + [s]([k]G), [t]H1(m) + [s]VRF_k(m))
	vrfx, vrfy := elliptic.Unmarshal(curve, vrf)
	if vrfx == nil {
		return ErrInvalidVRF
	}
	if len(proof) != 64 {
		return ErrInvalidVRF
	}

	// Parse proof into s and t.
	s := proof[0:32]
	t := proof[32:64]

	// [t]G + [s]([k]G) = [t+ks]G
	gTx, gTy := params.ScalarBaseMult(t)
	pkSx, pkSy := params.ScalarMult(pk.X, pk.Y, s)
	gTKSx, gTKSy := params.Add(gTx, gTy, pkSx, pkSy)

	// H = H1(m)
	// [t]H + [s]VRF = [t+ks]H
	hx, hy := H1(m)
	hTx, hTy := params.ScalarMult(hx, hy, t)
	vSx, vSy := params.ScalarMult(vrfx, vrfy, s)
	h1TKSx, h1TKSy := params.Add(hTx, hTy, vSx, vSy)

	// H2(m, [t]G + [s]([k]G), [t]H + [s]VRF)
	// = H2(m, [t+ks]G, [t+ks]H)
	// = H2(m, [r]G, [r]H)
	var b bytes.Buffer
	b.Write(m)
	b.Write(elliptic.Marshal(curve, gTKSx, gTKSy))
	b.Write(elliptic.Marshal(curve, h1TKSx, h1TKSy))
	h2 := H2(b.Bytes())

	// Left pad h2 with zeros if needed. This will ensure that h2 is padded
	// the same way s is.
	var buf bytes.Buffer
	buf.Write(make([]byte, 32-len(h2.Bytes())))
	buf.Write(h2.Bytes())

	if !hmac.Equal(s, buf.Bytes()) {
		return ErrInvalidVRF
	}
	return nil
}

// Index computes the index for a given vrf output.
func (k *PrivateKey) Index(vrf []byte) [32]byte {
	return sha256.Sum256(vrf)
}

// Index computes the index for a given vrf output.
func (pk *PublicKey) Index(vrf []byte) [32]byte {
	return sha256.Sum256(vrf)
}

// NewVRFSigner creates a signer object from a private key.
func NewVRFSigner(key *ecdsa.PrivateKey) (*PrivateKey, error) {
	if *(key.Params()) != *curve.Params() {
		return nil, ErrPointNotOnCurve
	}
	if !curve.IsOnCurve(key.X, key.Y) {
		return nil, ErrPointNotOnCurve
	}
	return &PrivateKey{key}, nil
}

// NewVRFVerifier creates a verifier object from a public key.
func NewVRFVerifier(pubkey *ecdsa.PublicKey) (*PublicKey, error) {
	if *(pubkey.Params()) != *curve.Params() {
		return nil, ErrPointNotOnCurve
	}
	if !curve.IsOnCurve(pubkey.X, pubkey.Y) {
		return nil, ErrPointNotOnCurve
	}
	return &PublicKey{pubkey}, nil
}

// NewVRFSignerFromPEM creates a vrf private key from a PEM data structure.
func NewVRFSignerFromPEM(b []byte) (*PrivateKey, error) {
	p, _ := pem.Decode(b)
	if p == nil {
		return nil, ErrNoPEMFound
	}
	return NewVRFSignerFromRawKey(p.Bytes)
}

// NewVRFSignerFromRawKey returns the private key from a raw private key bytes.
func NewVRFSignerFromRawKey(b []byte) (*PrivateKey, error) {
	k, err := x509.ParseECPrivateKey(b)
	if err != nil {
		return nil, err
	}
	return NewVRFSigner(k)
}

// NewVRFVerifierFromPEM creates a vrf public key from a PEM data structure.
func NewVRFVerifierFromPEM(b []byte) (*PublicKey, error) {
	p, _ := pem.Decode(b)
	if p == nil {
		return nil, ErrNoPEMFound
	}
	return NewVRFVerifierFromRawKey(p.Bytes)
}

// NewVRFVerifierFromRawKey returns the public key from a raw public key bytes.
func NewVRFVerifierFromRawKey(b []byte) (*PublicKey, error) {
	k, err := x509.ParsePKIXPublicKey(b)
	if err != nil {
		return nil, err
	}
	pk, ok := k.(*ecdsa.PublicKey)
	if !ok {
		return nil, ErrWrongKeyType
	}
	return NewVRFVerifier(pk)
}
<|endoftext|>


<commit_before>// Tideland Go CouchDB Client - CouchDB - Parameters
//
// Copyright (C) 2016 Frank Mueller / Tideland / Oldenburg / Germany
//
// All rights reserved. Use of this source code is governed
// by the new BSD license.

package couchdb

//--------------------
// IMPORTS
//--------------------

import (
	"encoding/base64"
)

//--------------------
// PARAMETERIZABLE
//--------------------

// KeyValue is used for generic query and header parameters.
type KeyValue struct {
	Key   string
	Value string
}

// Parameterizable defines the methods needed to apply the parameters.
type Parameterizable interface {
	// SetQuery sets a query parameter.
	SetQuery(key, value string)

	// AddQuery adds a query parameter to an existing one.
	AddQuery(key, value string)

	// SetHeader sets a header parameter.
	SetHeader(key, value string)

	// AddKeys adds view key parameters.
	AddKeys(keys ...interface{})
}

//--------------------
// PARAMETER
//--------------------

// Parameter is a function changing one (or if needed multile) parameter.
type Parameter func(pa Parameterizable)

// Query is generic for setting request query parameters.
func Query(kvs ...KeyValue) Parameter {
	return func(pa Parameterizable) {
		for _, kv := range kvs {
			pa.AddQuery(kv.Key, kv.Value)
		}
	}
}

// Header is generic for setting request header parameters.
func Header(kvs ...KeyValue) Parameter {
	return func(pa Parameterizable) {
		for _, kv := range kvs {
			pa.SetHeader(kv.Key, kv.Value)
		}
	}
}

// BasicAuthentication is intended for basic authentication
// against the database.
func BasicAuthentication(userID, password string) Parameter {
	return func(pa Parameterizable) {
		up := []byte(userID + ":" + password)
		auth := "Basic " + base64.StdEncoding.EncodeToString(up)

		pa.SetHeader("Authorization", auth)
	}
}

// Revision sets the revision for the access to concrete document revisions.
func Revision(revision string) Parameter {
	return func(pa Parameterizable) {
		pa.SetQuery("rev", revision)
	}
}

// Keys sets a number of keys wanted from a view request.
func Keys(keys ...interface{}) Parameter {
	return func(pa Parameterizable) {
		pa.AddKeys(keys...)
	}
}

// StartEndKey sets the startkey and endkey for view requests.
func StartEndKey(start, end string) Parameter {
	return func(pa Parameterizable) {
		pa.SetQuery("startkey", "\""+start+"\"")
		pa.SetQuery("endkey", "\""+end+"\"")
	}
}

// OneKey sets the startkey and endkey for view requests for
// only one key
func OneKey(key string) Parameter {
	return StartEndKey(key, key)
}

// IncludeDocuments sets the flag for the including of found view documents.
func SetIncludeDocuments() Parameter {
	return func(pa Parameterizable) {
		pa.SetQuery("include_docs", "true")
	}
}

// EOF
<commit_msg>Added parameter for skip and limit<commit_after>// Tideland Go CouchDB Client - CouchDB - Parameters
//
// Copyright (C) 2016 Frank Mueller / Tideland / Oldenburg / Germany
//
// All rights reserved. Use of this source code is governed
// by the new BSD license.

package couchdb

//--------------------
// IMPORTS
//--------------------

import (
	"encoding/base64"
	"strconv"
)

//--------------------
// PARAMETERIZABLE
//--------------------

// KeyValue is used for generic query and header parameters.
type KeyValue struct {
	Key   string
	Value string
}

// Parameterizable defines the methods needed to apply the parameters.
type Parameterizable interface {
	// SetQuery sets a query parameter.
	SetQuery(key, value string)

	// AddQuery adds a query parameter to an existing one.
	AddQuery(key, value string)

	// SetHeader sets a header parameter.
	SetHeader(key, value string)

	// AddKeys adds view key parameters.
	AddKeys(keys ...interface{})
}

//--------------------
// PARAMETER
//--------------------

// Parameter is a function changing one (or if needed multile) parameter.
type Parameter func(pa Parameterizable)

// Query is generic for setting request query parameters.
func Query(kvs ...KeyValue) Parameter {
	return func(pa Parameterizable) {
		for _, kv := range kvs {
			pa.AddQuery(kv.Key, kv.Value)
		}
	}
}

// Header is generic for setting request header parameters.
func Header(kvs ...KeyValue) Parameter {
	return func(pa Parameterizable) {
		for _, kv := range kvs {
			pa.SetHeader(kv.Key, kv.Value)
		}
	}
}

// BasicAuthentication is intended for basic authentication
// against the database.
func BasicAuthentication(userID, password string) Parameter {
	return func(pa Parameterizable) {
		up := []byte(userID + ":" + password)
		auth := "Basic " + base64.StdEncoding.EncodeToString(up)

		pa.SetHeader("Authorization", auth)
	}
}

// Revision sets the revision for the access to concrete document revisions.
func Revision(revision string) Parameter {
	return func(pa Parameterizable) {
		pa.SetQuery("rev", revision)
	}
}

// Keys sets a number of keys wanted from a view request.
func Keys(keys ...interface{}) Parameter {
	return func(pa Parameterizable) {
		pa.AddKeys(keys...)
	}
}

// StartEndKey sets the startkey and endkey for view requests.
func StartEndKey(start, end string) Parameter {
	return func(pa Parameterizable) {
		pa.SetQuery("startkey", "\""+start+"\"")
		pa.SetQuery("endkey", "\""+end+"\"")
	}
}

// OneKey sets the startkey and endkey for view requests for
// only one key
func OneKey(key string) Parameter {
	return StartEndKey(key, key)
}

// SkipLimit sets the number to skip and the limit for
// view requests.
func SkipLimit(skip, limit int) Parameter {
	return func(pa Parameterizable) {
		if skip > 0 {
			pa.SetQuery("skip", strconv.Itoa(skip))
		}
		if limit > 0 {
			pa.SetQuery("limit", strconv.Itoa(limit))
		}
	}
}

// IncludeDocuments sets the flag for the including of found view documents.
func SetIncludeDocuments() Parameter {
	return func(pa Parameterizable) {
		pa.SetQuery("include_docs", "true")
	}
}

// EOF
<|endoftext|>


<commit_before>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package driver

import (
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os/exec"
	"regexp"
	"sync"
	"testing"

	"github.com/google/pprof/internal/plugin"
	"github.com/google/pprof/profile"
	"runtime"
)

func TestWebInterface(t *testing.T) {
	if runtime.GOOS == "nacl" {
		t.Skip("test assumes tcp available")
	}

	prof := makeFakeProfile()

	// Custom http server creator
	var server *httptest.Server
	serverCreated := make(chan bool)
	creator := func(a *plugin.HTTPServerArgs) error {
		server = httptest.NewServer(http.HandlerFunc(
			func(w http.ResponseWriter, r *http.Request) {
				if h := a.Handlers[r.URL.Path]; h != nil {
					h.ServeHTTP(w, r)
				}
			}))
		serverCreated <- true
		return nil
	}

	// Start server and wait for it to be initialized
	go serveWebInterface("unused:1234", prof, &plugin.Options{
		Obj:        fakeObjTool{},
		UI:         &stdUI{},
		HTTPServer: creator,
	})
	<-serverCreated
	defer server.Close()

	haveDot := false
	if _, err := exec.LookPath("dot"); err == nil {
		haveDot = true
	}

	type testCase struct {
		path    string
		want    []string
		needDot bool
	}
	testcases := []testCase{
		{"/", []string{"F1", "F2", "F3", "testbin", "cpu"}, true},
		{"/top", []string{`"Name":"F2","InlineLabel":"","Flat":200,"Cum":300,"FlatFormat":"200ms","CumFormat":"300ms"}`}, false},
		{"/source?f=" + url.QueryEscape("F[12]"),
			[]string{"F1", "F2", "300ms +line1"}, false},
		{"/peek?f=" + url.QueryEscape("F[12]"),
			[]string{"300ms.*F1", "200ms.*300ms.*F2"}, false},
		{"/disasm?f=" + url.QueryEscape("F[12]"),
			[]string{"f1:asm", "f2:asm"}, false},
	}
	for _, c := range testcases {
		if c.needDot && !haveDot {
			t.Log("skipping", c.path, "since dot (graphviz) does not seem to be installed")
			continue
		}

		res, err := http.Get(server.URL + c.path)
		if err != nil {
			t.Error("could not fetch", c.path, err)
			continue
		}
		data, err := ioutil.ReadAll(res.Body)
		if err != nil {
			t.Error("could not read response", c.path, err)
			continue
		}
		result := string(data)
		for _, w := range c.want {
			if match, _ := regexp.MatchString(w, result); !match {
				t.Errorf("response for %s does not match "+
					"expected pattern '%s'; "+
					"actual result:\n%s", c.path, w, result)
			}
		}
	}

	// Also fetch all the test case URLs in parallel to test thread
	// safety when run under the race detector.
	var wg sync.WaitGroup
	for _, c := range testcases {
		if c.needDot && !haveDot {
			continue
		}
		path := server.URL + c.path
		for count := 0; count < 2; count++ {
			wg.Add(1)
			go func() {
				http.Get(path)
				wg.Done()
			}()
		}
	}
	wg.Wait()
}

// Implement fake object file support.

const addrBase = 0x1000
const fakeSource = "testdata/file1000.src"

type fakeObj struct{}

func (f fakeObj) Close() error    { return nil }
func (f fakeObj) Name() string    { return "testbin" }
func (f fakeObj) Base() uint64    { return 0 }
func (f fakeObj) BuildID() string { return "" }
func (f fakeObj) SourceLine(addr uint64) ([]plugin.Frame, error) {
	return nil, fmt.Errorf("SourceLine unimplemented")
}
func (f fakeObj) Symbols(r *regexp.Regexp, addr uint64) ([]*plugin.Sym, error) {
	return []*plugin.Sym{
		{[]string{"F1"}, fakeSource, addrBase, addrBase + 10},
		{[]string{"F2"}, fakeSource, addrBase + 10, addrBase + 20},
		{[]string{"F3"}, fakeSource, addrBase + 20, addrBase + 30},
	}, nil
}

type fakeObjTool struct{}

func (obj fakeObjTool) Open(file string, start, limit, offset uint64) (plugin.ObjFile, error) {
	return fakeObj{}, nil
}

func (obj fakeObjTool) Disasm(file string, start, end uint64) ([]plugin.Inst, error) {
	return []plugin.Inst{
		{Addr: addrBase + 0, Text: "f1:asm", Function: "F1"},
		{Addr: addrBase + 10, Text: "f2:asm", Function: "F2"},
		{Addr: addrBase + 20, Text: "d3:asm", Function: "F3"},
	}, nil
}

func makeFakeProfile() *profile.Profile {
	// Three functions: F1, F2, F3 with three lines, 11, 22, 33.
	funcs := []*profile.Function{
		{ID: 1, Name: "F1", Filename: fakeSource, StartLine: 3},
		{ID: 2, Name: "F2", Filename: fakeSource, StartLine: 5},
		{ID: 3, Name: "F3", Filename: fakeSource, StartLine: 7},
	}
	lines := []profile.Line{
		{Function: funcs[0], Line: 11},
		{Function: funcs[1], Line: 22},
		{Function: funcs[2], Line: 33},
	}
	mapping := []*profile.Mapping{
		{
			ID:             1,
			Start:          addrBase,
			Limit:          addrBase + 10,
			Offset:         0,
			File:           "testbin",
			HasFunctions:   true,
			HasFilenames:   true,
			HasLineNumbers: true,
		},
	}

	// Three interesting addresses: base+{10,20,30}
	locs := []*profile.Location{
		{ID: 1, Address: addrBase + 10, Line: lines[0:1], Mapping: mapping[0]},
		{ID: 2, Address: addrBase + 20, Line: lines[1:2], Mapping: mapping[0]},
		{ID: 3, Address: addrBase + 30, Line: lines[2:3], Mapping: mapping[0]},
	}

	// Two stack traces.
	return &profile.Profile{
		PeriodType:    &profile.ValueType{Type: "cpu", Unit: "milliseconds"},
		Period:        1,
		DurationNanos: 10e9,
		SampleType: []*profile.ValueType{
			{Type: "cpu", Unit: "milliseconds"},
		},
		Sample: []*profile.Sample{
			{
				Location: []*profile.Location{locs[2], locs[1], locs[0]},
				Value:    []int64{100},
			},
			{
				Location: []*profile.Location{locs[1], locs[0]},
				Value:    []int64{200},
			},
		},
		Location: locs,
		Function: funcs,
		Mapping:  mapping,
	}
}

func TestIsLocalHost(t *testing.T) {
	for _, s := range []string{"localhost:10000", "[::1]:10000", "127.0.0.1:10000"} {
		host, _, err := net.SplitHostPort(s)
		if err != nil {
			t.Error("unexpected error when splitting", s)
			continue
		}
		if !isLocalhost(host) {
			t.Errorf("host %s from %s not considered local", host, s)
		}
	}
}
<commit_msg>cmd/vendor/.../pprof: do not run test that opens web browser<commit_after>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package driver

import (
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os/exec"
	"regexp"
	"sync"
	"testing"
	"time"

	"runtime"

	"github.com/google/pprof/internal/plugin"
	"github.com/google/pprof/profile"
)

func TestWebInterface(t *testing.T) {
	// This test starts a web browser in a background goroutine
	// after a 500ms delay. Sometimes the test exits before it
	// can run the browser, but sometimes the browser does open.
	// That's obviously unacceptable.
	defer time.Sleep(2 * time.Second) // to see the browser open
	t.Skip("golang.org/issue/22651")

	if runtime.GOOS == "nacl" {
		t.Skip("test assumes tcp available")
	}

	prof := makeFakeProfile()

	// Custom http server creator
	var server *httptest.Server
	serverCreated := make(chan bool)
	creator := func(a *plugin.HTTPServerArgs) error {
		server = httptest.NewServer(http.HandlerFunc(
			func(w http.ResponseWriter, r *http.Request) {
				if h := a.Handlers[r.URL.Path]; h != nil {
					h.ServeHTTP(w, r)
				}
			}))
		serverCreated <- true
		return nil
	}

	// Start server and wait for it to be initialized
	go serveWebInterface("unused:1234", prof, &plugin.Options{
		Obj:        fakeObjTool{},
		UI:         &stdUI{},
		HTTPServer: creator,
	})
	<-serverCreated
	defer server.Close()

	haveDot := false
	if _, err := exec.LookPath("dot"); err == nil {
		haveDot = true
	}

	type testCase struct {
		path    string
		want    []string
		needDot bool
	}
	testcases := []testCase{
		{"/", []string{"F1", "F2", "F3", "testbin", "cpu"}, true},
		{"/top", []string{`"Name":"F2","InlineLabel":"","Flat":200,"Cum":300,"FlatFormat":"200ms","CumFormat":"300ms"}`}, false},
		{"/source?f=" + url.QueryEscape("F[12]"),
			[]string{"F1", "F2", "300ms +line1"}, false},
		{"/peek?f=" + url.QueryEscape("F[12]"),
			[]string{"300ms.*F1", "200ms.*300ms.*F2"}, false},
		{"/disasm?f=" + url.QueryEscape("F[12]"),
			[]string{"f1:asm", "f2:asm"}, false},
	}
	for _, c := range testcases {
		if c.needDot && !haveDot {
			t.Log("skipping", c.path, "since dot (graphviz) does not seem to be installed")
			continue
		}

		res, err := http.Get(server.URL + c.path)
		if err != nil {
			t.Error("could not fetch", c.path, err)
			continue
		}
		data, err := ioutil.ReadAll(res.Body)
		if err != nil {
			t.Error("could not read response", c.path, err)
			continue
		}
		result := string(data)
		for _, w := range c.want {
			if match, _ := regexp.MatchString(w, result); !match {
				t.Errorf("response for %s does not match "+
					"expected pattern '%s'; "+
					"actual result:\n%s", c.path, w, result)
			}
		}
	}

	// Also fetch all the test case URLs in parallel to test thread
	// safety when run under the race detector.
	var wg sync.WaitGroup
	for _, c := range testcases {
		if c.needDot && !haveDot {
			continue
		}
		path := server.URL + c.path
		for count := 0; count < 2; count++ {
			wg.Add(1)
			go func() {
				http.Get(path)
				wg.Done()
			}()
		}
	}
	wg.Wait()

	time.Sleep(5 * time.Second)
}

// Implement fake object file support.

const addrBase = 0x1000
const fakeSource = "testdata/file1000.src"

type fakeObj struct{}

func (f fakeObj) Close() error    { return nil }
func (f fakeObj) Name() string    { return "testbin" }
func (f fakeObj) Base() uint64    { return 0 }
func (f fakeObj) BuildID() string { return "" }
func (f fakeObj) SourceLine(addr uint64) ([]plugin.Frame, error) {
	return nil, fmt.Errorf("SourceLine unimplemented")
}
func (f fakeObj) Symbols(r *regexp.Regexp, addr uint64) ([]*plugin.Sym, error) {
	return []*plugin.Sym{
		{[]string{"F1"}, fakeSource, addrBase, addrBase + 10},
		{[]string{"F2"}, fakeSource, addrBase + 10, addrBase + 20},
		{[]string{"F3"}, fakeSource, addrBase + 20, addrBase + 30},
	}, nil
}

type fakeObjTool struct{}

func (obj fakeObjTool) Open(file string, start, limit, offset uint64) (plugin.ObjFile, error) {
	return fakeObj{}, nil
}

func (obj fakeObjTool) Disasm(file string, start, end uint64) ([]plugin.Inst, error) {
	return []plugin.Inst{
		{Addr: addrBase + 0, Text: "f1:asm", Function: "F1"},
		{Addr: addrBase + 10, Text: "f2:asm", Function: "F2"},
		{Addr: addrBase + 20, Text: "d3:asm", Function: "F3"},
	}, nil
}

func makeFakeProfile() *profile.Profile {
	// Three functions: F1, F2, F3 with three lines, 11, 22, 33.
	funcs := []*profile.Function{
		{ID: 1, Name: "F1", Filename: fakeSource, StartLine: 3},
		{ID: 2, Name: "F2", Filename: fakeSource, StartLine: 5},
		{ID: 3, Name: "F3", Filename: fakeSource, StartLine: 7},
	}
	lines := []profile.Line{
		{Function: funcs[0], Line: 11},
		{Function: funcs[1], Line: 22},
		{Function: funcs[2], Line: 33},
	}
	mapping := []*profile.Mapping{
		{
			ID:             1,
			Start:          addrBase,
			Limit:          addrBase + 10,
			Offset:         0,
			File:           "testbin",
			HasFunctions:   true,
			HasFilenames:   true,
			HasLineNumbers: true,
		},
	}

	// Three interesting addresses: base+{10,20,30}
	locs := []*profile.Location{
		{ID: 1, Address: addrBase + 10, Line: lines[0:1], Mapping: mapping[0]},
		{ID: 2, Address: addrBase + 20, Line: lines[1:2], Mapping: mapping[0]},
		{ID: 3, Address: addrBase + 30, Line: lines[2:3], Mapping: mapping[0]},
	}

	// Two stack traces.
	return &profile.Profile{
		PeriodType:    &profile.ValueType{Type: "cpu", Unit: "milliseconds"},
		Period:        1,
		DurationNanos: 10e9,
		SampleType: []*profile.ValueType{
			{Type: "cpu", Unit: "milliseconds"},
		},
		Sample: []*profile.Sample{
			{
				Location: []*profile.Location{locs[2], locs[1], locs[0]},
				Value:    []int64{100},
			},
			{
				Location: []*profile.Location{locs[1], locs[0]},
				Value:    []int64{200},
			},
		},
		Location: locs,
		Function: funcs,
		Mapping:  mapping,
	}
}

func TestIsLocalHost(t *testing.T) {
	for _, s := range []string{"localhost:10000", "[::1]:10000", "127.0.0.1:10000"} {
		host, _, err := net.SplitHostPort(s)
		if err != nil {
			t.Error("unexpected error when splitting", s)
			continue
		}
		if !isLocalhost(host) {
			t.Errorf("host %s from %s not considered local", host, s)
		}
	}
}
<|endoftext|>


<commit_before>package rivined

import (
	"errors"
	"fmt"
	"os"
	"os/signal"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/rivine/rivine/api"
	"github.com/rivine/rivine/build"
	"github.com/rivine/rivine/modules"
	"github.com/rivine/rivine/modules/blockcreator"
	"github.com/rivine/rivine/modules/consensus"
	"github.com/rivine/rivine/modules/explorer"
	"github.com/rivine/rivine/modules/gateway"
	"github.com/rivine/rivine/modules/transactionpool"
	"github.com/rivine/rivine/modules/wallet"

	"github.com/bgentry/speakeasy"
)

var (
	// globalConfig is used by the cobra package to fill out the configuration
	// variables.
	globalConfig Config
)

// The Config struct contains all configurable variables for siad. It is
// compatible with gcfg.
type Config struct {
	// The APIPassword is input by the user after the daemon starts up, if the
	// --authenticate-api flag is set.
	APIPassword string

	// The Rivined variables are referenced directly by cobra, and are set
	// according to the flags.
	Rivined RivinedCfg
}

// RivinedCfg holds variables referenced by cobra and set by flags
type RivinedCfg struct {
	APIaddr      string
	RPCaddr      string
	HostAddr     string
	AllowAPIBind bool

	Modules           string
	NoBootstrap       bool
	RequiredUserAgent string
	AuthenticateAPI   bool

	Profile    bool
	ProfileDir string
	RivineDir  string
}

// DefaultConfig returns the default daemon configuration
func DefaultConfig() RivinedCfg {
	return RivinedCfg{
		APIaddr:      "localhost:23110",
		RPCaddr:      "localhost:23112",
		HostAddr:     "",
		AllowAPIBind: false,

		Modules:           "cgtwb",
		NoBootstrap:       false,
		RequiredUserAgent: "Rivine-Agent",
		AuthenticateAPI:   false,

		Profile:    false,
		ProfileDir: "profiles",
		RivineDir:  "",
	}
}

// verifyAPISecurity checks that the security values are consistent with a
// sane, secure system.
func verifyAPISecurity(config Config) error {
	// Make sure that only the loopback address is allowed unless the
	// --disable-api-security flag has been used.
	if !config.Rivined.AllowAPIBind {
		addr := modules.NetAddress(config.Rivined.APIaddr)
		if !addr.IsLoopback() {
			if addr.Host() == "" {
				return fmt.Errorf("a blank host will listen on all interfaces, did you mean localhost:%v?\nyou must pass --disable-api-security to bind %s to a non-localhost address", addr.Port(), DaemonName)
			}
			return fmt.Errorf("you must pass --disable-api-security to bind %s to a non-localhost address", DaemonName)
		}
		return nil
	}

	// If the --disable-api-security flag is used, enforce that
	// --authenticate-api must also be used.
	if config.Rivined.AllowAPIBind && !config.Rivined.AuthenticateAPI {
		return errors.New("cannot use --disable-api-security without setting an api password")
	}
	return nil
}

// processNetAddr adds a ':' to a bare integer, so that it is a proper port
// number.
func processNetAddr(addr string) string {
	_, err := strconv.Atoi(addr)
	if err == nil {
		return ":" + addr
	}
	return addr
}

// processModules makes the modules string lowercase to make checking if a
// module in the string easier, and returns an error if the string contains an
// invalid module character.
func processModules(modules string) (string, error) {
	modules = strings.ToLower(modules)
	validModules := "cgtweb"
	invalidModules := modules
	for _, m := range validModules {
		invalidModules = strings.Replace(invalidModules, string(m), "", 1)
	}
	if len(invalidModules) > 0 {
		return "", errors.New("Unable to parse --modules flag, unrecognized or duplicate modules: " + invalidModules)
	}
	return modules, nil
}

// processConfig checks the configuration values and performs cleanup on
// incorrect-but-allowed values.
func processConfig(config Config) (Config, error) {
	var err1 error
	config.Rivined.APIaddr = processNetAddr(config.Rivined.APIaddr)
	config.Rivined.RPCaddr = processNetAddr(config.Rivined.RPCaddr)
	config.Rivined.HostAddr = processNetAddr(config.Rivined.HostAddr)
	config.Rivined.Modules, err1 = processModules(config.Rivined.Modules)
	err2 := verifyAPISecurity(config)
	err := build.JoinErrors([]error{err1, err2}, ", and ")
	if err != nil {
		return Config{}, err
	}
	return config, nil
}

// startDaemon uses the config parameters to initialize Sia modules and start
// siad.
func startDaemon(config Config) (err error) {
	// Prompt user for API password.
	if config.Rivined.AuthenticateAPI {
		config.APIPassword, err = speakeasy.Ask("Enter API password: ")
		if err != nil {
			return err
		}
		if config.APIPassword == "" {
			return errors.New("password cannot be blank")
		}
	}

	// Process the config variables after they are parsed by cobra.
	config, err = processConfig(config)
	if err != nil {
		return err
	}

	// Print a startup message.
	fmt.Println("Loading...")
	loadStart := time.Now()

	// Create the server and start serving daemon routes immediately.
	fmt.Printf("(0/%d) Loading "+DaemonName+"...\n", len(config.Rivined.Modules))
	srv, err := NewServer(config.Rivined.APIaddr, config.Rivined.RequiredUserAgent, config.APIPassword)
	if err != nil {
		return err
	}

	servErrs := make(chan error)
	go func() {
		servErrs <- srv.Serve()
	}()

	// Initialize the Rivine modules
	i := 0
	var g modules.Gateway
	if strings.Contains(config.Rivined.Modules, "g") {
		i++
		fmt.Printf("(%d/%d) Loading gateway...\n", i, len(config.Rivined.Modules))
		g, err = gateway.New(config.Rivined.RPCaddr, !config.Rivined.NoBootstrap, filepath.Join(config.Rivined.RivineDir, modules.GatewayDir))
		if err != nil {
			return err
		}
		defer func() {
			fmt.Println("Closing gateway...")
			err := g.Close()
			if err != nil {
				fmt.Println("Error during gateway shutdown:", err)
			}
		}()

	}
	var cs modules.ConsensusSet
	if strings.Contains(config.Rivined.Modules, "c") {
		i++
		fmt.Printf("(%d/%d) Loading consensus...\n", i, len(config.Rivined.Modules))
		cs, err = consensus.New(g, !config.Rivined.NoBootstrap, filepath.Join(config.Rivined.RivineDir, modules.ConsensusDir))
		if err != nil {
			return err
		}
		defer func() {
			fmt.Println("Closing consensus set...")
			err := cs.Close()
			if err != nil {
				fmt.Println("Error during consensus set shutdown:", err)
			}
		}()

	}
	var e modules.Explorer
	if strings.Contains(config.Rivined.Modules, "e") {
		i++
		fmt.Printf("(%d/%d) Loading explorer...\n", i, len(config.Rivined.Modules))
		e, err = explorer.New(cs, filepath.Join(config.Rivined.RivineDir, modules.ExplorerDir))
		if err != nil {
			return err
		}
		defer func() {
			fmt.Println("Closing explorer...")
			err := e.Close()
			if err != nil {
				fmt.Println("Error during explorer shutdown:", err)
			}
		}()

	}
	var tpool modules.TransactionPool
	if strings.Contains(config.Rivined.Modules, "t") {
		i++
		fmt.Printf("(%d/%d) Loading transaction pool...\n", i, len(config.Rivined.Modules))
		tpool, err = transactionpool.New(cs, g, filepath.Join(config.Rivined.RivineDir, modules.TransactionPoolDir))
		if err != nil {
			return err
		}
		defer func() {
			fmt.Println("Closing transaction pool...")
			err := tpool.Close()
			if err != nil {
				fmt.Println("Error during transaction pool shutdown:", err)
			}
		}()

	}
	var w modules.Wallet
	if strings.Contains(config.Rivined.Modules, "w") {
		i++
		fmt.Printf("(%d/%d) Loading wallet...\n", i, len(config.Rivined.Modules))
		w, err = wallet.New(cs, tpool, filepath.Join(config.Rivined.RivineDir, modules.WalletDir))
		if err != nil {
			return err
		}
		defer func() {
			fmt.Println("Closing wallet...")
			err := w.Close()
			if err != nil {
				fmt.Println("Error during wallet shutdown:", err)
			}
		}()

	}
	var b modules.BlockCreator
	if strings.Contains(config.Rivined.Modules, "b") {
		i++
		fmt.Printf("(%d/%d) Loading block creator...\n", i, len(config.Rivined.Modules))
		b, err = blockcreator.New(cs, tpool, w, filepath.Join(config.Rivined.RivineDir, modules.BlockCreatorDir))
		if err != nil {
			return err
		}
		defer func() {
			fmt.Println("Closing block creator...")
			err := b.Close()
			if err != nil {
				fmt.Println("Error during block creator shutdown:", err)
			}
		}()
	}

	// Create the Sia API
	a := api.New(
		config.Rivined.RequiredUserAgent,
		config.APIPassword,
		cs,
		e,
		g,
		tpool,
		w,
	)

	// connect the API to the server
	srv.mux.Handle("/", a)

	// stop the server if a kill signal is caught
	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, os.Interrupt, os.Kill)
	go func() {
		<-sigChan
		fmt.Println("\rCaught stop signal, quitting...")
		srv.Close()
	}()

	// Print a 'startup complete' message.
	startupTime := time.Since(loadStart)
	fmt.Println("Finished loading in", startupTime.Seconds(), "seconds")

	err = <-servErrs
	if err != nil {
		build.Critical(err)
	}

	return nil
}
<commit_msg>Fix: Make sure rpc port listens on all interfaces in default configuration<commit_after>package rivined

import (
	"errors"
	"fmt"
	"os"
	"os/signal"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/rivine/rivine/api"
	"github.com/rivine/rivine/build"
	"github.com/rivine/rivine/modules"
	"github.com/rivine/rivine/modules/blockcreator"
	"github.com/rivine/rivine/modules/consensus"
	"github.com/rivine/rivine/modules/explorer"
	"github.com/rivine/rivine/modules/gateway"
	"github.com/rivine/rivine/modules/transactionpool"
	"github.com/rivine/rivine/modules/wallet"

	"github.com/bgentry/speakeasy"
)

var (
	// globalConfig is used by the cobra package to fill out the configuration
	// variables.
	globalConfig Config
)

// The Config struct contains all configurable variables for siad. It is
// compatible with gcfg.
type Config struct {
	// The APIPassword is input by the user after the daemon starts up, if the
	// --authenticate-api flag is set.
	APIPassword string

	// The Rivined variables are referenced directly by cobra, and are set
	// according to the flags.
	Rivined RivinedCfg
}

// RivinedCfg holds variables referenced by cobra and set by flags
type RivinedCfg struct {
	APIaddr      string
	RPCaddr      string
	HostAddr     string
	AllowAPIBind bool

	Modules           string
	NoBootstrap       bool
	RequiredUserAgent string
	AuthenticateAPI   bool

	Profile    bool
	ProfileDir string
	RivineDir  string
}

// DefaultConfig returns the default daemon configuration
func DefaultConfig() RivinedCfg {
	return RivinedCfg{
		APIaddr:      "localhost:23110",
		RPCaddr:      ":23112",
		HostAddr:     "",
		AllowAPIBind: false,

		Modules:           "cgtwb",
		NoBootstrap:       false,
		RequiredUserAgent: "Rivine-Agent",
		AuthenticateAPI:   false,

		Profile:    false,
		ProfileDir: "profiles",
		RivineDir:  "",
	}
}

// verifyAPISecurity checks that the security values are consistent with a
// sane, secure system.
func verifyAPISecurity(config Config) error {
	// Make sure that only the loopback address is allowed unless the
	// --disable-api-security flag has been used.
	if !config.Rivined.AllowAPIBind {
		addr := modules.NetAddress(config.Rivined.APIaddr)
		if !addr.IsLoopback() {
			if addr.Host() == "" {
				return fmt.Errorf("a blank host will listen on all interfaces, did you mean localhost:%v?\nyou must pass --disable-api-security to bind %s to a non-localhost address", addr.Port(), DaemonName)
			}
			return fmt.Errorf("you must pass --disable-api-security to bind %s to a non-localhost address", DaemonName)
		}
		return nil
	}

	// If the --disable-api-security flag is used, enforce that
	// --authenticate-api must also be used.
	if config.Rivined.AllowAPIBind && !config.Rivined.AuthenticateAPI {
		return errors.New("cannot use --disable-api-security without setting an api password")
	}
	return nil
}

// processNetAddr adds a ':' to a bare integer, so that it is a proper port
// number.
func processNetAddr(addr string) string {
	_, err := strconv.Atoi(addr)
	if err == nil {
		return ":" + addr
	}
	return addr
}

// processModules makes the modules string lowercase to make checking if a
// module in the string easier, and returns an error if the string contains an
// invalid module character.
func processModules(modules string) (string, error) {
	modules = strings.ToLower(modules)
	validModules := "cgtweb"
	invalidModules := modules
	for _, m := range validModules {
		invalidModules = strings.Replace(invalidModules, string(m), "", 1)
	}
	if len(invalidModules) > 0 {
		return "", errors.New("Unable to parse --modules flag, unrecognized or duplicate modules: " + invalidModules)
	}
	return modules, nil
}

// processConfig checks the configuration values and performs cleanup on
// incorrect-but-allowed values.
func processConfig(config Config) (Config, error) {
	var err1 error
	config.Rivined.APIaddr = processNetAddr(config.Rivined.APIaddr)
	config.Rivined.RPCaddr = processNetAddr(config.Rivined.RPCaddr)
	config.Rivined.HostAddr = processNetAddr(config.Rivined.HostAddr)
	config.Rivined.Modules, err1 = processModules(config.Rivined.Modules)
	err2 := verifyAPISecurity(config)
	err := build.JoinErrors([]error{err1, err2}, ", and ")
	if err != nil {
		return Config{}, err
	}
	return config, nil
}

// startDaemon uses the config parameters to initialize Sia modules and start
// siad.
func startDaemon(config Config) (err error) {
	// Prompt user for API password.
	if config.Rivined.AuthenticateAPI {
		config.APIPassword, err = speakeasy.Ask("Enter API password: ")
		if err != nil {
			return err
		}
		if config.APIPassword == "" {
			return errors.New("password cannot be blank")
		}
	}

	// Process the config variables after they are parsed by cobra.
	config, err = processConfig(config)
	if err != nil {
		return err
	}

	// Print a startup message.
	fmt.Println("Loading...")
	loadStart := time.Now()

	// Create the server and start serving daemon routes immediately.
	fmt.Printf("(0/%d) Loading "+DaemonName+"...\n", len(config.Rivined.Modules))
	srv, err := NewServer(config.Rivined.APIaddr, config.Rivined.RequiredUserAgent, config.APIPassword)
	if err != nil {
		return err
	}

	servErrs := make(chan error)
	go func() {
		servErrs <- srv.Serve()
	}()

	// Initialize the Rivine modules
	i := 0
	var g modules.Gateway
	if strings.Contains(config.Rivined.Modules, "g") {
		i++
		fmt.Printf("(%d/%d) Loading gateway...\n", i, len(config.Rivined.Modules))
		g, err = gateway.New(config.Rivined.RPCaddr, !config.Rivined.NoBootstrap, filepath.Join(config.Rivined.RivineDir, modules.GatewayDir))
		if err != nil {
			return err
		}
		defer func() {
			fmt.Println("Closing gateway...")
			err := g.Close()
			if err != nil {
				fmt.Println("Error during gateway shutdown:", err)
			}
		}()

	}
	var cs modules.ConsensusSet
	if strings.Contains(config.Rivined.Modules, "c") {
		i++
		fmt.Printf("(%d/%d) Loading consensus...\n", i, len(config.Rivined.Modules))
		cs, err = consensus.New(g, !config.Rivined.NoBootstrap, filepath.Join(config.Rivined.RivineDir, modules.ConsensusDir))
		if err != nil {
			return err
		}
		defer func() {
			fmt.Println("Closing consensus set...")
			err := cs.Close()
			if err != nil {
				fmt.Println("Error during consensus set shutdown:", err)
			}
		}()

	}
	var e modules.Explorer
	if strings.Contains(config.Rivined.Modules, "e") {
		i++
		fmt.Printf("(%d/%d) Loading explorer...\n", i, len(config.Rivined.Modules))
		e, err = explorer.New(cs, filepath.Join(config.Rivined.RivineDir, modules.ExplorerDir))
		if err != nil {
			return err
		}
		defer func() {
			fmt.Println("Closing explorer...")
			err := e.Close()
			if err != nil {
				fmt.Println("Error during explorer shutdown:", err)
			}
		}()

	}
	var tpool modules.TransactionPool
	if strings.Contains(config.Rivined.Modules, "t") {
		i++
		fmt.Printf("(%d/%d) Loading transaction pool...\n", i, len(config.Rivined.Modules))
		tpool, err = transactionpool.New(cs, g, filepath.Join(config.Rivined.RivineDir, modules.TransactionPoolDir))
		if err != nil {
			return err
		}
		defer func() {
			fmt.Println("Closing transaction pool...")
			err := tpool.Close()
			if err != nil {
				fmt.Println("Error during transaction pool shutdown:", err)
			}
		}()

	}
	var w modules.Wallet
	if strings.Contains(config.Rivined.Modules, "w") {
		i++
		fmt.Printf("(%d/%d) Loading wallet...\n", i, len(config.Rivined.Modules))
		w, err = wallet.New(cs, tpool, filepath.Join(config.Rivined.RivineDir, modules.WalletDir))
		if err != nil {
			return err
		}
		defer func() {
			fmt.Println("Closing wallet...")
			err := w.Close()
			if err != nil {
				fmt.Println("Error during wallet shutdown:", err)
			}
		}()

	}
	var b modules.BlockCreator
	if strings.Contains(config.Rivined.Modules, "b") {
		i++
		fmt.Printf("(%d/%d) Loading block creator...\n", i, len(config.Rivined.Modules))
		b, err = blockcreator.New(cs, tpool, w, filepath.Join(config.Rivined.RivineDir, modules.BlockCreatorDir))
		if err != nil {
			return err
		}
		defer func() {
			fmt.Println("Closing block creator...")
			err := b.Close()
			if err != nil {
				fmt.Println("Error during block creator shutdown:", err)
			}
		}()
	}

	// Create the Sia API
	a := api.New(
		config.Rivined.RequiredUserAgent,
		config.APIPassword,
		cs,
		e,
		g,
		tpool,
		w,
	)

	// connect the API to the server
	srv.mux.Handle("/", a)

	// stop the server if a kill signal is caught
	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, os.Interrupt, os.Kill)
	go func() {
		<-sigChan
		fmt.Println("\rCaught stop signal, quitting...")
		srv.Close()
	}()

	// Print a 'startup complete' message.
	startupTime := time.Since(loadStart)
	fmt.Println("Finished loading in", startupTime.Seconds(), "seconds")

	err = <-servErrs
	if err != nil {
		build.Critical(err)
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"os/exec"
	"strings"
	"time"

	"github.com/go-check/check"

	"code.google.com/p/go.net/websocket"
)

func (s *DockerSuite) TestGetContainersAttachWebsocket(c *check.C) {
	runCmd := exec.Command(dockerBinary, "run", "-dit", "busybox", "cat")
	out, _, err := runCommandWithOutput(runCmd)
	if err != nil {
		c.Fatalf(out, err)
	}

	rwc, err := sockConn(time.Duration(10 * time.Second))
	if err != nil {
		c.Fatal(err)
	}

	cleanedContainerID := strings.TrimSpace(out)
	config, err := websocket.NewConfig(
		"/containers/"+cleanedContainerID+"/attach/ws?stream=1&stdin=1&stdout=1&stderr=1",
		"http://localhost",
	)
	if err != nil {
		c.Fatal(err)
	}

	ws, err := websocket.NewClient(config, rwc)
	if err != nil {
		c.Fatal(err)
	}
	defer ws.Close()

	expected := []byte("hello")
	actual := make([]byte, len(expected))
	outChan := make(chan string)
	go func() {
		if _, err := ws.Read(actual); err != nil {
			c.Fatal(err)
		}
		outChan <- "done"
	}()

	inChan := make(chan string)
	go func() {
		if _, err := ws.Write(expected); err != nil {
			c.Fatal(err)
		}
		inChan <- "done"
	}()

	<-inChan
	<-outChan

	if !bytes.Equal(expected, actual) {
		c.Fatal("Expected output on websocket to match input")
	}
}
<commit_msg>Remove c.Fatal from goroutine in TestGetContainersAttachWebsocket<commit_after>package main

import (
	"bytes"
	"os/exec"
	"strings"
	"time"

	"github.com/go-check/check"

	"code.google.com/p/go.net/websocket"
)

func (s *DockerSuite) TestGetContainersAttachWebsocket(c *check.C) {
	runCmd := exec.Command(dockerBinary, "run", "-dit", "busybox", "cat")
	out, _, err := runCommandWithOutput(runCmd)
	if err != nil {
		c.Fatalf(out, err)
	}

	rwc, err := sockConn(time.Duration(10 * time.Second))
	if err != nil {
		c.Fatal(err)
	}

	cleanedContainerID := strings.TrimSpace(out)
	config, err := websocket.NewConfig(
		"/containers/"+cleanedContainerID+"/attach/ws?stream=1&stdin=1&stdout=1&stderr=1",
		"http://localhost",
	)
	if err != nil {
		c.Fatal(err)
	}

	ws, err := websocket.NewClient(config, rwc)
	if err != nil {
		c.Fatal(err)
	}
	defer ws.Close()

	expected := []byte("hello")
	actual := make([]byte, len(expected))

	outChan := make(chan error)
	go func() {
		_, err := ws.Read(actual)
		outChan <- err
		close(outChan)
	}()

	inChan := make(chan error)
	go func() {
		_, err := ws.Write(expected)
		inChan <- err
		close(inChan)
	}()

	select {
	case err := <-inChan:
		if err != nil {
			c.Fatal(err)
		}
	case <-time.After(5 * time.Second):
		c.Fatal("Timeout writing to ws")
	}

	select {
	case err := <-outChan:
		if err != nil {
			c.Fatal(err)
		}
	case <-time.After(5 * time.Second):
		c.Fatal("Timeout reading from ws")
	}

	if !bytes.Equal(expected, actual) {
		c.Fatal("Expected output on websocket to match input")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"os/exec"
	"strings"
	"sync"
	"testing"
	"time"
)

const attachWait = 5 * time.Second

func TestMultipleAttachRestart(t *testing.T) {
	defer deleteAllContainers()

	endGroup := &sync.WaitGroup{}
	startGroup := &sync.WaitGroup{}
	endGroup.Add(3)
	startGroup.Add(3)

	if err := waitForContainer("attacher", "-d", "busybox", "/bin/sh", "-c", "while true; do sleep 1; echo hello; done"); err != nil {
		t.Fatal(err)
	}

	startDone := make(chan struct{})
	endDone := make(chan struct{})

	go func() {
		endGroup.Wait()
		close(endDone)
	}()

	go func() {
		startGroup.Wait()
		close(startDone)
	}()

	for i := 0; i < 3; i++ {
		go func() {
			c := exec.Command(dockerBinary, "attach", "attacher")

			defer func() {
				c.Wait()
				endGroup.Done()
			}()

			out, err := c.StdoutPipe()
			if err != nil {
				t.Fatal(err)
			}

			if _, err := startCommand(c); err != nil {
				t.Fatal(err)
			}

			buf := make([]byte, 1024)

			if _, err := out.Read(buf); err != nil && err != io.EOF {
				t.Fatal(err)
			}

			startGroup.Done()

			if !strings.Contains(string(buf), "hello") {
				t.Fatalf("unexpected output %s expected hello\n", string(buf))
			}
		}()
	}

	select {
	case <-startDone:
	case <-time.After(attachWait):
		t.Fatalf("Attaches did not initialize properly")
	}

	cmd := exec.Command(dockerBinary, "kill", "attacher")
	if _, err := runCommand(cmd); err != nil {
		t.Fatal(err)
	}

	select {
	case <-endDone:
	case <-time.After(attachWait):
		t.Fatalf("Attaches did not finish properly")
	}

	logDone("attach - multiple attach")
}
<commit_msg>Use prefix naming for attach tests<commit_after>package main

import (
	"io"
	"os/exec"
	"strings"
	"sync"
	"testing"
	"time"
)

const attachWait = 5 * time.Second

func TestAttachMultipleAndRestart(t *testing.T) {
	defer deleteAllContainers()

	endGroup := &sync.WaitGroup{}
	startGroup := &sync.WaitGroup{}
	endGroup.Add(3)
	startGroup.Add(3)

	if err := waitForContainer("attacher", "-d", "busybox", "/bin/sh", "-c", "while true; do sleep 1; echo hello; done"); err != nil {
		t.Fatal(err)
	}

	startDone := make(chan struct{})
	endDone := make(chan struct{})

	go func() {
		endGroup.Wait()
		close(endDone)
	}()

	go func() {
		startGroup.Wait()
		close(startDone)
	}()

	for i := 0; i < 3; i++ {
		go func() {
			c := exec.Command(dockerBinary, "attach", "attacher")

			defer func() {
				c.Wait()
				endGroup.Done()
			}()

			out, err := c.StdoutPipe()
			if err != nil {
				t.Fatal(err)
			}

			if _, err := startCommand(c); err != nil {
				t.Fatal(err)
			}

			buf := make([]byte, 1024)

			if _, err := out.Read(buf); err != nil && err != io.EOF {
				t.Fatal(err)
			}

			startGroup.Done()

			if !strings.Contains(string(buf), "hello") {
				t.Fatalf("unexpected output %s expected hello\n", string(buf))
			}
		}()
	}

	select {
	case <-startDone:
	case <-time.After(attachWait):
		t.Fatalf("Attaches did not initialize properly")
	}

	cmd := exec.Command(dockerBinary, "kill", "attacher")
	if _, err := runCommand(cmd); err != nil {
		t.Fatal(err)
	}

	select {
	case <-endDone:
	case <-time.After(attachWait):
		t.Fatalf("Attaches did not finish properly")
	}

	logDone("attach - multiple attach")
}
<|endoftext|>


<commit_before>package github

import (
	// Stdlib
	"encoding/json"
	"net/http"
	"strconv"

	// Internal
	"github.com/salsaflow/salsaflow-daemon/internal/log"
	"github.com/salsaflow/salsaflow-daemon/internal/trackers"
	"github.com/salsaflow/salsaflow-daemon/internal/utils/githubutils"
	"github.com/salsaflow/salsaflow-daemon/internal/utils/httputils"

	// Vendor
	"github.com/google/go-github/github"
	"github.com/salsaflow/salsaflow/github/issues"
)

func handleIssuesEvent(rw http.ResponseWriter, r *http.Request) {
	// Parse the payload.
	var event github.IssueActivityEvent
	if err := json.NewDecoder(r.Body).Decode(&event); err != nil {
		log.Warn(r, "failed to parse event: %v", err)
		httputils.Status(rw, http.StatusBadRequest)
		return
	}

	// Make sure this is a review issue event.
	// The label is sometimes missing in the webhook, we need to re-fetch.
	client, err := githubutils.NewClient()
	if err != nil {
		httputils.Error(rw, r, err)
		return
	}
	var (
		owner    = *event.Repo.Owner.Login
		repo     = *event.Repo.Name
		issueNum = *event.Issue.Number
	)
	log.Info(r, "Re-fetching issue %v/%v#%v", owner, repo, issueNum)
	issue, _, err := client.Issues.Get(owner, repo, issueNum)
	if err != nil {
		httputils.Error(rw, r, err)
		return
	}

	var isReviewIssue bool
	for _, label := range issue.Labels {
		if *label.Name == "review" {
			isReviewIssue = true
			break
		}
	}
	if !isReviewIssue {
		log.Info(r, "Issue %s is not a review issue", *issue.HTMLURL)
		httputils.Status(rw, http.StatusAccepted)
		return
	}

	// Do nothing unless this is an opened, closed or reopened event.
	switch *event.Action {
	case "opened":
	case "closed":
	case "reopened":
	default:
		httputils.Status(rw, http.StatusAccepted)
		return
	}

	// Parse issue body.
	reviewIssue, err := issues.ParseReviewIssue(issue)
	if err != nil {
		log.Error(r, err)
		httputils.Status(rw, httputils.StatusUnprocessableEntity)
		return
	}

	// We are done in case this is a commit review issue.
	storyIssue, ok := reviewIssue.(*issues.StoryReviewIssue)
	if !ok {
		httputils.Status(rw, http.StatusAccepted)
		return
	}

	// Instantiate the issue tracker.
	tracker, err := trackers.GetIssueTracker(storyIssue.TrackerName)
	if err != nil {
		log.Error(r, err)
		httputils.Status(rw, httputils.StatusUnprocessableEntity)
		return
	}

	// Find relevant story.
	story, err := tracker.FindStoryByTag(storyIssue.StoryKey)
	if err != nil {
		log.Error(r, err)
		httputils.Status(rw, httputils.StatusUnprocessableEntity)
		return
	}

	// Invoke relevant event handler.
	var (
		issueNumString = strconv.Itoa(*issue.Number)
		issueURL       = *issue.HTMLURL
		ex             error
	)
	switch *event.Action {
	case "opened":
		ex = story.OnReviewRequestOpened(issueNumString, issueURL)
	case "closed":
		ex = story.OnReviewRequestClosed(issueNumString, issueURL)
	case "reopened":
		ex = story.OnReviewRequestReopened(issueNumString, issueURL)
	default:
		panic("unreachable code reached")
	}
	if ex != nil {
		httputils.Error(rw, r, ex)
		return
	}

	if *event.Action == "closed" {
		if err := story.MarkAsReviewed(); err != nil {
			httputils.Error(rw, r, err)
			return
		}
	}

	httputils.Status(rw, http.StatusAccepted)
}
<commit_msg>handlers/github: Reject Close when not Implemented<commit_after>package github

import (
	// Stdlib
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"

	// Internal
	"github.com/salsaflow/salsaflow-daemon/internal/log"
	"github.com/salsaflow/salsaflow-daemon/internal/trackers"
	"github.com/salsaflow/salsaflow-daemon/internal/utils/githubutils"
	"github.com/salsaflow/salsaflow-daemon/internal/utils/httputils"

	// Vendor
	"github.com/google/go-github/github"
	"github.com/salsaflow/salsaflow/github/issues"
)

func handleIssuesEvent(rw http.ResponseWriter, r *http.Request) {
	// Parse the payload.
	var event github.IssueActivityEvent
	if err := json.NewDecoder(r.Body).Decode(&event); err != nil {
		log.Warn(r, "failed to parse event: %v", err)
		httputils.Status(rw, http.StatusBadRequest)
		return
	}

	// Make sure this is a review issue event.
	// The label is sometimes missing in the webhook, we need to re-fetch.
	client, err := githubutils.NewClient()
	if err != nil {
		httputils.Error(rw, r, err)
		return
	}
	var (
		owner    = *event.Repo.Owner.Login
		repo     = *event.Repo.Name
		issueNum = *event.Issue.Number
	)
	log.Info(r, "Re-fetching issue %v/%v#%v", owner, repo, issueNum)
	issue, _, err := client.Issues.Get(owner, repo, issueNum)
	if err != nil {
		httputils.Error(rw, r, err)
		return
	}

	// Make sure this is a review issue.
	labeledWith := func(label string) bool {
		for _, labelPtr := range issue.Labels {
			if *labelPtr.Name == label {
				return true
			}
		}
		return false
	}

	if !labeledWith("review") {
		log.Info(r, "Issue %s is not a review issue", *issue.HTMLURL)
		httputils.Status(rw, http.StatusAccepted)
		return
	}

	// Do nothing unless this is an opened, closed or reopened event.
	switch *event.Action {
	case "opened":
	case "closed":
		// Make sure the issue is marked as implemented.
		if !labeledWith("implemented") {
			rejectClose(rw, r, client, &event)
			return
		}

	case "reopened":
	default:
		httputils.Status(rw, http.StatusAccepted)
		return
	}

	// Parse issue body.
	reviewIssue, err := issues.ParseReviewIssue(issue)
	if err != nil {
		log.Error(r, err)
		httputils.Status(rw, httputils.StatusUnprocessableEntity)
		return
	}

	// We are done in case this is a commit review issue.
	storyIssue, ok := reviewIssue.(*issues.StoryReviewIssue)
	if !ok {
		httputils.Status(rw, http.StatusAccepted)
		return
	}

	// Instantiate the issue tracker.
	tracker, err := trackers.GetIssueTracker(storyIssue.TrackerName)
	if err != nil {
		log.Error(r, err)
		httputils.Status(rw, httputils.StatusUnprocessableEntity)
		return
	}

	// Find relevant story.
	story, err := tracker.FindStoryByTag(storyIssue.StoryKey)
	if err != nil {
		log.Error(r, err)
		httputils.Status(rw, httputils.StatusUnprocessableEntity)
		return
	}

	// Invoke relevant event handler.
	var (
		issueNumString = strconv.Itoa(*issue.Number)
		issueURL       = *issue.HTMLURL
		ex             error
	)
	switch *event.Action {
	case "opened":
		ex = story.OnReviewRequestOpened(issueNumString, issueURL)
	case "closed":
		ex = story.OnReviewRequestClosed(issueNumString, issueURL)
	case "reopened":
		ex = story.OnReviewRequestReopened(issueNumString, issueURL)
	default:
		panic("unreachable code reached")
	}
	if ex != nil {
		httputils.Error(rw, r, ex)
		return
	}

	if *event.Action == "closed" {
		if err := story.MarkAsReviewed(); err != nil {
			httputils.Error(rw, r, err)
			return
		}
	}

	httputils.Status(rw, http.StatusAccepted)
}

func rejectClose(
	rw http.ResponseWriter,
	r *http.Request,
	client *github.Client,
	event *github.IssueActivityEvent) {

	var (
		owner    = *event.Repo.Owner.Login
		repo     = *event.Repo.Name
		issueNum = *event.Issue.Number
		sender   = *event.Sender.Login
	)

	// Log stuff.
	log.Info(r, "Reopening review issue %v/%v#%v, not implemented yet", owner, repo, issueNum)

	// Re-open the issue.
	_, _, err := client.Issues.Edit(owner, repo, issueNum, &github.IssueRequest{
		State: github.String("open"),
	})
	if err != nil {
		httputils.Error(rw, r, err)
		return
	}

	// Add a comment to notify the sender.
	var body bytes.Buffer
	fmt.Fprintf(&body,
		"@%v Reopening review issue #%v, the associated story is not implemented yet.\n",
		sender, issueNum)
	fmt.Fprintln(&body,
		"The review issue needs to be labeled with `implemented`, then it can be closed.")

	_, _, err = client.Issues.CreateComment(owner, repo, issueNum, &github.IssueComment{
		Body: github.String(body.String()),
	})
	if err != nil {
		httputils.Error(rw, r, err)
		return
	}

	httputils.Status(rw, http.StatusAccepted)
}
<|endoftext|>


<commit_before>package aphid

import (
	// External requirements.
	_ "github.com/BurntSushi/toml"
	_ "github.com/gdamore/mangos"
	_ "github.com/huandu/skiplist"
	_ "github.com/microcosm-cc/bluemonday"
	_ "github.com/russross/blackfriday"
	_ "golang.org/x/crypto/scrypt"
)

import (
	// Yak requirements.
	_ "github.com/strickyak/redhed"
	_ "github.com/strickyak/rye"
	_ "github.com/yak-labs/chirp-lang"
	_ "github.com/yak-labs/chirp-lang/goapi/default"
	_ "github.com/yak-labs/chirp-lang/http"
	_ "github.com/yak-labs/chirp-lang/img"
	_ "github.com/yak-labs/chirp-lang/posix"
	_ "github.com/yak-labs/chirp-lang/rpc"
	_ "github.com/yak-labs/chirp-lang/ryba"
)

import (
	_ "io"
)
<commit_msg>require "github.com/syndtr/goleveldb/leveldb" for aphid.<commit_after>package aphid

import (
	// External requirements.
	_ "github.com/BurntSushi/toml"
	_ "github.com/gdamore/mangos"
	_ "github.com/huandu/skiplist"
	_ "github.com/microcosm-cc/bluemonday"
	_ "github.com/russross/blackfriday"
	_ "golang.org/x/crypto/scrypt"
	_ "github.com/syndtr/goleveldb/leveldb"
)

import (
	// Yak requirements.
	_ "github.com/strickyak/redhed"
	_ "github.com/strickyak/rye"
	_ "github.com/yak-labs/chirp-lang"
	_ "github.com/yak-labs/chirp-lang/goapi/default"
	_ "github.com/yak-labs/chirp-lang/http"
	_ "github.com/yak-labs/chirp-lang/img"
	_ "github.com/yak-labs/chirp-lang/posix"
	_ "github.com/yak-labs/chirp-lang/rpc"
	_ "github.com/yak-labs/chirp-lang/ryba"
)

import (
	_ "io"
)
<|endoftext|>


<commit_before>package api

import (
	"github.com/30x/apid-core"
	"github.com/gorilla/mux"
	"net/http"
	"expvar"
	"fmt"
	"github.com/30x/goscaffold"
)

// todo: handle TLS
// todo: handle other basic router config, errors, etc.

const (
	configAPIPort = "api_port"
	configExpVarPath = "api_expvar_path"
	configReadyPath = "api_ready"
	configHealthyPath = "api_healthy"
)

var log apid.LogService
var config apid.ConfigService
var requests *expvar.Map = expvar.NewMap("requests")

func CreateService() apid.APIService {
	config = apid.Config()
	log = apid.Log().ForModule("api")

	config.SetDefault(configAPIPort, 9000)
	config.SetDefault(configReadyPath, "/ready")
	config.SetDefault(configHealthyPath, "/healthy")

	r := mux.NewRouter()
	rw := &router{r}
	scaffold := goscaffold.CreateHTTPScaffold()
	return &service{rw, scaffold}
}

type service struct {
	*router
	scaffold *goscaffold.HTTPScaffold
}

func (s *service) Listen() error {
	port := config.GetInt(configAPIPort)
	log.Infof("opening api port %d", port)

	s.InitExpVar()

	s.scaffold.SetInsecurePort(port)

	// Direct the scaffold to catch common signals and trigger a graceful shutdown.
	s.scaffold.CatchSignals()

	// Set an URL that may be used by a load balancer to test if the server is ready to handle requests
	if config.GetString(configReadyPath) != "" {
		s.scaffold.SetReadyPath(config.GetString(configReadyPath))
	}

	// Set an URL that may be used by infrastructure to test
	// if the server is working or if it needs to be restarted or replaced
	if config.GetString(configHealthyPath) != "" {
		s.scaffold.SetReadyPath(config.GetString(configHealthyPath))
	}

	err := s.scaffold.StartListen(s.r)
	if err != nil {
		return err
	}

	apid.Events().Emit(apid.SystemEventsSelector, apid.APIListeningEvent)

	return s.scaffold.WaitForShutdown()
}

func (s *service) Close() {
	s.scaffold.Shutdown(nil)
	s.scaffold = nil
}

func (s *service) InitExpVar() {
	if config.IsSet(configExpVarPath) {
		log.Infof("expvar available on path: %s", config.Get(configExpVarPath))
		s.HandleFunc(config.GetString(configExpVarPath), expvarHandler)
	}
}

// for testing
func (s *service) Router() apid.Router {
	s.InitExpVar()
	return s
}

func (s *service) Vars(r *http.Request) map[string]string {
	return mux.Vars(r)
}

func expvarHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	fmt.Fprint(w, "{\n")
	first := true
	expvar.Do(func(kv expvar.KeyValue) {
		if !first {
			fmt.Fprint(w, ",\n")
		}
		first = false
		fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value)
	})
	fmt.Fprint(w, "\n}\n")
}

type router struct {
	r *mux.Router
}

func (r *router) Handle(path string, handler http.Handler) apid.Route {
	log.Infof("Handle %s: %v", path, handler)
	return &route{r.r.Handle(path, handler)}
}

func (r *router) HandleFunc(path string, handlerFunc http.HandlerFunc) apid.Route {
	log.Infof("Handle %s: %v", path, handlerFunc)
	return &route{r.r.HandleFunc(path, handlerFunc)}
}

func (r *router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	requests.Add(req.URL.Path, 1)
	log.Infof("Handling %s", req.URL.Path)
	r.r.ServeHTTP(w, req)
}

type route struct {
	r *mux.Route
}

func (r *route) Methods(methods ...string) apid.Route {
	return &route{r.r.Methods(methods...)}
}
<commit_msg>allow apid to bind listener to a host (127.0.0.1 by default)<commit_after>package api

import (
	"expvar"
	"fmt"
	"net/http"

	"net"

	"github.com/30x/apid-core"
	"github.com/30x/goscaffold"
	"github.com/gorilla/mux"
)

const (
	configAPIListen   = "api_listen"
	configExpVarPath  = "api_expvar_path"
	configReadyPath   = "api_ready"
	configHealthyPath = "api_healthy"
)

var log apid.LogService
var config apid.ConfigService
var requests *expvar.Map = expvar.NewMap("requests")

func CreateService() apid.APIService {
	config = apid.Config()
	log = apid.Log().ForModule("api")

	config.SetDefault(configAPIListen, "127.0.0.1:9000")
	config.SetDefault(configReadyPath, "/ready")
	config.SetDefault(configHealthyPath, "/healthy")

	listen := config.GetString(configAPIListen)
	h, p, err := net.SplitHostPort(listen)
	if err != nil {
		log.Panicf("%s config: err parsing '%s': %v", configAPIListen, listen, err)
	}
	var ip net.IP
	if h != "" {
		ips, err := net.LookupIP(h)
		if err != nil {
			log.Panicf("%s config: unable to resolve IP for '%s': %v", configAPIListen, listen, err)
		}
		ip = ips[0]
	}
	port, err := net.LookupPort("tcp", p)
	if err != nil {
		log.Panicf("%s config: unable to resolve port for '%s': %v", configAPIListen, listen, err)
	}

	log.Infof("will open api port %d bound to %s", port, ip)

	r := mux.NewRouter()
	rw := &router{r}
	scaffold := goscaffold.CreateHTTPScaffold()
	if ip != nil {
		scaffold.SetlocalBindIPAddressV4(ip)
	}
	scaffold.SetInsecurePort(port)
	scaffold.CatchSignals()

	// Set an URL that may be used by a load balancer to test if the server is ready to handle requests
	if config.GetString(configReadyPath) != "" {
		scaffold.SetReadyPath(config.GetString(configReadyPath))
	}

	// Set an URL that may be used by infrastructure to test
	// if the server is working or if it needs to be restarted or replaced
	if config.GetString(configHealthyPath) != "" {
		scaffold.SetReadyPath(config.GetString(configHealthyPath))
	}

	return &service{rw, scaffold}
}

type service struct {
	*router
	scaffold *goscaffold.HTTPScaffold
}

func (s *service) Listen() error {
	err := s.scaffold.StartListen(s.r)
	if err != nil {
		return err
	}

	apid.Events().Emit(apid.SystemEventsSelector, apid.APIListeningEvent)

	return s.scaffold.WaitForShutdown()
}

func (s *service) Close() {
	s.scaffold.Shutdown(nil)
	s.scaffold = nil
}

func (s *service) InitExpVar() {
	if config.IsSet(configExpVarPath) {
		log.Infof("expvar available on path: %s", config.Get(configExpVarPath))
		s.HandleFunc(config.GetString(configExpVarPath), expvarHandler)
	}
}

// for testing
func (s *service) Router() apid.Router {
	s.InitExpVar()
	return s
}

func (s *service) Vars(r *http.Request) map[string]string {
	return mux.Vars(r)
}

func expvarHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	fmt.Fprint(w, "{\n")
	first := true
	expvar.Do(func(kv expvar.KeyValue) {
		if !first {
			fmt.Fprint(w, ",\n")
		}
		first = false
		fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value)
	})
	fmt.Fprint(w, "\n}\n")
}

type router struct {
	r *mux.Router
}

func (r *router) Handle(path string, handler http.Handler) apid.Route {
	log.Infof("Handle %s: %v", path, handler)
	return &route{r.r.Handle(path, handler)}
}

func (r *router) HandleFunc(path string, handlerFunc http.HandlerFunc) apid.Route {
	log.Infof("Handle %s: %v", path, handlerFunc)
	return &route{r.r.HandleFunc(path, handlerFunc)}
}

func (r *router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	requests.Add(req.URL.Path, 1)
	log.Infof("Handling %s", req.URL.Path)
	r.r.ServeHTTP(w, req)
}

type route struct {
	r *mux.Route
}

func (r *route) Methods(methods ...string) apid.Route {
	return &route{r.r.Methods(methods...)}
}
<|endoftext|>


<commit_before>package api

type MetricKey string

type TagSet map[string]string

type TaggedMetric struct {
	MetricKey MetricKey
	TagSet    TagSet
}

type GraphiteMetric string

type Api struct {
}

func (api *Api) AddMetric(metric TaggedMetric) error {
  return nil
}

func (api *Api) ToGraphiteName(metric TaggedMetric) GraphiteMetric {
  return ""
}

func (api *Api) ToTaggedName(metric GraphiteMetric) TaggedMetric {
  return TaggedMetric{}
}

func (api *Api) GetAllTags(metricKey MetricKey) []TagSet {
  return nil
}

func (api *Api) GetMericsForTag(tagKey string, tagValue string) []MetricKey {
  return nil
}
<commit_msg>Adding more documentation to API.<commit_after>// Package api holds common data types and public interface exposed by the indexer library.
package api

// Refer to the doc
// https://docs.google.com/a/squareup.com/document/d/1k0Wgi2wnJPQoyDyReb9dyIqRrD8-v0u8hz37S282ii4/edit
// for the terminology.

// MetricKey is the logical name of a given metric.
// MetricKey should not contain any variable component in it.
type MetricKey string

// TagSet is the set of key-value pairs associated with a given metric.
type TagSet map[string]string

// TaggedMetric is composition of a MetricKey and a TagSet.
// TaggedMetric should uniquely identify a single series of metric.
type TaggedMetric struct {
	MetricKey MetricKey
	TagSet    TagSet
}

// GraphiteMetric is a flat, dot-separated identifier to a series of metric.
type GraphiteMetric string

// API is the set of public methods exposed by the indexer library.
type API interface {
	// AddMetric adds the metric to the system.
	AddMetric(metric TaggedMetric) error

  // RemoveMetric removes the metric from the system.
	RemoveMetric(metric TaggedMetric) error

  // Convert the given tag-based metric name to graphite metric name,
  // using the configured rules. May error out.
	ToGraphiteName(metric TaggedMetric) (GraphiteMetric, error)

  // Converts the given graphite metric to the tag-based meric,
  // using the configured rules. May error out.
	ToTaggedName(metric GraphiteMetric) (TaggedMetric, error)

  // For a given MetricKey, retrieve all the tagsets associated with it.
	GetAllTags(metricKey MetricKey) []TagSet

  // For a given tag key-value pair, obtain the list of all the MetricKeys
  // associated with them.
	GetMericsForTag(tagKey string, tagValue string) []MetricKey
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hashers

import (
	"fmt"

	"github.com/google/trillian"
)

// LogHasher provides the hash functions needed to compute dense merkle trees.
type LogHasher interface {
	// EmptyRoot supports returning a special case for the root of an empty tree.
	EmptyRoot() []byte
	// HashLeaf computes the hash of a leaf that exists.
	HashLeaf(leaf []byte) ([]byte, error)
	// HashChildren computes interior nodes.
	HashChildren(l, r []byte) []byte
	// Size is the number of bytes in the underlying hash function.
	// TODO(gbelvin): Replace Size() with BitLength().
	Size() int
}

// MapHasher provides the hash functions needed to compute sparse merkle trees.
type MapHasher interface {
	// HashEmpty returns the hash of an empty branch at a given depth.
	// A height of 0 indicates an empty leaf. The maximum height is Size*8.
	// TODO(gbelvin) fully define index.
	HashEmpty(treeID int64, index []byte, height int) []byte
	// HashLeaf computes the hash of a leaf that exists.
	HashLeaf(treeID int64, index []byte, leaf []byte) ([]byte, error)
	// HashChildren computes interior nodes.
	HashChildren(l, r []byte) []byte
	// Size is the number of bytes in the underlying hash function.
	// TODO(gbelvin): Replace Size() with BitLength().
	Size() int
	// BitLen returns the number of bits in the underlying hash function.
	// It is also the height of the merkle tree.
	BitLen() int
}

var (
	logHashers = make(map[trillian.HashStrategy]LogHasher)
	mapHashers = make(map[trillian.HashStrategy]MapHasher)
)

// RegisterLogHasher registers a hasher for use.
func RegisterLogHasher(h trillian.HashStrategy, f LogHasher) {
	if h == trillian.HashStrategy_UNKNOWN_HASH_STRATEGY {
		panic(fmt.Sprintf("RegisterLogHasher(%s) of unknown hasher", h))
	}
	if logHashers[h] != nil {
		panic(fmt.Sprintf("%v already registered as a LogHasher", h))
	}
	logHashers[h] = f
}

// RegisterMapHasher registers a hasher for use.
func RegisterMapHasher(h trillian.HashStrategy, f MapHasher) {
	if h == trillian.HashStrategy_UNKNOWN_HASH_STRATEGY {
		panic(fmt.Sprintf("RegisterMapHasher(%s) of unknown hasher", h))
	}
	if mapHashers[h] != nil {
		panic(fmt.Sprintf("%v already registered as a MapHasher", h))
	}
	mapHashers[h] = f
}

// NewLogHasher returns a LogHasher.
func NewLogHasher(h trillian.HashStrategy) (LogHasher, error) {
	f := logHashers[h]
	if f != nil {
		return f, nil
	}
	return nil, fmt.Errorf("LogHasher(%s) is an unknown hasher", h)
}

// NewMapHasher returns a MapHasher.
func NewMapHasher(h trillian.HashStrategy) (MapHasher, error) {
	f := mapHashers[h]
	if f != nil {
		return f, nil
	}
	return nil, fmt.Errorf("MapHasher(%s) is an unknown hasher", h)
}
<commit_msg>merkle: clarify hashes for empty/zero-len leaves<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hashers

import (
	"fmt"

	"github.com/google/trillian"
)

// LogHasher provides the hash functions needed to compute dense merkle trees.
type LogHasher interface {
	// EmptyRoot supports returning a special case for the root of an empty tree.
	EmptyRoot() []byte
	// HashLeaf computes the hash of a leaf that exists.
	HashLeaf(leaf []byte) ([]byte, error)
	// HashChildren computes interior nodes.
	HashChildren(l, r []byte) []byte
	// Size is the number of bytes in the underlying hash function.
	// TODO(gbelvin): Replace Size() with BitLength().
	Size() int
}

// MapHasher provides the hash functions needed to compute sparse merkle trees.
type MapHasher interface {
	// HashEmpty returns the hash of an empty branch at a given depth.
	// A height of 0 indicates an empty leaf. The maximum height is Size*8.
	// TODO(gbelvin) fully define index.
	HashEmpty(treeID int64, index []byte, height int) []byte
	// HashLeaf computes the hash of a leaf that exists.  This method
	// is *not* used for computing the hash of a leaf that does not exist
	// (instead, HashEmpty(treeID, index, 0) is used), as the hash value
	// can be different between:
	//  - a leaf that is unset
	//  - a leaf that has been explicitly set, including set to []byte{}.
	HashLeaf(treeID int64, index []byte, leaf []byte) ([]byte, error)
	// HashChildren computes interior nodes, when at least one of the child
	// subtrees is non-empty.
	HashChildren(l, r []byte) []byte
	// Size is the number of bytes in the underlying hash function.
	// TODO(gbelvin): Replace Size() with BitLength().
	Size() int
	// BitLen returns the number of bits in the underlying hash function.
	// It is also the height of the merkle tree.
	BitLen() int
}

var (
	logHashers = make(map[trillian.HashStrategy]LogHasher)
	mapHashers = make(map[trillian.HashStrategy]MapHasher)
)

// RegisterLogHasher registers a hasher for use.
func RegisterLogHasher(h trillian.HashStrategy, f LogHasher) {
	if h == trillian.HashStrategy_UNKNOWN_HASH_STRATEGY {
		panic(fmt.Sprintf("RegisterLogHasher(%s) of unknown hasher", h))
	}
	if logHashers[h] != nil {
		panic(fmt.Sprintf("%v already registered as a LogHasher", h))
	}
	logHashers[h] = f
}

// RegisterMapHasher registers a hasher for use.
func RegisterMapHasher(h trillian.HashStrategy, f MapHasher) {
	if h == trillian.HashStrategy_UNKNOWN_HASH_STRATEGY {
		panic(fmt.Sprintf("RegisterMapHasher(%s) of unknown hasher", h))
	}
	if mapHashers[h] != nil {
		panic(fmt.Sprintf("%v already registered as a MapHasher", h))
	}
	mapHashers[h] = f
}

// NewLogHasher returns a LogHasher.
func NewLogHasher(h trillian.HashStrategy) (LogHasher, error) {
	f := logHashers[h]
	if f != nil {
		return f, nil
	}
	return nil, fmt.Errorf("LogHasher(%s) is an unknown hasher", h)
}

// NewMapHasher returns a MapHasher.
func NewMapHasher(h trillian.HashStrategy) (MapHasher, error) {
	f := mapHashers[h]
	if f != nil {
		return f, nil
	}
	return nil, fmt.Errorf("MapHasher(%s) is an unknown hasher", h)
}
<|endoftext|>


<commit_before>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package messenger

import (
	"bytes"
	"fmt"
	"github.com/mesos/mesos-go/upid"
	"io/ioutil"
	"net"
	"net/http"
	"strings"

	log "github.com/golang/glog"
	"golang.org/x/net/context"
)

// HTTPTransporter implements the interfaces of the Transporter.
type HTTPTransporter struct {
	// If the host is empty("") then it will listen on localhost.
	// If the port is empty("") then it will listen on random port.
	upid         *upid.UPID
	listener     net.Listener // TODO(yifan): Change to TCPListener.
	mux          *http.ServeMux
	tr           *http.Transport
	client       *http.Client // TODO(yifan): Set read/write deadline.
	messageQueue chan *Message
}

// NewHTTPTransporter creates a new http transporter.
func NewHTTPTransporter(upid *upid.UPID) *HTTPTransporter {
	tr := &http.Transport{}
	return &HTTPTransporter{
		upid:         upid,
		messageQueue: make(chan *Message, defaultQueueSize),
		mux:          http.NewServeMux(),
		client:       &http.Client{Transport: tr},
		tr:           tr,
	}
}

// Send sends the message to its specified upid.
func (t *HTTPTransporter) Send(ctx context.Context, msg *Message) error {
	log.V(2).Infof("Sending message to %v via http\n", msg.UPID)
	req, err := t.makeLibprocessRequest(msg)
	if err != nil {
		log.Errorf("Failed to make libprocess request: %v\n", err)
		return err
	}
	return t.httpDo(ctx, req, func(resp *http.Response, err error) error {
		if err != nil {
			log.Infof("Failed to POST: %v\n", err)
			return err
		}
		defer resp.Body.Close()

		// ensure master acknowledgement.
		if (resp.StatusCode != http.StatusOK) &&
			(resp.StatusCode != http.StatusAccepted) {
			msg := fmt.Sprintf("Master %s rejected %s.  Returned status %s.", msg.UPID, msg.RequestURI(), resp.Status)
			log.Warning(msg)
			return fmt.Errorf(msg)
		}
		return nil
	})
}

func (t *HTTPTransporter) httpDo(ctx context.Context, req *http.Request, f func(*http.Response, error) error) error {
	c := make(chan error, 1)
	go func() { c <- f(t.client.Do(req)) }()
	select {
	case <-ctx.Done():
		t.tr.CancelRequest(req)
		<-c // Wait for f to return.
		return ctx.Err()
	case err := <-c:
		return err
	}
}

// Recv returns the message, one at a time.
func (t *HTTPTransporter) Recv() *Message {
	return <-t.messageQueue
}

//Inject places a message into the incoming message queue.
func (t *HTTPTransporter) Inject(ctx context.Context, msg *Message) error {
	select {
	case <-ctx.Done():
		return ctx.Err()
	case t.messageQueue <- msg:
		return nil
	}
}

// Install the request URI according to the message's name.
func (t *HTTPTransporter) Install(msgName string) {
	requestURI := fmt.Sprintf("/%s/%s", t.upid.ID, msgName)
	t.mux.HandleFunc(requestURI, t.messageHandler)
}

// Listen starts listen on UPID. If UPID is empty, the transporter
// will listen on a random port, and then fill the UPID with the
// host:port it is listening.
func (t *HTTPTransporter) Listen() error {
	host := t.upid.Host
	port := t.upid.Port
	// NOTE: Explicitly specifies IPv4 because Libprocess
	// only supports IPv4 for now.
	ln, err := net.Listen("tcp4", net.JoinHostPort(host, port))
	if err != nil {
		log.Errorf("HTTPTransporter failed to listen: %v\n", err)
		return err
	}
	// Save the host:port in case they are not specified in upid.
	host, port, _ = net.SplitHostPort(ln.Addr().String())
	t.upid.Host, t.upid.Port = host, port
	t.listener = ln
	return nil
}

// Start starts the http transporter. This will block, should be put
// in a goroutine.
func (t *HTTPTransporter) Start() error {
	// TODO(yifan): Set read/write deadline.
	if err := http.Serve(t.listener, t.mux); err != nil {
		return err
	}
	return nil
}

// Stop stops the http transporter by closing the listener.
func (t *HTTPTransporter) Stop() error {
	return t.listener.Close()
}

// UPID returns the upid of the transporter.
func (t *HTTPTransporter) UPID() *upid.UPID {
	return t.upid
}

func (t *HTTPTransporter) messageHandler(w http.ResponseWriter, r *http.Request) {
	// Verify it's a libprocess request.
	from, err := getLibprocessFrom(r)
	if err != nil {
		log.Errorf("Ignoring the request, because it's not a libprocess request: %v\n", err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	data, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.Errorf("Failed to read HTTP body: %v\n", err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	log.V(2).Infof("Receiving message from %v, length %v\n", from, len(data))
	w.WriteHeader(http.StatusAccepted)
	t.messageQueue <- &Message{
		UPID:  from,
		Name:  extractNameFromRequestURI(r.RequestURI),
		Bytes: data,
	}
}

func (t *HTTPTransporter) makeLibprocessRequest(msg *Message) (*http.Request, error) {
	hostport := net.JoinHostPort(msg.UPID.Host, msg.UPID.Port)
	targetURL := fmt.Sprintf("http://%s%s", hostport, msg.RequestURI())
	log.V(2).Infof("libproc target URL %s", targetURL)
	req, err := http.NewRequest("POST", targetURL, bytes.NewReader(msg.Bytes))
	if err != nil {
		log.Errorf("Failed to create request: %v\n", err)
		return nil, err
	}
	req.Header.Add("Libprocess-From", t.upid.String())
	req.Header.Add("Content-Type", "application/x-protobuf")
	req.Header.Add("Connection", "Keep-Alive")

	return req, nil
}

func getLibprocessFrom(r *http.Request) (*upid.UPID, error) {
	if r.Method != "POST" {
		return nil, fmt.Errorf("Not a POST request")
	}
	ua, ok := r.Header["User-Agent"]
	if ok && strings.HasPrefix(ua[0], "libprocess/") {
		// TODO(yifan): Just take the first field for now.
		return upid.Parse(ua[0][len("libprocess/"):])
	}
	lf, ok := r.Header["Libprocess-From"]
	if ok {
		// TODO(yifan): Just take the first field for now.
		return upid.Parse(lf[0])
	}
	return nil, fmt.Errorf("Cannot find 'User-Agent' or 'Libprocess-From'")
}
<commit_msg>initial implementation of retry for recoverable network errors<commit_after>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package messenger

import (
	"bytes"
	"fmt"
	"github.com/mesos/mesos-go/upid"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"

	log "github.com/golang/glog"
	"golang.org/x/net/context"
)

// HTTPTransporter implements the interfaces of the Transporter.
type HTTPTransporter struct {
	// If the host is empty("") then it will listen on localhost.
	// If the port is empty("") then it will listen on random port.
	upid         *upid.UPID
	listener     net.Listener // TODO(yifan): Change to TCPListener.
	mux          *http.ServeMux
	tr           *http.Transport
	client       *http.Client // TODO(yifan): Set read/write deadline.
	messageQueue chan *Message
}

// NewHTTPTransporter creates a new http transporter.
func NewHTTPTransporter(upid *upid.UPID) *HTTPTransporter {
	tr := &http.Transport{}
	return &HTTPTransporter{
		upid:         upid,
		messageQueue: make(chan *Message, defaultQueueSize),
		mux:          http.NewServeMux(),
		client:       &http.Client{Transport: tr},
		tr:           tr,
	}
}

// some network errors are probably recoverable, attempt to determine that here.
func isRecoverableError(err error) bool {
	if urlErr, ok := err.(*url.Error); ok {
		log.V(2).Infof("checking url.Error for recoverability")
		return urlErr.Op == "Post" && isRecoverableError(urlErr.Err)
	} else if netErr, ok := err.(*net.OpError); ok && netErr.Err != nil {
		log.V(2).Infof("checking net.OpError for recoverability: %#v", err)
		if netErr.Temporary() {
			return true
		}
		//TODO(jdef) this is pretty hackish, there's probably a better way
		return (netErr.Op == "dial" && netErr.Net == "tcp" && strings.HasSuffix(netErr.Error(), ": connection refused"))
	}
	log.V(2).Infof("unrecoverable error: %#v", err)
	return false
}

type recoverableError struct {
	Err error
}

func (e *recoverableError) Error() string {
	if e == nil {
		return ""
	}
	return e.Err.Error()
}

// Send sends the message to its specified upid.
func (t *HTTPTransporter) Send(ctx context.Context, msg *Message) (sendError error) {
	log.V(2).Infof("Sending message to %v via http\n", msg.UPID)
	req, err := t.makeLibprocessRequest(msg)
	if err != nil {
		log.Errorf("Failed to make libprocess request: %v\n", err)
		return err
	}
	duration := 1 * time.Second
	for attempt := 0; attempt < 5; attempt++ { //TODO(jdef) extract/parameterize constant
		if sendError != nil {
			duration *= 2
			log.Warningf("attempting to recover from error '%v', waiting before retry: %v", sendError, duration)
			select {
			case <-ctx.Done():
				return ctx.Err()
			case <-time.After(duration):
				// ..retry request, continue
			}
		}
		sendError = t.httpDo(ctx, req, func(resp *http.Response, err error) error {
			if err != nil {
				if isRecoverableError(err) {
					return &recoverableError{Err:err}
				}
				log.Infof("Failed to POST: %v\n", err)
				return err
			}
			defer resp.Body.Close()

			// ensure master acknowledgement.
			if (resp.StatusCode != http.StatusOK) &&
				(resp.StatusCode != http.StatusAccepted) {
				msg := fmt.Sprintf("Master %s rejected %s.  Returned status %s.",
					msg.UPID, msg.RequestURI(), resp.Status)
				log.Warning(msg)
				return fmt.Errorf(msg)
			}
			return nil
		})
		if sendError == nil {
			// success
			return
		} else if _, ok := sendError.(*recoverableError); ok {
			// recoverable, attempt backoff?
			continue
		}
		// unrecoverable
		break
	}
	if recoverable, ok := sendError.(*recoverableError); ok {
		sendError = recoverable.Err
	}
	return
}

func (t *HTTPTransporter) httpDo(ctx context.Context, req *http.Request, f func(*http.Response, error) error) error {
	c := make(chan error, 1)
	go func() { c <- f(t.client.Do(req)) }()
	select {
	case <-ctx.Done():
		t.tr.CancelRequest(req)
		<-c // Wait for f to return.
		return ctx.Err()
	case err := <-c:
		return err
	}
}

// Recv returns the message, one at a time.
func (t *HTTPTransporter) Recv() *Message {
	return <-t.messageQueue
}

//Inject places a message into the incoming message queue.
func (t *HTTPTransporter) Inject(ctx context.Context, msg *Message) error {
	select {
	case <-ctx.Done():
		return ctx.Err()
	case t.messageQueue <- msg:
		return nil
	}
}

// Install the request URI according to the message's name.
func (t *HTTPTransporter) Install(msgName string) {
	requestURI := fmt.Sprintf("/%s/%s", t.upid.ID, msgName)
	t.mux.HandleFunc(requestURI, t.messageHandler)
}

// Listen starts listen on UPID. If UPID is empty, the transporter
// will listen on a random port, and then fill the UPID with the
// host:port it is listening.
func (t *HTTPTransporter) Listen() error {
	host := t.upid.Host
	port := t.upid.Port
	// NOTE: Explicitly specifies IPv4 because Libprocess
	// only supports IPv4 for now.
	ln, err := net.Listen("tcp4", net.JoinHostPort(host, port))
	if err != nil {
		log.Errorf("HTTPTransporter failed to listen: %v\n", err)
		return err
	}
	// Save the host:port in case they are not specified in upid.
	host, port, _ = net.SplitHostPort(ln.Addr().String())
	t.upid.Host, t.upid.Port = host, port
	t.listener = ln
	return nil
}

// Start starts the http transporter. This will block, should be put
// in a goroutine.
func (t *HTTPTransporter) Start() error {
	// TODO(yifan): Set read/write deadline.
	if err := http.Serve(t.listener, t.mux); err != nil {
		return err
	}
	return nil
}

// Stop stops the http transporter by closing the listener.
func (t *HTTPTransporter) Stop() error {
	return t.listener.Close()
}

// UPID returns the upid of the transporter.
func (t *HTTPTransporter) UPID() *upid.UPID {
	return t.upid
}

func (t *HTTPTransporter) messageHandler(w http.ResponseWriter, r *http.Request) {
	// Verify it's a libprocess request.
	from, err := getLibprocessFrom(r)
	if err != nil {
		log.Errorf("Ignoring the request, because it's not a libprocess request: %v\n", err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	data, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.Errorf("Failed to read HTTP body: %v\n", err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	log.V(2).Infof("Receiving message from %v, length %v\n", from, len(data))
	w.WriteHeader(http.StatusAccepted)
	t.messageQueue <- &Message{
		UPID:  from,
		Name:  extractNameFromRequestURI(r.RequestURI),
		Bytes: data,
	}
}

func (t *HTTPTransporter) makeLibprocessRequest(msg *Message) (*http.Request, error) {
	hostport := net.JoinHostPort(msg.UPID.Host, msg.UPID.Port)
	targetURL := fmt.Sprintf("http://%s%s", hostport, msg.RequestURI())
	log.V(2).Infof("libproc target URL %s", targetURL)
	req, err := http.NewRequest("POST", targetURL, bytes.NewReader(msg.Bytes))
	if err != nil {
		log.Errorf("Failed to create request: %v\n", err)
		return nil, err
	}
	req.Header.Add("Libprocess-From", t.upid.String())
	req.Header.Add("Content-Type", "application/x-protobuf")
	req.Header.Add("Connection", "Keep-Alive")

	return req, nil
}

func getLibprocessFrom(r *http.Request) (*upid.UPID, error) {
	if r.Method != "POST" {
		return nil, fmt.Errorf("Not a POST request")
	}
	ua, ok := r.Header["User-Agent"]
	if ok && strings.HasPrefix(ua[0], "libprocess/") {
		// TODO(yifan): Just take the first field for now.
		return upid.Parse(ua[0][len("libprocess/"):])
	}
	lf, ok := r.Header["Libprocess-From"]
	if ok {
		// TODO(yifan): Just take the first field for now.
		return upid.Parse(lf[0])
	}
	return nil, fmt.Errorf("Cannot find 'User-Agent' or 'Libprocess-From'")
}
<|endoftext|>