<commit_before>package client

import (
	"errors"
	"time"

	"github.com/cpuguy83/drax/api"
	"github.com/cpuguy83/drax/rpc"
	"github.com/docker/libkv/store"
)

var errNotImplemented = errors.New("Call not implemented in current backend")

// Client is used to facilitate communications with a drax cluster
type Client struct {
	addr        string
	dialTimeout time.Duration
	streamLayer *rpc.StreamLayer
}

// New creates a new drax Client
func New(addr string, dialTimeout time.Duration, dialer rpc.DialerFn) *Client {
	sl := rpc.NewStreamLayer(nil, byte(api.ClientMessage), dialer)
	return &Client{addr, dialTimeout, sl}
}

// Get gets the k/v pair for the specified key
func (c *Client) Get(key string) (*store.KVPair, error) {
	req := &api.Request{
		Action: api.Get,
		Key:    key,
	}

	res, err := c.do(req)
	if err != nil {
		return nil, err
	}

	return kvToLibKV(res.KV), nil
}

// Put sets the key to the given value
func (c *Client) Put(key string, value []byte, options *store.WriteOptions) error {
	req := &api.Request{
		Action: api.Put,
		Key:    key,
		Value:  value,
	}
	if options != nil {
		req.TTL = options.TTL
	}

	_, err := c.do(req)
	return err
}

// Delete deletes the key
func (c *Client) Delete(key string) error {
	req := &api.Request{
		Action: api.Delete,
		Key:    key,
	}

	_, err := c.do(req)
	return err
}

// Exists checks for the existence of a key
func (c *Client) Exists(key string) (bool, error) {
	req := &api.Request{
		Action: api.Exists,
		Key:    key,
	}

	res, err := c.do(req)
	return res.Exists, err
}

// List lists the key/value pairs that have the provided key prefix
func (c *Client) List(prefix string) ([]*store.KVPair, error) {
	req := &api.Request{
		Action: api.List,
		Key:    prefix,
	}

	res, err := c.do(req)
	if err != nil {
		return nil, err
	}

	var ls []*store.KVPair
	for _, kv := range res.List {
		ls = append(ls, kvToLibKV(kv))
	}
	if len(ls) == 0 {
		return nil, store.ErrKeyNotFound
	}

	return ls, nil
}

// DeleteTree deletes the specified dir and all subdirs
func (c *Client) DeleteTree(dir string) error {
	req := &api.Request{
		Action: api.DeleteTree,
		Key:    dir,
	}

	_, err := c.do(req)
	return err
}

// Watch watches a key for changes and notifies the caller
// Watch is not implemented
func (c *Client) Watch(key string, stopCh <-chan struct{}) (<-chan *store.KVPair, error) {
	return nil, errNotImplemented
}

// WatchTree watches a dir and all subdirs for changes and notifies the caller
// WatchTree is not implemented
func (c *Client) WatchTree(dir string, stopCh <-chan struct{}) (<-chan []*store.KVPair, error) {
	return nil, errNotImplemented
}

// NewLock creates a new lock that can be used to lock access to the given key, like a sync.Mutex
func (c *Client) NewLock(key string, options *store.LockOptions) (store.Locker, error) {
	return nil, errNotImplemented
}

// AtomicPut is like `Put`, but ensures there are no changes to the key while the action is performed
// If the key is changed, this returns an error and does not perorm the requested change
func (c *Client) AtomicPut(key string, value []byte, previous *store.KVPair, options *store.WriteOptions) (bool, *store.KVPair, error) {
	req := api.Request{
		Action:   api.AtomicPut,
		Key:      key,
		Value:    value,
		Previous: libkvToKV(previous),
		TTL:      options.TTL,
	}

	res, err := c.do(&req)
	if err != nil {
		return false, nil, err
	}
	return res.Completed, kvToLibKV(res.KV), nil
}

// AtomicDelete is like `Delete`, but makes sure the key is not changed while performing the action.
func (c *Client) AtomicDelete(key string, previous *store.KVPair) (bool, error) {
	req := api.Request{
		Action:   api.AtomicDelete,
		Key:      key,
		Previous: libkvToKV(previous),
	}

	res, err := c.do(&req)
	if err != nil {
		return false, err
	}
	return res.Completed, nil
}
<commit_msg>cleanup client dial<commit_after>package client

import (
	"errors"

	"github.com/cpuguy83/drax/api"
	"github.com/cpuguy83/drax/rpc"
	"github.com/docker/libkv/store"
)

var errNotImplemented = errors.New("Call not implemented in current backend")

// Client is used to facilitate communications with a drax cluster
type Client struct {
	addr        string
	streamLayer *rpc.StreamLayer
}

// New creates a new drax Client
func New(addr string, dialer rpc.DialerFn) *Client {
	sl := rpc.NewStreamLayer(nil, byte(api.ClientMessage), dialer)
	return &Client{addr, sl}
}

// Get gets the k/v pair for the specified key
func (c *Client) Get(key string) (*store.KVPair, error) {
	req := &api.Request{
		Action: api.Get,
		Key:    key,
	}

	res, err := c.do(req)
	if err != nil {
		return nil, err
	}

	return kvToLibKV(res.KV), nil
}

// Put sets the key to the given value
func (c *Client) Put(key string, value []byte, options *store.WriteOptions) error {
	req := &api.Request{
		Action: api.Put,
		Key:    key,
		Value:  value,
	}
	if options != nil {
		req.TTL = options.TTL
	}

	_, err := c.do(req)
	return err
}

// Delete deletes the key
func (c *Client) Delete(key string) error {
	req := &api.Request{
		Action: api.Delete,
		Key:    key,
	}

	_, err := c.do(req)
	return err
}

// Exists checks for the existence of a key
func (c *Client) Exists(key string) (bool, error) {
	req := &api.Request{
		Action: api.Exists,
		Key:    key,
	}

	res, err := c.do(req)
	return res.Exists, err
}

// List lists the key/value pairs that have the provided key prefix
func (c *Client) List(prefix string) ([]*store.KVPair, error) {
	req := &api.Request{
		Action: api.List,
		Key:    prefix,
	}

	res, err := c.do(req)
	if err != nil {
		return nil, err
	}

	var ls []*store.KVPair
	for _, kv := range res.List {
		ls = append(ls, kvToLibKV(kv))
	}
	if len(ls) == 0 {
		return nil, store.ErrKeyNotFound
	}

	return ls, nil
}

// DeleteTree deletes the specified dir and all subdirs
func (c *Client) DeleteTree(dir string) error {
	req := &api.Request{
		Action: api.DeleteTree,
		Key:    dir,
	}

	_, err := c.do(req)
	return err
}

// Watch watches a key for changes and notifies the caller
// Watch is not implemented
func (c *Client) Watch(key string, stopCh <-chan struct{}) (<-chan *store.KVPair, error) {
	return nil, errNotImplemented
}

// WatchTree watches a dir and all subdirs for changes and notifies the caller
// WatchTree is not implemented
func (c *Client) WatchTree(dir string, stopCh <-chan struct{}) (<-chan []*store.KVPair, error) {
	return nil, errNotImplemented
}

// NewLock creates a new lock that can be used to lock access to the given key, like a sync.Mutex
func (c *Client) NewLock(key string, options *store.LockOptions) (store.Locker, error) {
	return nil, errNotImplemented
}

// AtomicPut is like `Put`, but ensures there are no changes to the key while the action is performed
// If the key is changed, this returns an error and does not perorm the requested change
func (c *Client) AtomicPut(key string, value []byte, previous *store.KVPair, options *store.WriteOptions) (bool, *store.KVPair, error) {
	req := api.Request{
		Action:   api.AtomicPut,
		Key:      key,
		Value:    value,
		Previous: libkvToKV(previous),
		TTL:      options.TTL,
	}

	res, err := c.do(&req)
	if err != nil {
		return false, nil, err
	}
	return res.Completed, kvToLibKV(res.KV), nil
}

// AtomicDelete is like `Delete`, but makes sure the key is not changed while performing the action.
func (c *Client) AtomicDelete(key string, previous *store.KVPair) (bool, error) {
	req := api.Request{
		Action:   api.AtomicDelete,
		Key:      key,
		Previous: libkvToKV(previous),
	}

	res, err := c.do(&req)
	if err != nil {
		return false, err
	}
	return res.Completed, nil
}
<|endoftext|>


<commit_before>//   Copyright 2017 MSolution.IO
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package costs

import (
	"context"
	"encoding/json"
	"strings"

	"github.com/olivere/elastic"

	"github.com/trackit/jsonlog"
	"github.com/trackit/trackit/errors"
)

// S3BucketCost represents the detail of the costs for a given bucket
type S3BucketCost struct {
	GbMonth       float64
	StorageCost   float64
	BandwidthCost float64
	DataIn        float64
	DataOut       float64
	RequestsCost  float64
	Requests      float64
}

type BucketsInfo = map[string]*S3BucketCost
type bucket = map[string]interface{}

// bucketCostGetter represents a function used to retrieve infos from a bucket
type bucketCostGetter func(*S3BucketCost, bucket) *S3BucketCost

// resultTypeToBucketCostGetter maps result type names to their associated
// getter function pointer
var resultTypeToBucketCostGetter = map[string]bucketCostGetter{
	"storage":      getBucketStorage,
	"requests":     getBucketRequests,
	"bandwidthIn":  getBucketBandwidthIn,
	"bandwidthOut": getBucketBandwidthOut,
}

// isValidBucket is used to filter errored requests that appears in the billing data
func isValidBucket(bucketName string) bool {
	return len(bucketName) > 0 && !strings.HasPrefix(bucketName, "[Error:")
}

// getBucketInfoByName returns the S3BucketCost associated to the bucketName
func getBucketInfoByName(buckets BucketsInfo, bucketName string) *S3BucketCost {
	if val, ok := buckets[bucketName]; ok {
		return val
	}
	buckets[bucketName] = &S3BucketCost{}
	return buckets[bucketName]
}

// getBucketStorage gets the storage informations from the bucketData
// and returns an *S3BucketCost filled with these data
func getBucketStorage(bucketInfo *S3BucketCost, bucketData bucket) *S3BucketCost {
	bucketInfo.GbMonth = bucketData["usage"].(bucket)["value"].(float64)
	bucketInfo.StorageCost = bucketData["cost"].(bucket)["value"].(float64)
	return bucketInfo
}

// getBucketRequests gets the requests informations from the bucketData
// and returns an *S3BucketCost filled with these data
func getBucketRequests(bucketInfo *S3BucketCost, bucketData bucket) *S3BucketCost {
	bucketInfo.Requests = bucketData["usage"].(bucket)["value"].(float64)
	bucketInfo.RequestsCost = bucketData["cost"].(bucket)["value"].(float64)
	return bucketInfo
}

// getBucketBandwidthIn gets the bandwidthIn informations from the bucketData
// and returns an *S3BucketCost filled with these data
func getBucketBandwidthIn(bucketInfo *S3BucketCost, bucketData bucket) *S3BucketCost {
	bucketInfo.DataIn = bucketData["usage"].(bucket)["value"].(float64)
	bucketInfo.BandwidthCost += bucketData["cost"].(bucket)["value"].(float64)
	return bucketInfo
}

// getBucketBandwidthOut gets the bandwidthOut informations from the bucketData
// and returns an *S3BucketCost filled with these data
func getBucketBandwidthOut(bucketInfo *S3BucketCost, bucketData bucket) *S3BucketCost {
	bucketInfo.DataOut = bucketData["usage"].(bucket)["value"].(float64)
	bucketInfo.BandwidthCost += bucketData["cost"].(bucket)["value"].(float64)
	return bucketInfo
}

// parseBuckets iterates through all the buckets and calls the getter function corresponding
// to the resultType
func parseBuckets(buckets BucketsInfo, parsedDocument bucket, resultType string) BucketsInfo {
	bucketsField := parsedDocument["buckets"].([]interface{})
	for _, bucketData := range bucketsField {
		bucketData := bucketData.(bucket)
		bucketName := bucketData["key"].(string)
		// The billing data can contain billings for errored requests that we do not want to see
		if isValidBucket(bucketName) {
			bucketInfo := getBucketInfoByName(buckets, bucketName)
			if resultTypePtr, ok := resultTypeToBucketCostGetter[resultType]; ok {
				resultTypePtr(bucketInfo, bucketData)
			}
		}
	}
	return buckets
}

// parseESResult parses an *elastic.SearchResult according to it's resultType
func parseESResult(ctx context.Context, buckets BucketsInfo, res *elastic.SearchResult, resultType string) (BucketsInfo, error) {
	var logger = jsonlog.LoggerFromContextOrDefault(ctx)
	var parsedDocument bucket
	err := json.Unmarshal(*res.Aggregations["buckets"], &parsedDocument)
	if err != nil {
		logger.Error("Failed to parse elasticsearch document.", err.Error())
		return buckets, errors.GetErrorMessage(ctx, err)
	}
	buckets = parseBuckets(buckets, parsedDocument, resultType)
	return buckets, nil
}

// prepareResponse parses the results from elasticsearch and returns a map of buckets with their usage informations
func prepareResponse(ctx context.Context, resStorage, resRequests, resBandwidthIn, resBandwidthOut *elastic.SearchResult) (BucketsInfo, error) {
	buckets := make(BucketsInfo)
	var err error
	var components = [...]struct {
		k  string
		sr *elastic.SearchResult
	}{
		{"storage", resStorage},
		{"requests", resRequests},
		{"bandwidthIn", resBandwidthIn},
		{"bandwidthOut", resBandwidthOut},
	}
	for _, cpn := range components {
		buckets, err = parseESResult(ctx, buckets, cpn.sr, cpn.k)
		if err != nil {
			return nil, err
		}
	}
	return buckets, nil
}
<commit_msg>fix(s3): Move bucketInfo creation to occur as narrowly as possible<commit_after>//   Copyright 2017 MSolution.IO
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package costs

import (
	"context"
	"encoding/json"
	"strings"

	"github.com/olivere/elastic"

	"github.com/trackit/jsonlog"
	"github.com/trackit/trackit/errors"
)

// S3BucketCost represents the detail of the costs for a given bucket
type S3BucketCost struct {
	GbMonth       float64
	StorageCost   float64
	BandwidthCost float64
	DataIn        float64
	DataOut       float64
	RequestsCost  float64
	Requests      float64
}

type BucketsInfo = map[string]*S3BucketCost
type bucket = map[string]interface{}

// bucketCostGetter represents a function used to retrieve infos from a bucket
type bucketCostGetter func(*S3BucketCost, bucket) *S3BucketCost

// resultTypeToBucketCostGetter maps result type names to their associated
// getter function pointer
var resultTypeToBucketCostGetter = map[string]bucketCostGetter{
	"storage":      getBucketStorage,
	"requests":     getBucketRequests,
	"bandwidthIn":  getBucketBandwidthIn,
	"bandwidthOut": getBucketBandwidthOut,
}

// isValidBucket is used to filter errored requests that appears in the billing data
func isValidBucket(bucketName string) bool {
	return len(bucketName) > 0 && !strings.HasPrefix(bucketName, "[Error:")
}

// getBucketInfoByName returns the S3BucketCost associated to the bucketName
func getBucketInfoByName(buckets BucketsInfo, bucketName string) *S3BucketCost {
	if val, ok := buckets[bucketName]; ok {
		return val
	}
	buckets[bucketName] = &S3BucketCost{}
	return buckets[bucketName]
}

// getBucketStorage gets the storage informations from the bucketData
// and returns an *S3BucketCost filled with these data
func getBucketStorage(bucketInfo *S3BucketCost, bucketData bucket) *S3BucketCost {
	bucketInfo.GbMonth = bucketData["usage"].(bucket)["value"].(float64)
	bucketInfo.StorageCost = bucketData["cost"].(bucket)["value"].(float64)
	return bucketInfo
}

// getBucketRequests gets the requests informations from the bucketData
// and returns an *S3BucketCost filled with these data
func getBucketRequests(bucketInfo *S3BucketCost, bucketData bucket) *S3BucketCost {
	bucketInfo.Requests = bucketData["usage"].(bucket)["value"].(float64)
	bucketInfo.RequestsCost = bucketData["cost"].(bucket)["value"].(float64)
	return bucketInfo
}

// getBucketBandwidthIn gets the bandwidthIn informations from the bucketData
// and returns an *S3BucketCost filled with these data
func getBucketBandwidthIn(bucketInfo *S3BucketCost, bucketData bucket) *S3BucketCost {
	bucketInfo.DataIn = bucketData["usage"].(bucket)["value"].(float64)
	bucketInfo.BandwidthCost += bucketData["cost"].(bucket)["value"].(float64)
	return bucketInfo
}

// getBucketBandwidthOut gets the bandwidthOut informations from the bucketData
// and returns an *S3BucketCost filled with these data
func getBucketBandwidthOut(bucketInfo *S3BucketCost, bucketData bucket) *S3BucketCost {
	bucketInfo.DataOut = bucketData["usage"].(bucket)["value"].(float64)
	bucketInfo.BandwidthCost += bucketData["cost"].(bucket)["value"].(float64)
	return bucketInfo
}

// parseBuckets iterates through all the buckets and calls the getter function corresponding
// to the resultType
func parseBuckets(buckets BucketsInfo, parsedDocument bucket, resultType string) BucketsInfo {
	bucketsField := parsedDocument["buckets"].([]interface{})
	for _, bucketData := range bucketsField {
		bucketData := bucketData.(bucket)
		bucketName := bucketData["key"].(string)
		// The billing data can contain billings for errored requests that we do not want to see
		if isValidBucket(bucketName) {
			if resultTypePtr, ok := resultTypeToBucketCostGetter[resultType]; ok {
				resultTypePtr(getBucketInfoByName(buckets, bucketName), bucketData)
			}
		}
	}
	return buckets
}

// parseESResult parses an *elastic.SearchResult according to it's resultType
func parseESResult(ctx context.Context, buckets BucketsInfo, res *elastic.SearchResult, resultType string) (BucketsInfo, error) {
	var logger = jsonlog.LoggerFromContextOrDefault(ctx)
	var parsedDocument bucket
	err := json.Unmarshal(*res.Aggregations["buckets"], &parsedDocument)
	if err != nil {
		logger.Error("Failed to parse elasticsearch document.", err.Error())
		return buckets, errors.GetErrorMessage(ctx, err)
	}
	buckets = parseBuckets(buckets, parsedDocument, resultType)
	return buckets, nil
}

// prepareResponse parses the results from elasticsearch and returns a map of buckets with their usage informations
func prepareResponse(ctx context.Context, resStorage, resRequests, resBandwidthIn, resBandwidthOut *elastic.SearchResult) (BucketsInfo, error) {
	buckets := make(BucketsInfo)
	var err error
	var components = [...]struct {
		k  string
		sr *elastic.SearchResult
	}{
		{"storage", resStorage},
		{"requests", resRequests},
		{"bandwidthIn", resBandwidthIn},
		{"bandwidthOut", resBandwidthOut},
	}
	for _, cpn := range components {
		buckets, err = parseESResult(ctx, buckets, cpn.sr, cpn.k)
		if err != nil {
			return nil, err
		}
	}
	return buckets, nil
}
<|endoftext|>


<commit_before>package paillier

import (
	"errors"
	"fmt"
	"math/big"
	"reflect"
	"testing"
	"time"
)

func TestAsyncGenerator(t *testing.T) {
	concurrencyLevel := 4

	var tests = map[string]struct {
		bitLen        int
		timeout       time.Duration
		expectedError error
	}{
		"primes successfully generated": {
			bitLen:        512,
			timeout:       60 * time.Second,
			expectedError: nil,
		},
		"generator timed out": {
			bitLen:        8192,
			timeout:       1 * time.Second,
			expectedError: errors.New("generator timed out after 1s"),
		},
		"bit length is 5": {
			bitLen:        5,
			timeout:       1 * time.Second,
			expectedError: errors.New("safe prime size must be at least 6 bits"),
		},
		"bit length is 6": {
			bitLen:        6,
			timeout:       60 * time.Second,
			expectedError: nil,
		},
	}

	for testName, test := range tests {
		t.Run(testName, func(t *testing.T) {
			p, q, err := GenerateSafePrime(
				test.bitLen,
				concurrencyLevel,
				test.timeout,
			)

			if test.expectedError != nil {
				if !reflect.DeepEqual(test.expectedError, err) {
					t.Fatalf(
						"Unexpected error\nActual: %v\nExpected: %v",
						err,
						test.expectedError,
					)
				}
			} else {
				if err != nil {
					t.Fatal(err)
				}

				if !p.ProbablyPrime(20) {
					t.Errorf("p is not prime; p = %v", p)
				}

				if !q.ProbablyPrime(20) {
					t.Errorf("q is not prime; q = %v", q)
				}

				// p = 2q + 1 ?
				expectedP := new(big.Int)
				expectedP.Mul(big.NewInt(2), q)
				expectedP.Add(expectedP, big.NewInt(1))

				if expectedP.Cmp(p) != 0 {
					t.Errorf("2q + 1 != p")
				}

				if len(p.Bytes()) != test.bitLen {
					fmt.Errorf(
						"Unexpected p bit length\nActual: %v\nExpected: %v",
						len(p.Bytes()),
						test.bitLen,
					)
				}

				if len(q.Bytes()) != test.bitLen-1 {
					fmt.Errorf(
						"Unexpected q bit length\nActual: %v\nExpected: %v",
						len(q.Bytes()),
						test.bitLen-1,
					)
				}
			}
		})
	}
}
<commit_msg>Fixed bit lenght assertions for safe prime generator tests<commit_after>package paillier

import (
	"errors"
	"math/big"
	"reflect"
	"testing"
	"time"
)

func TestAsyncGenerator(t *testing.T) {
	concurrencyLevel := 4

	var tests = map[string]struct {
		bitLen        int
		timeout       time.Duration
		expectedError error
	}{
		"primes successfully generated": {
			bitLen:        512,
			timeout:       60 * time.Second,
			expectedError: nil,
		},
		"generator timed out": {
			bitLen:        8192,
			timeout:       1 * time.Second,
			expectedError: errors.New("generator timed out after 1s"),
		},
		"bit length is 5": {
			bitLen:        5,
			timeout:       1 * time.Second,
			expectedError: errors.New("safe prime size must be at least 6 bits"),
		},
		"bit length is 6": {
			bitLen:        6,
			timeout:       60 * time.Second,
			expectedError: nil,
		},
	}

	for testName, test := range tests {
		t.Run(testName, func(t *testing.T) {
			p, q, err := GenerateSafePrime(
				test.bitLen,
				concurrencyLevel,
				test.timeout,
			)

			if test.expectedError != nil {
				if !reflect.DeepEqual(test.expectedError, err) {
					t.Fatalf(
						"Unexpected error\nActual: %v\nExpected: %v",
						err,
						test.expectedError,
					)
				}
			} else {
				if err != nil {
					t.Fatal(err)
				}

				if !p.ProbablyPrime(20) {
					t.Errorf("p is not prime; p = %v", p)
				}

				if !q.ProbablyPrime(20) {
					t.Errorf("q is not prime; q = %v", q)
				}

				// p = 2q + 1 ?
				expectedP := new(big.Int)
				expectedP.Mul(big.NewInt(2), q)
				expectedP.Add(expectedP, big.NewInt(1))

				if expectedP.Cmp(p) != 0 {
					t.Errorf("2q + 1 != p")
				}

				if p.BitLen() != test.bitLen {
					t.Fatalf(
						"Unexpected p bit length\nActual: %v\nExpected: %v",
						p.BitLen(),
						test.bitLen,
					)
				}

				if q.BitLen() != test.bitLen-1 {
					t.Fatalf(
						"Unexpected q bit length\nActual: %v\nExpected: %v",
						q.BitLen(),
						test.bitLen-1,
					)
				}
			}
		})
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Factom Foundation
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package main

const usage = `
walletmgr [subcommand]

    exportseeds             Export seeds for deterministic wallet re-creation
    help                    Print this help message
`

// man returns an usage error string for the specified sub command.
func man(s string) {
	m := map[string]string{
		"exportseeds":    "walletmgr exportseeds",
		"help":           usage,
		"default":        "More Help can be found by typing:\n\n  walletmgr help",
	}

	if m[s] != "" {
		errorln(m[s])
		return
	}

	errorln(m["default"])
}
<commit_msg>changed to 1 seed per wallet after removing setup cmd<commit_after>// Copyright 2016 Factom Foundation
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package main

const usage = `
walletmgr [subcommand]

    exportseed              Export seed for future deterministic wallet re-creation
    help                    Print this help message
`

// man returns an usage error string for the specified sub command.
func man(s string) {
	m := map[string]string{
		"exportseed":    "walletmgr exportseed",
		"help":           usage,
		"default":        "More Help can be found by typing:\n\n  walletmgr help",
	}

	if m[s] != "" {
		errorln(m[s])
		return
	}

	errorln(m["default"])
}
<|endoftext|>


<commit_before>// +build freebsd
// tun_freebsd.go -- tun interface with cgo for linux / bsd
//

package samtun

/*

#include <string.h>
#include <unistd.h>
#include <fcntl.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <arpa/inet.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <net/if.h>
#include <net/if_tun.h>
#include <stdio.h>

int tundev_open(char * ifname) {
  if (strlen(ifname) > IFNAMSIZ) {
    return -1;
  }
  char name[IFNAMSIZ];
  sprintf(name, "/dev/%s", ifname);
  int fd = open(name, O_RDWR);
  if (fd > 0) {
    int i = 0;
    ioctl(fd, TUNSLMODE, &i);
    ioctl(fd, TUNSIFHEAD, &i);
  }
  return fd;
}

int tundev_up(char * ifname, char * addr, char * dstaddr, int mtu) {

  struct ifreq ifr;
  memset(&ifr, 0, sizeof(struct ifreq));
  strncpy(ifr.ifr_name, ifname, IFNAMSIZ);
  int fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP);
  if ( fd > 0 ) {
    if ( ioctl(fd, SIOCGIFINDEX, (void*) &ifr) < 0 ) {
      perror("SIOCGIFINDEX");
      close(fd);
      return -1;
    }
    int idx = ifr.ifr_index;
    ifr.ifr_mtu = mtu;
    if ( ioctl(fd, SIOCSIFMTU, (void*) &ifr) < 0) {
      close(fd);
      perror("SIOCSIFMTU");
      return -1;
    }

    if ( ioctl(fd, SIOCGIFFLAGS, (void*)&ifr) < 0 ) {
      close(fd);
      perror("SIOCGIFFLAGS");
      return -1;
    }
    ifr.ifr_flags |= IFF_POINTOPOINT;
    if ( ioctl(fd, SIOCSIFFLAGS, (void*)&ifr) < 0 ) {
      close(fd);
      return -1;
    }

    struct sockaddr_in dst;
    memset(&dst, 0, sizeof(struct sockaddr_in));
    dst.sin_family = AF_INET;
    if ( ! inet_aton(dstaddr, &dst.sin_addr) ) {
      printf("invalid dstaddr %s\n", dstaddr);
      close(fd);
      return -1;
    }

    struct sockaddr_in src;
    memset(&src, 0, sizeof(struct sockaddr_in));
    src.sin_family = AF_INET;
    if ( ! inet_aton(addr, &src.sin_addr) ) {
      printf("invalid srcaddr %s\n", addr);
      close(fd);
      return -1;
    }
    memset(&ifr, 0, sizeof(struct ifreq));
    ifr.if_index = idx;
    memcpy(&ifr.ifr_addr, &src, sizeof(struct sockaddr_in));
    if ( ioctl(fd, SIOCSIFADDR, (void*)&ifr) < 0 ) {
      close(fd);
      perror("SIOCSIFADDR");
     return -1;
    }

    memcpy(&ifr.ifr_dstaddr, &dst, sizeof(struct sockaddr_in));
    if ( ioctl(fd, SIOCSIFDSTADDR, (void*)&ifr) < 0 ) {
      close(fd);
      perror("SIOCSIFADDR");
     return -1;
    }

    if ( ioctl(fd, SIOCGIFFLAGS, (void*)&ifr) < 0 ) {
      close(fd);
      perror("SIOCGIFFLAGS");
      return -1;
    }

    ifr.ifr_flags |= IFF_UP;
    if ( ioctl(fd, SIOCSIFFLAGS, (void*)&ifr) < 0 ) {
      close(fd);
      return -1;
    }

    close(fd);
    return 0;
  }
  return -1;
}

void tundev_close(int fd) {
  close(fd);
}

*/
import "C"

import (
  "errors"
)

type tunDev C.int

func newTun(ifname, addr, dstaddr string, mtu int) (t tunDev, err error) {
  fd := C.tundev_open(C.CString(ifname))

  if fd == -1 {
    err = errors.New("cannot open tun interface")
  } else {
    if C.tundev_up(C.CString(ifname), C.CString(addr), C.CString(dstaddr), C.int(mtu)) < C.int(0) {
      err = errors.New("cannot put up interface")
    } else {
      return tunDev(fd), nil
    }
  }
  return -1, err
}

func (d tunDev) Close() {
  C.tundev_close(C.int(d))
}
<commit_msg>try fixing freebsd<commit_after>// +build freebsd
// tun_freebsd.go -- tun interface with cgo for linux / bsd
//

package samtun

/*

#include <string.h>
#include <unistd.h>
#include <fcntl.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <arpa/inet.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <net/if.h>
#include <net/if_tun.h>
#include <stdio.h>

int tundev_open(char * ifname) {
  if (strlen(ifname) > IFNAMSIZ) {
    return -1;
  }
  char name[IFNAMSIZ];
  sprintf(name, "/dev/%s", ifname);
  int fd = open(name, O_RDWR);
  if (fd > 0) {
    int i = 0;
    ioctl(fd, TUNSLMODE, &i);
    ioctl(fd, TUNSIFHEAD, &i);
  }
  return fd;
}

int tundev_up(char * ifname, char * addr, char * dstaddr, int mtu) {

  struct ifreq ifr;
  memset(&ifr, 0, sizeof(struct ifreq));
  strncpy(ifr.ifr_name, ifname, IFNAMSIZ);
  int fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP);
  if ( fd > 0 ) {
    if ( ioctl(fd, SIOCGIFINDEX, (void*) &ifr) < 0 ) {
      perror("SIOCGIFINDEX");
      close(fd);
      return -1;
    }
    int idx = ifr.ifr_index;
    ifr.ifr_mtu = mtu;
    if ( ioctl(fd, SIOCSIFMTU, (void*) &ifr) < 0) {
      close(fd);
      perror("SIOCSIFMTU");
      return -1;
    }

    if ( ioctl(fd, SIOCGIFFLAGS, (void*)&ifr) < 0 ) {
      close(fd);
      perror("SIOCGIFFLAGS");
      return -1;
    }
    ifr.ifr_flags |= IFF_POINTOPOINT;
    if ( ioctl(fd, SIOCSIFFLAGS, (void*)&ifr) < 0 ) {
      close(fd);
      return -1;
    }

    struct sockaddr_in dst;
    memset(&dst, 0, sizeof(struct sockaddr_in));
    dst.sin_family = AF_INET;
    if ( ! inet_aton(dstaddr, &dst.sin_addr) ) {
      printf("invalid dstaddr %s\n", dstaddr);
      close(fd);
      return -1;
    }

    struct sockaddr_in src;
    memset(&src, 0, sizeof(struct sockaddr_in));
    src.sin_family = AF_INET;
    if ( ! inet_aton(addr, &src.sin_addr) ) {
      printf("invalid srcaddr %s\n", addr);
      close(fd);
      return -1;
    }
    memset(&ifr, 0, sizeof(struct ifreq));
    ifr.ifr_index = idx;
    memcpy(&ifr.ifr_addr, &src, sizeof(struct sockaddr_in));
    if ( ioctl(fd, SIOCSIFADDR, (void*)&ifr) < 0 ) {
      close(fd);
      perror("SIOCSIFADDR");
     return -1;
    }

    memcpy(&ifr.ifr_dstaddr, &dst, sizeof(struct sockaddr_in));
    if ( ioctl(fd, SIOCSIFDSTADDR, (void*)&ifr) < 0 ) {
      close(fd);
      perror("SIOCSIFADDR");
     return -1;
    }

    if ( ioctl(fd, SIOCGIFFLAGS, (void*)&ifr) < 0 ) {
      close(fd);
      perror("SIOCGIFFLAGS");
      return -1;
    }

    ifr.ifr_flags |= IFF_UP;
    if ( ioctl(fd, SIOCSIFFLAGS, (void*)&ifr) < 0 ) {
      close(fd);
      return -1;
    }

    close(fd);
    return 0;
  }
  return -1;
}

void tundev_close(int fd) {
  close(fd);
}

*/
import "C"

import (
  "errors"
)

type tunDev C.int

func newTun(ifname, addr, dstaddr string, mtu int) (t tunDev, err error) {
  fd := C.tundev_open(C.CString(ifname))

  if fd == -1 {
    err = errors.New("cannot open tun interface")
  } else {
    if C.tundev_up(C.CString(ifname), C.CString(addr), C.CString(dstaddr), C.int(mtu)) < C.int(0) {
      err = errors.New("cannot put up interface")
    } else {
      return tunDev(fd), nil
    }
  }
  return -1, err
}

func (d tunDev) Close() {
  C.tundev_close(C.int(d))
}
<|endoftext|>


<commit_before>package lib

import (
	"github.com/subutai-io/base/agent/config"
	"github.com/subutai-io/base/agent/lib/container"
	"github.com/subutai-io/base/agent/lib/net"
	"github.com/subutai-io/base/agent/lib/template"
)

func LxcDestroy(name string) {
	net.DelIface(container.GetConfigItem(config.Agent.LxcPrefix+name+"/config", "lxc.network.veth.pair"))
	container.Destroy(name)

	if name == "management" {
		template.MngStop()
		template.MngDel()
	}
}
<commit_msg>Destroy argument check added<commit_after>package lib

import (
	"github.com/subutai-io/base/agent/config"
	"github.com/subutai-io/base/agent/lib/container"
	"github.com/subutai-io/base/agent/lib/net"
	"github.com/subutai-io/base/agent/lib/template"
	"github.com/subutai-io/base/agent/log"
)

func LxcDestroy(name string) {
	if len(name) == 0 {
		log.Error("Please specify container or template name")
	}
	net.DelIface(container.GetConfigItem(config.Agent.LxcPrefix+name+"/config", "lxc.network.veth.pair"))
	container.Destroy(name)

	if name == "management" {
		template.MngStop()
		template.MngDel()
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 Dimitris Dinodimos. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package decimal

import "database/sql/driver"

// NullDec represents a decimal that may be null.
type NullDec struct {
	dec   Dec
	valid bool
}

// Null returns true if the value of d is null
func (d NullDec) Null() bool {
	return !d.valid
}

// SetNull sets null to d and returns d.
func (d *NullDec) SetNull() *NullDec {
	d.valid = false
	return d
}

// String returns the value of d.
// Returns an empty string if the value is null.
func (d NullDec) String() string {
	if d.Null() {
		return ""
	}
	return d.dec.String()
}

// Bytes returns the value of d.
// Returns nil if the value is null.
func (d NullDec) Bytes() []byte {
	if d.Null() {
		return nil
	}
	return d.dec.Bytes()
}

// Scan implements the database Scanner interface.
func (d *NullDec) Scan(value interface{}) error {
	if value == nil {
		d.valid = false
		return nil
	}
	err := d.dec.Scan(value)
	if err != nil {
		return err
	}
	d.valid = true
	return nil
}

// Value implements the database driver Valuer interface.
func (d NullDec) Value() (driver.Value, error) {
	if d.Null() {
		return nil, nil
	}
	return d.dec.Value()
}

// SetDec sets d to x and returns d.
func (d *NullDec) SetDec(x *Dec) *NullDec {
	d.valid = true
	d.dec.Set(x)
	return d
}

// Set sets d to x and returns d.
func (d *NullDec) Set(x *NullDec) *NullDec {
	d.valid = x.valid
	d.dec.Set(&x.dec)
	return d
}

// SetFloat64 sets d to the value of f
func (d *NullDec) SetFloat64(f float64) error {
	err := d.dec.SetFloat64(f)
	if err != nil {
		return err
	}
	d.valid = true
	return nil
}

// SetInt64 sets d to the value of i and returns d.
func (d *NullDec) SetInt64(i int64) *NullDec {
	d.valid = true
	d.dec.SetInt64(i)
	return d
}

// SetInt128 sets d to the value of x and returns d.
func (d *NullDec) SetInt128(x *Int128) *NullDec {
	d.valid = true
	d.dec.SetInt128(x)
	return d
}

// SetString sets d to the value of s
func (d *NullDec) SetString(s string) error {
	if s == "" {
		d.SetNull()
		return nil
	}
	err := d.dec.SetString(s)
	if err != nil {
		return err
	}
	d.valid = true
	return nil
}

// SetBytes sets d to the value of buf
func (d *NullDec) SetBytes(buf []byte) error {
	if buf == nil {
		d.SetNull()
		return nil
	}
	err := d.dec.SetBytes(buf)
	if err != nil {
		return err
	}
	d.valid = true
	return nil
}

// Sign returns:
//
//      -1 if d <  0
//       0 if d == 0
//      +1 if d >  0
//
func (d NullDec) Sign() int {
	if d.Null() {
		return 0
	}
	return d.dec.Sign()
}

// Cmp compares x and y and returns:
//
//   -1 if x <  y
//    0 if x == y
//   +1 if x >  y
//
func (x NullDec) Cmp(y *NullDec) int {
	if x.Null() || y.Null() {
		return 0
	}
	return x.dec.Cmp(&y.dec)
}

// Abs sets d to |x| (the absolute value of x) and returns d.
func (d *NullDec) Abs(x *NullDec) *NullDec {
	if x.Null() {
		d.SetNull()
	} else {
		d.dec.Abs(&x.dec)
		d.valid = true
	}
	return d
}

// Neg sets d to -x and returns d.
func (d *NullDec) Neg(x *NullDec) *NullDec {
	if x.Null() {
		d.SetNull()
	} else {
		d.dec.Neg(&x.dec)
		d.valid = true
	}
	return d
}

// Add sets d to the sum x+y and returns d.
// The scale of d is the larger of the scales of the two operands.
func (d *NullDec) Add(x, y *NullDec) *NullDec {
	if x.Null() || y.Null() {
		d.SetNull()
	} else {
		d.dec.Add(&x.dec, &y.dec)
		d.valid = true
	}
	return d
}

// Sub sets d to the difference x-y and returns d.
// The scale of d is the larger of the scales of the two operands.
func (d *NullDec) Sub(x, y *NullDec) *NullDec {
	if x.Null() || y.Null() {
		d.SetNull()
	} else {
		d.dec.Sub(&x.dec, &y.dec)
		d.valid = true
	}
	return d
}

// Mul sets d to the product x*y and returns d.
// The scale of d is the sum of the scales of the two operands.
func (d *NullDec) Mul(x, y *NullDec) *NullDec {
	if x.Null() || y.Null() {
		d.SetNull()
	} else {
		d.dec.Mul(&x.dec, &y.dec)
		d.valid = true
	}
	return d
}

// Div sets d to the rounded quotient x/y and returns d.
// If y is zero panics with Division by zero.
// The resulting value is rounded half up to the given scale.
func (d *NullDec) Div(x, y *NullDec, scale uint8) *NullDec {
	if x.Null() || y.Null() {
		d.SetNull()
	} else {
		d.dec.Div(&x.dec, &y.dec, scale)
		d.valid = true
	}
	return d
}

// Round d half up to the given scale and returns d
func (d *NullDec) Round(scale uint8) *NullDec {
	if !d.Null() {
		d.dec.Round(scale)
	}
	return d
}

// Power sets d = x^n and returns d
func (d *NullDec) Power(x *NullDec, n int) *NullDec {
	if x.Null() {
		d.SetNull()
	} else {
		d.dec.Power(&x.dec, n)
		d.valid = true
	}
	return d
}

// Float64 returns the nearest float64 representation of d.
func (d NullDec) Float64() float64 {
	if d.Null() {
		return 0
	}
	return d.dec.Float64()
}
<commit_msg>return Dec from NullDec<commit_after>// Copyright 2014 Dimitris Dinodimos. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package decimal

import "database/sql/driver"

// NullDec represents a decimal that may be null.
type NullDec struct {
	dec   Dec
	valid bool
}

// Null returns true if the value of d is null
func (d NullDec) Null() bool {
	return !d.valid
}

// SetNull sets null to d and returns d.
func (d *NullDec) SetNull() *NullDec {
	d.valid = false
	return d
}

// String returns the value of d.
// Returns an empty string if the value is null.
func (d NullDec) String() string {
	if d.Null() {
		return ""
	}
	return d.dec.String()
}

// Bytes returns the value of d.
// Returns nil if the value is null.
func (d NullDec) Bytes() []byte {
	if d.Null() {
		return nil
	}
	return d.dec.Bytes()
}

// Scan implements the database Scanner interface.
func (d *NullDec) Scan(value interface{}) error {
	if value == nil {
		d.valid = false
		return nil
	}
	err := d.dec.Scan(value)
	if err != nil {
		return err
	}
	d.valid = true
	return nil
}

// Value implements the database driver Valuer interface.
func (d NullDec) Value() (driver.Value, error) {
	if d.Null() {
		return nil, nil
	}
	return d.dec.Value()
}

// SetDec sets d to x and returns d.
func (d *NullDec) SetDec(x *Dec) *NullDec {
	d.valid = true
	d.dec.Set(x)
	return d
}

// Dec returns d.
func (d *NullDec) Dec() *Dec {
	if d.Null() {
		return nil
	}
	return &d.dec
}

// Set sets d to x and returns d.
func (d *NullDec) Set(x *NullDec) *NullDec {
	d.valid = x.valid
	d.dec.Set(&x.dec)
	return d
}

// SetFloat64 sets d to the value of f
func (d *NullDec) SetFloat64(f float64) error {
	err := d.dec.SetFloat64(f)
	if err != nil {
		return err
	}
	d.valid = true
	return nil
}

// SetInt64 sets d to the value of i and returns d.
func (d *NullDec) SetInt64(i int64) *NullDec {
	d.valid = true
	d.dec.SetInt64(i)
	return d
}

// SetInt128 sets d to the value of x and returns d.
func (d *NullDec) SetInt128(x *Int128) *NullDec {
	d.valid = true
	d.dec.SetInt128(x)
	return d
}

// SetString sets d to the value of s
func (d *NullDec) SetString(s string) error {
	if s == "" {
		d.SetNull()
		return nil
	}
	err := d.dec.SetString(s)
	if err != nil {
		return err
	}
	d.valid = true
	return nil
}

// SetBytes sets d to the value of buf
func (d *NullDec) SetBytes(buf []byte) error {
	if buf == nil {
		d.SetNull()
		return nil
	}
	err := d.dec.SetBytes(buf)
	if err != nil {
		return err
	}
	d.valid = true
	return nil
}

// Sign returns:
//
//      -1 if d <  0
//       0 if d == 0
//      +1 if d >  0
//
func (d NullDec) Sign() int {
	if d.Null() {
		return 0
	}
	return d.dec.Sign()
}

// Cmp compares x and y and returns:
//
//   -1 if x <  y
//    0 if x == y
//   +1 if x >  y
//
func (x NullDec) Cmp(y *NullDec) int {
	if x.Null() || y.Null() {
		return 0
	}
	return x.dec.Cmp(&y.dec)
}

// Abs sets d to |x| (the absolute value of x) and returns d.
func (d *NullDec) Abs(x *NullDec) *NullDec {
	if x.Null() {
		d.SetNull()
	} else {
		d.dec.Abs(&x.dec)
		d.valid = true
	}
	return d
}

// Neg sets d to -x and returns d.
func (d *NullDec) Neg(x *NullDec) *NullDec {
	if x.Null() {
		d.SetNull()
	} else {
		d.dec.Neg(&x.dec)
		d.valid = true
	}
	return d
}

// Add sets d to the sum x+y and returns d.
// The scale of d is the larger of the scales of the two operands.
func (d *NullDec) Add(x, y *NullDec) *NullDec {
	if x.Null() || y.Null() {
		d.SetNull()
	} else {
		d.dec.Add(&x.dec, &y.dec)
		d.valid = true
	}
	return d
}

// Sub sets d to the difference x-y and returns d.
// The scale of d is the larger of the scales of the two operands.
func (d *NullDec) Sub(x, y *NullDec) *NullDec {
	if x.Null() || y.Null() {
		d.SetNull()
	} else {
		d.dec.Sub(&x.dec, &y.dec)
		d.valid = true
	}
	return d
}

// Mul sets d to the product x*y and returns d.
// The scale of d is the sum of the scales of the two operands.
func (d *NullDec) Mul(x, y *NullDec) *NullDec {
	if x.Null() || y.Null() {
		d.SetNull()
	} else {
		d.dec.Mul(&x.dec, &y.dec)
		d.valid = true
	}
	return d
}

// Div sets d to the rounded quotient x/y and returns d.
// If y is zero panics with Division by zero.
// The resulting value is rounded half up to the given scale.
func (d *NullDec) Div(x, y *NullDec, scale uint8) *NullDec {
	if x.Null() || y.Null() {
		d.SetNull()
	} else {
		d.dec.Div(&x.dec, &y.dec, scale)
		d.valid = true
	}
	return d
}

// Round d half up to the given scale and returns d
func (d *NullDec) Round(scale uint8) *NullDec {
	if !d.Null() {
		d.dec.Round(scale)
	}
	return d
}

// Power sets d = x^n and returns d
func (d *NullDec) Power(x *NullDec, n int) *NullDec {
	if x.Null() {
		d.SetNull()
	} else {
		d.dec.Power(&x.dec, n)
		d.valid = true
	}
	return d
}

// Float64 returns the nearest float64 representation of d.
func (d NullDec) Float64() float64 {
	if d.Null() {
		return 0
	}
	return d.dec.Float64()
}
<|endoftext|>


<commit_before>package main

import "github.com/boltdb/bolt"

type Request struct {
	Service string
	Method  string
	Path    string
	Data    string
}

func (r Request) ServiceBucket(tx *bolt.Tx) (*bolt.Bucket, error) {
	sb, err := tx.CreateBucketIfNotExists([]byte("services"))
	if err != nil {
		return nil, err
	}

	b, err := sb.CreateBucketIfNotExists([]byte(r.Service))
	if err != nil {
		return nil, err
	}

	return b, nil
}

func (r Request) PathBucket(tx *bolt.Tx) (*bolt.Bucket, error) {
	s, err := r.ServiceBucket(tx)
	if err != nil {
		return nil, err
	}

	pb, err := s.CreateBucketIfNotExists([]byte("paths"))
	if err != nil {
		return nil, err
	}

	b, err := pb.CreateBucketIfNotExists([]byte(r.Path))
	if err != nil {
		return nil, err
	}

	return b, nil
}

func (r Request) MethodBucket(tx *bolt.Tx) (*bolt.Bucket, error) {
	s, err := r.PathBucket(tx)
	if err != nil {
		return nil, err
	}

	b, err := s.CreateBucketIfNotExists([]byte(r.Method))
	if err != nil {
		return nil, err
	}

	return b, nil
}
<commit_msg>Get service from db if not set by flag<commit_after>package main

import "github.com/boltdb/bolt"

type Request struct {
	Service string
	Method  string
	Path    string
	Data    string
}

func (r Request) ServiceBucket(tx *bolt.Tx) (*bolt.Bucket, error) {
	if r.Service == "" {
		info := tx.Bucket([]byte("info"))
		current := info.Get([]byte("current"))
		r.Service = string(current)
	}

	sb, err := tx.CreateBucketIfNotExists([]byte("services"))
	if err != nil {
		return nil, err
	}

	b, err := sb.CreateBucketIfNotExists([]byte(r.Service))
	if err != nil {
		return nil, err
	}

	return b, nil
}

func (r Request) PathBucket(tx *bolt.Tx) (*bolt.Bucket, error) {
	s, err := r.ServiceBucket(tx)
	if err != nil {
		return nil, err
	}

	pb, err := s.CreateBucketIfNotExists([]byte("paths"))
	if err != nil {
		return nil, err
	}

	b, err := pb.CreateBucketIfNotExists([]byte(r.Path))
	if err != nil {
		return nil, err
	}

	return b, nil
}

func (r Request) MethodBucket(tx *bolt.Tx) (*bolt.Bucket, error) {
	s, err := r.PathBucket(tx)
	if err != nil {
		return nil, err
	}

	b, err := s.CreateBucketIfNotExists([]byte(r.Method))
	if err != nil {
		return nil, err
	}

	return b, nil
}
<|endoftext|>


<commit_before>package main

import (
	cryptoRand "crypto/rand"
	"errors"
	mathRand "math/rand"
	"net"
	"strconv"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
	"golang.org/x/crypto/ssh"
)

type globalRequestPayload interface {
	reply(context *connContext) []byte
	logEntry(context *connContext) logEntry
}

type globalRequestPayloadParser func(data []byte) (globalRequestPayload, error)

type tcpipRequest struct {
	Address string
	Port    uint32
}

func (request tcpipRequest) reply(context *connContext) []byte {
	if request.Port != 0 {
		return nil
	}
	return ssh.Marshal(struct{ port uint32 }{uint32(mathRand.Intn(65536-1024) + 1024)})
}
func (request tcpipRequest) logEntry(context *connContext) logEntry {
	return tcpipForwardLog{
		Address: net.JoinHostPort(request.Address, strconv.Itoa(int(request.Port))),
	}
}

type cancelTCPIPRequest struct {
	Address string
	Port    uint32
}

func (request cancelTCPIPRequest) reply(context *connContext) []byte {
	return nil
}
func (request cancelTCPIPRequest) logEntry(context *connContext) logEntry {
	return cancelTCPIPForwardLog{
		Address: net.JoinHostPort(request.Address, strconv.Itoa(int(request.Port))),
	}
}

type noMoreSessionsRequest struct {
}

func (request noMoreSessionsRequest) reply(context *connContext) []byte {
	return nil
}
func (request noMoreSessionsRequest) logEntry(context *connContext) logEntry {
	return noMoreSessionsLog{}
}

type hostKeysProveRequest struct {
	hostKeyIndices []int
}

func (request hostKeysProveRequest) reply(context *connContext) []byte {
	signatures := make([][]byte, len(request.hostKeyIndices))
	for i, index := range request.hostKeyIndices {
		signature, err := context.cfg.parsedHostKeys[index].Sign(cryptoRand.Reader, ssh.Marshal(struct {
			requestType, sessionID, hostKey string
		}{
			"hostkeys-prove-00@openssh.com",
			string(context.SessionID()),
			string(context.cfg.parsedHostKeys[index].PublicKey().Marshal()),
		}))
		if err != nil {
			warningLogger.Printf("Failed to sign host key: %v", err)
			return nil
		}
		signatures[i] = ssh.Marshal(signature)
	}
	return marshalBytes(signatures)
}
func (request hostKeysProveRequest) logEntry(context *connContext) logEntry {
	hostKeyFiles := make([]string, len(request.hostKeyIndices))
	for i, index := range request.hostKeyIndices {
		hostKeyFiles[i] = context.cfg.Server.HostKeys[index]
	}
	return hostKeysProveLog{
		HostKeyFiles: hostKeyFiles,
	}
}

var globalRequestPayloads = map[string]globalRequestPayloadParser{
	"tcpip-forward": func(data []byte) (globalRequestPayload, error) {
		payload := &tcpipRequest{}
		if err := ssh.Unmarshal(data, payload); err != nil {
			return nil, err
		}
		return payload, nil
	},
	"cancel-tcpip-forward": func(data []byte) (globalRequestPayload, error) {
		payload := &cancelTCPIPRequest{}
		if err := ssh.Unmarshal(data, payload); err != nil {
			return nil, err
		}
		return payload, nil
	},
	"no-more-sessions@openssh.com": func(data []byte) (globalRequestPayload, error) {
		if len(data) != 0 {
			return nil, errors.New("invalid request payload")
		}
		return &noMoreSessionsRequest{}, nil
	},
}

var (
	globalRequestsMetric = promauto.NewCounterVec(prometheus.CounterOpts{
		Name: "sshesame_global_requests_total",
		Help: "Total number of global requests",
	}, []string{"type"})
)

func handleGlobalRequest(request *ssh.Request, context *connContext) error {
	parser := globalRequestPayloads[request.Type]
	if parser == nil {
		globalRequestsMetric.WithLabelValues("unknown").Inc()
		warningLogger.Printf("Unsupported global request type %v", request.Type)
		if request.WantReply {
			if err := request.Reply(false, nil); err != nil {
				return err
			}
		}
		return nil
	}
	globalRequestsMetric.WithLabelValues(request.Type).Inc()
	payload, err := parser(request.Payload)
	if err != nil {
		return err
	}
	switch payload.(type) {
	case *noMoreSessionsRequest:
		context.noMoreSessions = true
	}
	if request.WantReply {
		if err := request.Reply(true, payload.reply(context)); err != nil {
			return err
		}
	}
	context.logEvent(payload.logEntry(context))
	return nil
}

func marshalBytes(data [][]byte) []byte {
	var result []byte
	for _, b := range data {
		result = append(result, ssh.Marshal(struct{ string }{string(b)})...)
	}
	return result
}
<commit_msg>requests: add unmarshalBytes<commit_after>package main

import (
	cryptoRand "crypto/rand"
	"errors"
	mathRand "math/rand"
	"net"
	"strconv"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
	"golang.org/x/crypto/ssh"
)

type globalRequestPayload interface {
	reply(context *connContext) []byte
	logEntry(context *connContext) logEntry
}

type globalRequestPayloadParser func(data []byte) (globalRequestPayload, error)

type tcpipRequest struct {
	Address string
	Port    uint32
}

func (request tcpipRequest) reply(context *connContext) []byte {
	if request.Port != 0 {
		return nil
	}
	return ssh.Marshal(struct{ port uint32 }{uint32(mathRand.Intn(65536-1024) + 1024)})
}
func (request tcpipRequest) logEntry(context *connContext) logEntry {
	return tcpipForwardLog{
		Address: net.JoinHostPort(request.Address, strconv.Itoa(int(request.Port))),
	}
}

type cancelTCPIPRequest struct {
	Address string
	Port    uint32
}

func (request cancelTCPIPRequest) reply(context *connContext) []byte {
	return nil
}
func (request cancelTCPIPRequest) logEntry(context *connContext) logEntry {
	return cancelTCPIPForwardLog{
		Address: net.JoinHostPort(request.Address, strconv.Itoa(int(request.Port))),
	}
}

type noMoreSessionsRequest struct {
}

func (request noMoreSessionsRequest) reply(context *connContext) []byte {
	return nil
}
func (request noMoreSessionsRequest) logEntry(context *connContext) logEntry {
	return noMoreSessionsLog{}
}

type hostKeysProveRequest struct {
	hostKeyIndices []int
}

func (request hostKeysProveRequest) reply(context *connContext) []byte {
	signatures := make([][]byte, len(request.hostKeyIndices))
	for i, index := range request.hostKeyIndices {
		signature, err := context.cfg.parsedHostKeys[index].Sign(cryptoRand.Reader, ssh.Marshal(struct {
			requestType, sessionID, hostKey string
		}{
			"hostkeys-prove-00@openssh.com",
			string(context.SessionID()),
			string(context.cfg.parsedHostKeys[index].PublicKey().Marshal()),
		}))
		if err != nil {
			warningLogger.Printf("Failed to sign host key: %v", err)
			return nil
		}
		signatures[i] = ssh.Marshal(signature)
	}
	return marshalBytes(signatures)
}
func (request hostKeysProveRequest) logEntry(context *connContext) logEntry {
	hostKeyFiles := make([]string, len(request.hostKeyIndices))
	for i, index := range request.hostKeyIndices {
		hostKeyFiles[i] = context.cfg.Server.HostKeys[index]
	}
	return hostKeysProveLog{
		HostKeyFiles: hostKeyFiles,
	}
}

var globalRequestPayloads = map[string]globalRequestPayloadParser{
	"tcpip-forward": func(data []byte) (globalRequestPayload, error) {
		payload := &tcpipRequest{}
		if err := ssh.Unmarshal(data, payload); err != nil {
			return nil, err
		}
		return payload, nil
	},
	"cancel-tcpip-forward": func(data []byte) (globalRequestPayload, error) {
		payload := &cancelTCPIPRequest{}
		if err := ssh.Unmarshal(data, payload); err != nil {
			return nil, err
		}
		return payload, nil
	},
	"no-more-sessions@openssh.com": func(data []byte) (globalRequestPayload, error) {
		if len(data) != 0 {
			return nil, errors.New("invalid request payload")
		}
		return &noMoreSessionsRequest{}, nil
	},
}

var (
	globalRequestsMetric = promauto.NewCounterVec(prometheus.CounterOpts{
		Name: "sshesame_global_requests_total",
		Help: "Total number of global requests",
	}, []string{"type"})
)

func handleGlobalRequest(request *ssh.Request, context *connContext) error {
	parser := globalRequestPayloads[request.Type]
	if parser == nil {
		globalRequestsMetric.WithLabelValues("unknown").Inc()
		warningLogger.Printf("Unsupported global request type %v", request.Type)
		if request.WantReply {
			if err := request.Reply(false, nil); err != nil {
				return err
			}
		}
		return nil
	}
	globalRequestsMetric.WithLabelValues(request.Type).Inc()
	payload, err := parser(request.Payload)
	if err != nil {
		return err
	}
	switch payload.(type) {
	case *noMoreSessionsRequest:
		context.noMoreSessions = true
	}
	if request.WantReply {
		if err := request.Reply(true, payload.reply(context)); err != nil {
			return err
		}
	}
	context.logEvent(payload.logEntry(context))
	return nil
}

func marshalBytes(data [][]byte) []byte {
	var result []byte
	for _, b := range data {
		result = append(result, ssh.Marshal(struct{ string }{string(b)})...)
	}
	return result
}

func unmarshalBytes(data []byte) ([][]byte, error) {
	var result [][]byte
	for len(data) > 0 {
		if len(data) < 4 {
			return nil, errors.New("invalid request payload")
		}
		length := binary.BigEndian.Uint32(data[:4])
		if len(data) < 4+int(length) {
			return nil, errors.New("invalid request payload")
		}
		var s struct { S string }
		if err := ssh.Unmarshal(data[:4+length], &s); err != nil {
			return nil, err
		}
		result = append(result, []byte(s.S))
		data = data[4+length:]
	}
	return result, nil
}
<|endoftext|>


<commit_before>package crawl

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

// Request - HTTP Request.
// Method is "GET" by default.
// URL can be absolute URL or a relative to source URL.
// If Raw is set to false, it means we expect HTML response
// and crawler, if request will be successfull will parse
// Response body as HTML.
// If Form is not empty, request will be send with POST method (if empty)
// and with form encoded body.
// Multipart form is not implemented.
type Request struct {
	URL       string                 `json:"url,omitempty"`
	Method    string                 `json:"method,omitempty"`
	Referer   string                 `json:"referer,omitempty"`
	Form      map[string]string      `json:"form,omitempty"`
	Header    map[string]string      `json:"header,omitempty"`
	Raw       bool                   `json:"raw,omitempty"`
	Callbacks []string               `json:"callbacks,omitempty"`
	Metadata  map[string]interface{} `json:"metadata,omitempty"`
}

// Callbacks - Helper for creating list of strings (callback names).
func Callbacks(v ...string) []string {
	return v
}

// ConstructHTTPRequest - Constructs a http.Request structure.
func ConstructHTTPRequest(req *Request) (r *http.Request, err error) {
	u, err := req.ParseURL()
	if err != nil {
		return
	}

	// Make http request
	r = &http.Request{
		URL:    u,
		Method: req.GetMethod(),
		Header: make(http.Header),
	}

	for key, value := range req.Header {
		r.Header.Set(key, value)
	}

	// Set referer if any
	if req.Referer != "" {
		r.Header.Set("Referer", req.Referer)
	}

	// Return now if no form
	if req.Form != nil {
		setRequestForm(req, r)
	}

	return
}

func setRequestForm(req *Request, r *http.Request) {
	// Set Content-Type header
	r.Header.Set("Content-Type", "application/x-www-form-urlencoded")

	// Set request method if currently is empty
	if req.Method == "" {
		r.Method = "POST"
		req.Method = "POST"
	}

	// Copy form values to request form
	formdata := make(url.Values)
	for key, value := range req.Form {
		formdata.Set(key, value)
	}

	// Set request body
	data := strings.NewReader(formdata.Encode())
	r.Body = ioutil.NopCloser(data)

	// Request content length
	r.ContentLength = int64(data.Len())
}

// ParseURL - Parses request URL.
// If request Source is set, parsed - URL is resolved
// with reference to source request URL.
func (req *Request) ParseURL() (u *url.URL, err error) {
	u, err = url.Parse(req.URL)
	if err != nil {
		return
	}
	if req.Referer != "" {
		ref, err := url.Parse(req.Referer)
		if err != nil {
			return nil, err
		}
		u = ref.ResolveReference(u)
	}
	return
}

// GetMethod - Returns request Method. If empty returns "GET".
func (req *Request) GetMethod() string {
	if req.Method == "" {
		return "GET"
	}
	return req.Method
}

// String - Returns "{method} {url}" formatted string.
func (req *Request) String() string {
	return fmt.Sprintf("%s %s", req.GetMethod(), req.URL)
}
<commit_msg>request: remove metadata<commit_after>package crawl

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

// Request - HTTP Request.
// Method is "GET" by default.
// URL can be absolute URL or a relative to source URL.
// If Raw is set to false, it means we expect HTML response
// and crawler, if request will be successfull will parse
// Response body as HTML.
// If Form is not empty, request will be send with POST method (if empty)
// and with form encoded body.
// Multipart form is not implemented.
type Request struct {
	URL       string            `json:"url,omitempty"`
	Method    string            `json:"method,omitempty"`
	Referer   string            `json:"referer,omitempty"`
	Form      map[string]string `json:"form,omitempty"`
	Header    map[string]string `json:"header,omitempty"`
	Raw       bool              `json:"raw,omitempty"`
	Callbacks []string          `json:"callbacks,omitempty"`
}

// Callbacks - Helper for creating list of strings (callback names).
func Callbacks(v ...string) []string {
	return v
}

// ConstructHTTPRequest - Constructs a http.Request structure.
func ConstructHTTPRequest(req *Request) (r *http.Request, err error) {
	u, err := req.ParseURL()
	if err != nil {
		return
	}

	// Make http request
	r = &http.Request{
		URL:    u,
		Method: req.GetMethod(),
		Header: make(http.Header),
	}

	for key, value := range req.Header {
		r.Header.Set(key, value)
	}

	// Set referer if any
	if req.Referer != "" {
		r.Header.Set("Referer", req.Referer)
	}

	// Return now if no form
	if req.Form != nil {
		setRequestForm(req, r)
	}

	return
}

func setRequestForm(req *Request, r *http.Request) {
	// Set Content-Type header
	r.Header.Set("Content-Type", "application/x-www-form-urlencoded")

	// Set request method if currently is empty
	if req.Method == "" {
		r.Method = "POST"
		req.Method = "POST"
	}

	// Copy form values to request form
	formdata := make(url.Values)
	for key, value := range req.Form {
		formdata.Set(key, value)
	}

	// Set request body
	data := strings.NewReader(formdata.Encode())
	r.Body = ioutil.NopCloser(data)

	// Request content length
	r.ContentLength = int64(data.Len())
}

// ParseURL - Parses request URL.
// If request Source is set, parsed - URL is resolved
// with reference to source request URL.
func (req *Request) ParseURL() (u *url.URL, err error) {
	u, err = url.Parse(req.URL)
	if err != nil {
		return
	}
	if req.Referer != "" {
		ref, err := url.Parse(req.Referer)
		if err != nil {
			return nil, err
		}
		u = ref.ResolveReference(u)
	}
	return
}

// GetMethod - Returns request Method. If empty returns "GET".
func (req *Request) GetMethod() string {
	if req.Method == "" {
		return "GET"
	}
	return req.Method
}

// String - Returns "{method} {url}" formatted string.
func (req *Request) String() string {
	return fmt.Sprintf("%s %s", req.GetMethod(), req.URL)
}
<|endoftext|>


<commit_before>/**
 * Copyright 2015 Comcast Cable Communications Management, LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package jtl

import (
	"encoding/json"
	"fmt"
	"net"
	"net/http"
	"os"
	"time"

	. "github.com/Comcast/eel/eel/util"
)

// StatusHandler http handler for health and status checks. Writes JSON containing config.json, handler configs and basic stats to w.
func StatusHandler(w http.ResponseWriter, r *http.Request) {
	start := time.Now()
	ctx := Gctx.SubContext()
	w.Header().Set("Content-Type", "application/json")
	state := make(map[string]interface{}, 0)
	state["Version"] = GetConfig(ctx).Version
	state["Config"] = GetConfig(ctx)
	callstats := make(map[string]interface{}, 0)
	if ctx.Value(EelDispatcher) != nil {
		callstats["WorkQueueFillLevel"] = len(GetWorkDispatcher(ctx).WorkQueue)
		callstats["WorkersIdle"] = len(GetWorkDispatcher(ctx).WorkerQueue)
	}
	if ctx.Value(EelTotalStats) != nil {
		callstats["TotalStats"] = ctx.Value(EelTotalStats)
	}
	if ctx.Value(Eel1MinStats) != nil {
		callstats[Eel1MinStats] = ctx.Value(Eel1MinStats)
	}
	if ctx.Value(Eel5MinStats) != nil {
		callstats[Eel5MinStats] = ctx.Value(Eel5MinStats)
	}
	if ctx.Value(Eel1hrStats) != nil {
		callstats[Eel1hrStats] = ctx.Value(Eel1hrStats)
	}
	if ctx.Value(Eel24hrStats) != nil {
		callstats[Eel24hrStats] = ctx.Value(Eel24hrStats)
	}
	callstats["StartTime"] = ctx.Value(EelStartTime)
	host, _ := os.Hostname()
	if host != "" {
		callstats["Hostname"] = host
	}
	elapsed1 := time.Since(start)
	addrs, err := net.InterfaceAddrs()
	if err == nil {
		for _, addr := range addrs {
			if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
				if ipnet.IP.To4() != nil {
					callstats["IpAddress"] = ipnet.IP.String()
				}
			}
		}
	}
	elapsed2 := time.Since(start)
	state["Stats"] = callstats
	state["CustomHandlers"] = GetHandlerFactory(ctx).CustomHandlerMap
	state["TopicHandlers"] = GetHandlerFactory(ctx).TopicHandlerMap
	buf, err := json.MarshalIndent(state, "", "\t")
	elapsed3 := time.Since(start)
	if err != nil {
		fmt.Fprintf(w, `{"error":"%s"}`, err.Error())
	} else {
		fmt.Fprintf(w, string(buf))
	}
	elapsed4 := time.Since(start)
	ctx.Log().Info("action", "health", "d1", elapsed1/1e6, "d2", elapsed2/1e6, "d3", elapsed3/1e6, "d4", elapsed4/1e6)
}

// VetHandler http handler for vetting all handler configurations. Writes JSON with list of warnings (if any) to w.
func VetHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	_, warnings := NewHandlerFactory(Gctx, HandlerPaths)
	if len(warnings) == 0 {
		fmt.Fprintf(w, `{"status":"ok"}`)
	} else {
		buf, err := json.MarshalIndent(warnings, "", "\t")
		if err != nil {
			fmt.Fprintf(w, `{"error":"%s"}`, err.Error())
		} else {
			fmt.Fprintf(w, string(buf))
		}
	}
}

// NilHandler http handler to to do almost nothing.
func NilHandler(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusOK)
	fmt.Fprintf(w, `{"status":"ok"}`)
}

// ReloadConfigHandler http handler to relaod all configs from disk. Response is similar to StatusHandler.
func ReloadConfigHandler(w http.ResponseWriter, r *http.Request) {
	if Gctx.Value(EelDispatcher) != nil {
		dp := Gctx.Value(EelDispatcher).(*WorkDispatcher)
		dp.Stop(Gctx)
	}
	ReloadConfig()
	InitHttpTransport(Gctx)
	dp := NewWorkDispatcher(GetConfig(Gctx).WorkerPoolSize, GetConfig(Gctx).MessageQueueDepth)
	dp.Start(Gctx)
	Gctx.AddValue(EelDispatcher, dp)
	dc := NewLocalInMemoryDupChecker(GetConfig(Gctx).DuplicateTimeout, 10000)
	Gctx.AddValue(EelDuplicateChecker, dc)
	StatusHandler(w, r)
}
<commit_msg>logging<commit_after>/**
 * Copyright 2015 Comcast Cable Communications Management, LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package jtl

import (
	"encoding/json"
	"fmt"
	"net"
	"net/http"
	"os"
	"time"

	. "github.com/Comcast/eel/eel/util"
)

// StatusHandler http handler for health and status checks. Writes JSON containing config.json, handler configs and basic stats to w.
func StatusHandler(w http.ResponseWriter, r *http.Request) {
	start := time.Now()
	ctx := Gctx.SubContext()
	w.Header().Set("Content-Type", "application/json")
	state := make(map[string]interface{}, 0)
	state["Version"] = GetConfig(ctx).Version
	state["Config"] = GetConfig(ctx)
	callstats := make(map[string]interface{}, 0)
	if ctx.Value(EelDispatcher) != nil {
		callstats["WorkQueueFillLevel"] = len(GetWorkDispatcher(ctx).WorkQueue)
		callstats["WorkersIdle"] = len(GetWorkDispatcher(ctx).WorkerQueue)
	}
	if ctx.Value(EelTotalStats) != nil {
		callstats["TotalStats"] = ctx.Value(EelTotalStats)
	}
	if ctx.Value(Eel1MinStats) != nil {
		callstats[Eel1MinStats] = ctx.Value(Eel1MinStats)
	}
	if ctx.Value(Eel5MinStats) != nil {
		callstats[Eel5MinStats] = ctx.Value(Eel5MinStats)
	}
	if ctx.Value(Eel1hrStats) != nil {
		callstats[Eel1hrStats] = ctx.Value(Eel1hrStats)
	}
	if ctx.Value(Eel24hrStats) != nil {
		callstats[Eel24hrStats] = ctx.Value(Eel24hrStats)
	}
	callstats["StartTime"] = ctx.Value(EelStartTime)
	host, _ := os.Hostname()
	if host != "" {
		callstats["Hostname"] = host
	}
	elapsed1 := time.Since(start)
	addrs, err := net.InterfaceAddrs()
	if err == nil {
		for _, addr := range addrs {
			if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
				if ipnet.IP.To4() != nil {
					callstats["IpAddress"] = ipnet.IP.String()
				}
			}
		}
	}
	elapsed2 := time.Since(start)
	state["Stats"] = callstats
	state["CustomHandlers"] = GetHandlerFactory(ctx).CustomHandlerMap
	state["TopicHandlers"] = GetHandlerFactory(ctx).TopicHandlerMap
	buf, err := json.MarshalIndent(state, "", "\t")
	elapsed3 := time.Since(start)
	if err != nil {
		fmt.Fprintf(w, `{"error":"%s"}`, err.Error())
	} else {
		fmt.Fprintf(w, string(buf))
	}
	elapsed4 := time.Since(start)
	ctx.Log().Info("action", "health", "d1", int64(elapsed1/1e6), "d2", int64(elapsed2/1e6), "d3", int64(elapsed3/1e6), "d4", int64(elapsed4/1e6))
}

// VetHandler http handler for vetting all handler configurations. Writes JSON with list of warnings (if any) to w.
func VetHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	_, warnings := NewHandlerFactory(Gctx, HandlerPaths)
	if len(warnings) == 0 {
		fmt.Fprintf(w, `{"status":"ok"}`)
	} else {
		buf, err := json.MarshalIndent(warnings, "", "\t")
		if err != nil {
			fmt.Fprintf(w, `{"error":"%s"}`, err.Error())
		} else {
			fmt.Fprintf(w, string(buf))
		}
	}
}

// NilHandler http handler to to do almost nothing.
func NilHandler(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusOK)
	fmt.Fprintf(w, `{"status":"ok"}`)
}

// ReloadConfigHandler http handler to relaod all configs from disk. Response is similar to StatusHandler.
func ReloadConfigHandler(w http.ResponseWriter, r *http.Request) {
	if Gctx.Value(EelDispatcher) != nil {
		dp := Gctx.Value(EelDispatcher).(*WorkDispatcher)
		dp.Stop(Gctx)
	}
	ReloadConfig()
	InitHttpTransport(Gctx)
	dp := NewWorkDispatcher(GetConfig(Gctx).WorkerPoolSize, GetConfig(Gctx).MessageQueueDepth)
	dp.Start(Gctx)
	Gctx.AddValue(EelDispatcher, dp)
	dc := NewLocalInMemoryDupChecker(GetConfig(Gctx).DuplicateTimeout, 10000)
	Gctx.AddValue(EelDuplicateChecker, dc)
	StatusHandler(w, r)
}
<|endoftext|>


<commit_before>// +build integration

package main

import (
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"sync"

	"github.com/boltdb/bolt"
	"github.com/gorilla/handlers"
	"github.com/spacemonkeygo/flagfile"
)

var handlerForCookie = make(map[string]http.Handler)
var currentHandlerId int
var handlerForCookieLock sync.Mutex
var dbs []*bolt.DB
var dbsLock sync.Mutex

func main() {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Kill, os.Interrupt)

	flagfile.Load()
	if generalConfig.Integration == false {
		log.Panic("Attempted to run in normal mode in an integration binary!")
	}
	mux := http.NewServeMux()
	mux.HandleFunc("/quit", func(w http.ResponseWriter, r *http.Request) {
		c <- nil
		w.WriteHeader(200)
	})
	mux.HandleFunc("/", muxTest)

	l, err := net.Listen("tcp", httpConfig.Listen)
	if err != nil {
		log.Fatalf("Failed to open tcp port, err %s", err)
	}
	go func(c <-chan os.Signal) {
		<-c
		l.Close()
	}(c)
	if err := http.Serve(l, handlers.CompressHandler(&requestLogger{mux})); err != nil {
		if oe, ok := err.(*net.OpError); ok && oe.Op == "accept" && oe.Net == "tcp" && oe.Err.Error() == "use of closed network connection" {
			log.Print("Port nicely closed")
		} else {
			log.Fatalf("%#v", err)
		}
	}
	dbsLock.Lock()
	defer dbsLock.Unlock()
	for _, db := range dbs {
		path := db.Path()
		db.Close()
		err := os.Remove(path)
		if err != nil {
			log.Printf("Failed to remove database %s, err %s", path, err)
		}
	}
	log.Print("Cleanly done")
}

func muxTest(w http.ResponseWriter, r *http.Request) {
	var handler http.Handler
	if cookie, err := r.Cookie("ClientID"); err != nil {
		handler = setupNewHandlers()
		handlerForCookieLock.Lock()
		newValue := currentHandlerId
		currentHandlerId++
		cookieValue := strconv.Itoa(newValue)
		handlerForCookie[cookieValue] = handler
		handlerForCookieLock.Unlock()
		cookie = &http.Cookie{
			Name:     "ClientID",
			Value:    cookieValue,
			HttpOnly: true,
			Path:     "/",
		}
		http.SetCookie(w, cookie)
	} else {
		cookieValue := cookie.Value
		handlerForCookieLock.Lock()
		var ok bool
		handler, ok = handlerForCookie[cookieValue]
		handlerForCookieLock.Unlock()
		if !ok {
			w.Write([]byte("Invalid cookie value!"))
			log.Print("Invalid cookie value!")
			return
		}
	}
	handler.ServeHTTP(w, r)
}

func setupNewHandlers() http.Handler {
	file, err := ioutil.TempFile("", "records")
	if err != nil {
		log.Fatalf("Failed to get temporary file, err: %s", err)
	}
	file.Close()
	db, err := bolt.Open(file.Name(), 0600, &bolt.Options{Timeout: 1})
	if err != nil {
		log.Fatalf("Failed to open bolt database, err: %s", err)
	}
	dbsLock.Lock()
	dbs = append(dbs, db)
	dbsLock.Unlock()
	mux := http.NewServeMux()
	setupStandardHandlers(mux, db)
	return mux
}
<commit_msg>Make sure the system shuts down only after all request are satisfied.<commit_after>// +build integration

package main

import (
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"sync"
	"time"

	"github.com/boltdb/bolt"
	"github.com/gorilla/handlers"
	"github.com/spacemonkeygo/flagfile"
)

var handlerForCookie = make(map[string]http.Handler)
var currentHandlerId int
var handlerForCookieLock sync.Mutex
var dbs []*bolt.DB
var dbsLock sync.Mutex
var wg sync.WaitGroup

func main() {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Kill, os.Interrupt)

	flagfile.Load()
	if generalConfig.Integration == false {
		log.Panic("Attempted to run in normal mode in an integration binary!")
	}
	mux := http.NewServeMux()
	mux.HandleFunc("/quit", func(w http.ResponseWriter, r *http.Request) {
		c <- nil
		w.WriteHeader(200)
	})
	mux.HandleFunc("/", muxTest)

	l, err := net.Listen("tcp", httpConfig.Listen)
	if err != nil {
		log.Fatalf("Failed to open tcp port, err %s", err)
	}
	go func(c <-chan os.Signal) {
		<-c
		l.Close()
	}(c)
	server := http.Server{
		Handler:      handlers.CompressHandler(&requestLogger{mux}),
		ReadTimeout:  time.Second * 60,
		WriteTimeout: time.Second * 60,
		ConnState:    ConnectionAccounting,
	}
	if err := server.Serve(l); err != nil {
		if oe, ok := err.(*net.OpError); ok && oe.Op == "accept" && oe.Net == "tcp" && oe.Err.Error() == "use of closed network connection" {
			log.Print("Port nicely closed")
		} else {
			log.Fatalf("%#v", err)
		}
	}
	wg.Wait()
	dbsLock.Lock()
	defer dbsLock.Unlock()
	for _, db := range dbs {
		path := db.Path()
		db.Close()
		err := os.Remove(path)
		if err != nil {
			log.Printf("Failed to remove database %s, err %s", path, err)
		}
	}
	log.Print("Cleanly done")
}

func ConnectionAccounting(_ net.Conn, state http.ConnState) {
	switch state {
	case http.StateNew:
		wg.Add(1)
	case http.StateClosed:
		wg.Done()
	}
}

func muxTest(w http.ResponseWriter, r *http.Request) {
	var handler http.Handler
	if cookie, err := r.Cookie("ClientID"); err != nil {
		handler = setupNewHandlers()
		handlerForCookieLock.Lock()
		newValue := currentHandlerId
		currentHandlerId++
		cookieValue := strconv.Itoa(newValue)
		handlerForCookie[cookieValue] = handler
		handlerForCookieLock.Unlock()
		cookie = &http.Cookie{
			Name:     "ClientID",
			Value:    cookieValue,
			HttpOnly: true,
			Path:     "/",
		}
		http.SetCookie(w, cookie)
	} else {
		cookieValue := cookie.Value
		handlerForCookieLock.Lock()
		var ok bool
		handler, ok = handlerForCookie[cookieValue]
		handlerForCookieLock.Unlock()
		if !ok {
			w.Write([]byte("Invalid cookie value!"))
			log.Print("Invalid cookie value!")
			return
		}
	}
	handler.ServeHTTP(w, r)
}

func setupNewHandlers() http.Handler {
	file, err := ioutil.TempFile("", "records")
	if err != nil {
		log.Fatalf("Failed to get temporary file, err: %s", err)
	}
	file.Close()
	db, err := bolt.Open(file.Name(), 0600, &bolt.Options{Timeout: 1})
	if err != nil {
		log.Fatalf("Failed to open bolt database, err: %s", err)
	}
	dbsLock.Lock()
	dbs = append(dbs, db)
	dbsLock.Unlock()
	mux := http.NewServeMux()
	setupStandardHandlers(mux, db)
	return mux
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"go/build"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"path/filepath"
	"strings"

	. "github.com/AlekSi/nut"
)

var (
	cmdGet = &Command{
		Run:       runGet,
		UsageLine: "get [-p prefix] [-v] [name, import path or URL]",
		Short:     "download and install nut and dependencies",
	}

	getP string
	getV bool
)

func init() {
	cmdGet.Long = `
Downloads and installs nut and dependencies from http://gonuts.io/ or specified URL.

Examples:
    nut install AlekSi/nut
    nut install AlekSi/nut/0.2.0
    nut install gonuts.io/AlekSi/nut
    nut install gonuts.io/AlekSi/nut/0.2.0
    nut install http://gonuts.io/AlekSi/nut
    nut install http://gonuts.io/AlekSi/nut/0.2.0
`

	cmdGet.Flag.StringVar(&getP, "p", "", "install prefix in workspace, uses hostname from URL if omitted")
	cmdGet.Flag.BoolVar(&getV, "v", false, vHelp)
}

func ArgToURL(s string) *url.URL {
	var p []string
	var host string
	var ok bool

	// full URL - as is
	if strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://") {
		goto parse
	}

	p = strings.Split(s, "/")
	if len(p) > 0 {
		host, ok = NutImportPrefixes[p[0]]
	}
	if ok {
		// import path style
		p[0] = "http://" + host
		s = strings.Join(p, "/")
	} else {
		// short style
		s = fmt.Sprintf("http://%s/%s", NutImportPrefixes["gonuts.io"], s)
	}

parse:
	u, err := url.Parse(s)
	PanicIfErr(err)
	return u
}

func get(url *url.URL) (b []byte, err error) {
	if getV {
		log.Printf("Getting %s ...", url)
	}

	req, err := http.NewRequest("GET", url.String(), nil)
	if err != nil {
		return
	}
	req.Header.Set("User-Agent", "nut getter")
	req.Header.Set("Accept", "application/zip")

	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return
	}
	defer res.Body.Close()

	b, err = ioutil.ReadAll(res.Body)
	if err != nil {
		return
	}

	if res.StatusCode/100 != 2 {
		err = fmt.Errorf("Status code %d", res.StatusCode)
		return
	}

	if getV {
		log.Printf("Status code %d", res.StatusCode)
	}

	return
}

func runGet(cmd *Command) {
	if !getV {
		getV = Config.V
	}

	args := cmd.Flag.Args()

	// zero arguments is a special case – install dependencies for package in current directory
	if len(args) == 0 {
		pack, err := build.ImportDir(".", 0)
		PanicIfErr(err)
		args = NutImports(pack.Imports)
		if getV && len(args) != 0 {
			log.Printf("%s depends on nuts: %s", pack.Name, strings.Join(args, ","))
		}
	}

	installPaths := make(map[string]bool, len(args))
	for len(args) != 0 {
		arg := args[0]
		args = args[1:]

		url := ArgToURL(arg)
		b, err := get(url)
		PanicIfErr(err)

		nf := new(NutFile)
		nf.ReadFrom(bytes.NewReader(b))
		deps := NutImports(nf.Imports)
		if getV && len(deps) != 0 {
			log.Printf("%s depends on nuts: %s", nf.Name, strings.Join(deps, ","))
		}
		args = append(args, deps...)

		p := getP
		if p == "" {
			if strings.Contains(url.Host, ":") {
				p, _, err = net.SplitHostPort(url.Host)
				PanicIfErr(err)
			} else {
				p = url.Host
			}
			if strings.HasPrefix(p, "www.") {
				p = p[4:]
			}
		}
		fileName := WriteNut(b, p, getV)
		path := filepath.Join(p, nf.Name, nf.Version.String())

		UnpackNut(fileName, filepath.Join(SrcDir, path), true, getV)
		installPaths[path] = true
	}

	for path := range installPaths {
		InstallPackage(path, getV)
	}
}
<commit_msg>`nut get`: print error.<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"go/build"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"path/filepath"
	"strings"

	. "github.com/AlekSi/nut"
)

var (
	cmdGet = &Command{
		Run:       runGet,
		UsageLine: "get [-p prefix] [-v] [name, import path or URL]",
		Short:     "download and install nut and dependencies",
	}

	getP string
	getV bool
)

func init() {
	cmdGet.Long = `
Downloads and installs nut and dependencies from http://gonuts.io/ or specified URL.

Examples:
    nut install AlekSi/nut
    nut install AlekSi/nut/0.2.0
    nut install gonuts.io/AlekSi/nut
    nut install gonuts.io/AlekSi/nut/0.2.0
    nut install http://gonuts.io/AlekSi/nut
    nut install http://gonuts.io/AlekSi/nut/0.2.0
`

	cmdGet.Flag.StringVar(&getP, "p", "", "install prefix in workspace, uses hostname from URL if omitted")
	cmdGet.Flag.BoolVar(&getV, "v", false, vHelp)
}

func ArgToURL(s string) *url.URL {
	var p []string
	var host string
	var ok bool

	// full URL - as is
	if strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://") {
		goto parse
	}

	p = strings.Split(s, "/")
	if len(p) > 0 {
		host, ok = NutImportPrefixes[p[0]]
	}
	if ok {
		// import path style
		p[0] = "http://" + host
		s = strings.Join(p, "/")
	} else {
		// short style
		s = fmt.Sprintf("http://%s/%s", NutImportPrefixes["gonuts.io"], s)
	}

parse:
	u, err := url.Parse(s)
	PanicIfErr(err)
	return u
}

func get(url *url.URL) (b []byte, err error) {
	if getV {
		log.Printf("Getting %s ...", url)
	}

	req, err := http.NewRequest("GET", url.String(), nil)
	if err != nil {
		return
	}
	req.Header.Set("User-Agent", "nut getter")
	req.Header.Set("Accept", "application/zip")

	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return
	}
	defer res.Body.Close()

	b, err = ioutil.ReadAll(res.Body)
	if err != nil {
		return
	}

	if res.StatusCode/100 != 2 {
		err = fmt.Errorf("Status code %d", res.StatusCode)
		return
	}

	if getV {
		log.Printf("Status code %d", res.StatusCode)
	}

	return
}

func runGet(cmd *Command) {
	if !getV {
		getV = Config.V
	}

	args := cmd.Flag.Args()

	// zero arguments is a special case – install dependencies for package in current directory
	if len(args) == 0 {
		pack, err := build.ImportDir(".", 0)
		PanicIfErr(err)
		args = NutImports(pack.Imports)
		if getV && len(args) != 0 {
			log.Printf("%s depends on nuts: %s", pack.Name, strings.Join(args, ","))
		}
	}

	installPaths := make(map[string]bool, len(args))
	for len(args) != 0 {
		arg := args[0]
		args = args[1:]

		url := ArgToURL(arg)
		b, err := get(url)
		if err != nil {
			log.Print(err)

			var body map[string]interface{}
			err = json.Unmarshal(b, &body)
			if err != nil {
				log.Print(err)
			}
			m, ok := body["Message"]
			if ok {
				log.Fatalf("%s", m)
			} else {
				log.Fatalf("Response: %#q", body)
			}
		}

		nf := new(NutFile)
		nf.ReadFrom(bytes.NewReader(b))
		deps := NutImports(nf.Imports)
		if getV && len(deps) != 0 {
			log.Printf("%s depends on nuts: %s", nf.Name, strings.Join(deps, ","))
		}
		args = append(args, deps...)

		p := getP
		if p == "" {
			if strings.Contains(url.Host, ":") {
				p, _, err = net.SplitHostPort(url.Host)
				PanicIfErr(err)
			} else {
				p = url.Host
			}
			if strings.HasPrefix(p, "www.") {
				p = p[4:]
			}
		}
		fileName := WriteNut(b, p, getV)
		path := filepath.Join(p, nf.Name, nf.Version.String())

		UnpackNut(fileName, filepath.Join(SrcDir, path), true, getV)
		installPaths[path] = true
	}

	for path := range installPaths {
		InstallPackage(path, getV)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"fmt"
	"sync"
	"time"

	v3 "github.com/coreos/etcd/clientv3"

	"github.com/spf13/cobra"
	"golang.org/x/net/context"
	"gopkg.in/cheggaaa/pb.v1"
)

// watchGetCmd represents the watch command
var watchGetCmd = &cobra.Command{
	Use:   "watch-get",
	Short: "Benchmark watch with get",
	Long:  `Benchmark for serialized key gets with many unsynced watchers`,
	Run:   watchGetFunc,
}

var (
	watchGetTotalStreams int
	watchEvents          int
	firstWatch           sync.Once
)

func init() {
	RootCmd.AddCommand(watchGetCmd)
	watchGetCmd.Flags().IntVar(&watchGetTotalStreams, "watchers", 10000, "Total number of watchers")
	watchGetCmd.Flags().IntVar(&watchEvents, "events", 8, "Number of events per watcher")
}

func watchGetFunc(cmd *cobra.Command, args []string) {
	clients := mustCreateClients(totalClients, totalConns)
	getClient := mustCreateClients(1, 1)

	// setup keys for watchers
	watchRev := int64(0)
	for i := 0; i < watchEvents; i++ {
		v := fmt.Sprintf("%d", i)
		resp, err := clients[0].Put(context.TODO(), "watchkey", v)
		if err != nil {
			panic(err)
		}
		if i == 0 {
			watchRev = resp.Header.Revision
		}
	}

	streams := make([]v3.Watcher, watchGetTotalStreams)
	for i := range streams {
		streams[i] = v3.NewWatcher(clients[i%len(clients)])
	}

	// results from trying to do serialized gets with concurrent watchers
	results = make(chan result)

	bar = pb.New(watchGetTotalStreams * watchEvents)
	bar.Format("Bom !")
	bar.Start()

	pdoneC := printReport(results)
	wg.Add(len(streams))
	ctx, cancel := context.WithCancel(context.TODO())
	f := func() {
		doSerializedGet(ctx, getClient[0], results)
	}
	for i := range streams {
		go doUnsyncWatch(streams[i], watchRev, f)
	}
	wg.Wait()
	cancel()
	bar.Finish()
	fmt.Printf("Get during watch summary:\n")
	<-pdoneC
}

func doSerializedGet(ctx context.Context, client *v3.Client, results chan result) {
	for {
		st := time.Now()
		_, err := client.Get(ctx, "abc", v3.WithSerializable())
		if ctx.Err() != nil {
			break
		}
		var errStr string
		if err != nil {
			errStr = err.Error()
		}
		res := result{errStr: errStr, duration: time.Since(st), happened: time.Now()}
		results <- res
	}
	close(results)
}

func doUnsyncWatch(stream v3.Watcher, rev int64, f func()) {
	wch := stream.Watch(context.TODO(), "watchkey", v3.WithRev(rev))
	if wch == nil {
		panic("could not open watch channel")
	}
	firstWatch.Do(func() { go f() })
	i := 0
	for i < watchEvents {
		wev := <-wch
		i += len(wev.Events)
		bar.Add(len(wev.Events))
	}
	wg.Done()
}
<commit_msg>benchmark: make number of watcher streams configurable in watch-get<commit_after>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"fmt"
	"sync"
	"time"

	v3 "github.com/coreos/etcd/clientv3"

	"github.com/spf13/cobra"
	"golang.org/x/net/context"
	"gopkg.in/cheggaaa/pb.v1"
)

// watchGetCmd represents the watch command
var watchGetCmd = &cobra.Command{
	Use:   "watch-get",
	Short: "Benchmark watch with get",
	Long:  `Benchmark for serialized key gets with many unsynced watchers`,
	Run:   watchGetFunc,
}

var (
	watchGetTotalWatchers int
	watchGetTotalStreams  int
	watchEvents           int
	firstWatch            sync.Once
)

func init() {
	RootCmd.AddCommand(watchGetCmd)
	watchGetCmd.Flags().IntVar(&watchGetTotalWatchers, "watchers", 10000, "Total number of watchers")
	watchGetCmd.Flags().IntVar(&watchGetTotalStreams, "streams", 1, "Total number of watcher streams")
	watchGetCmd.Flags().IntVar(&watchEvents, "events", 8, "Number of events per watcher")
}

func watchGetFunc(cmd *cobra.Command, args []string) {
	clients := mustCreateClients(totalClients, totalConns)
	getClient := mustCreateClients(1, 1)

	// setup keys for watchers
	watchRev := int64(0)
	for i := 0; i < watchEvents; i++ {
		v := fmt.Sprintf("%d", i)
		resp, err := clients[0].Put(context.TODO(), "watchkey", v)
		if err != nil {
			panic(err)
		}
		if i == 0 {
			watchRev = resp.Header.Revision
		}
	}

	streams := make([]v3.Watcher, watchGetTotalStreams)
	for i := range streams {
		streams[i] = v3.NewWatcher(clients[i%len(clients)])
	}

	// results from trying to do serialized gets with concurrent watchers
	results = make(chan result)

	bar = pb.New(watchGetTotalWatchers * watchEvents)
	bar.Format("Bom !")
	bar.Start()

	pdoneC := printReport(results)
	wg.Add(watchGetTotalWatchers)
	ctx, cancel := context.WithCancel(context.TODO())
	f := func() {
		doSerializedGet(ctx, getClient[0], results)
	}
	for i := 0; i < watchGetTotalWatchers; i++ {
		go doUnsyncWatch(streams[i%len(streams)], watchRev, f)
	}
	wg.Wait()
	cancel()
	bar.Finish()
	fmt.Printf("Get during watch summary:\n")
	<-pdoneC
}

func doSerializedGet(ctx context.Context, client *v3.Client, results chan result) {
	for {
		st := time.Now()
		_, err := client.Get(ctx, "abc", v3.WithSerializable())
		if ctx.Err() != nil {
			break
		}
		var errStr string
		if err != nil {
			errStr = err.Error()
		}
		res := result{errStr: errStr, duration: time.Since(st), happened: time.Now()}
		results <- res
	}
	close(results)
}

func doUnsyncWatch(stream v3.Watcher, rev int64, f func()) {
	wch := stream.Watch(context.TODO(), "watchkey", v3.WithRev(rev))
	if wch == nil {
		panic("could not open watch channel")
	}
	firstWatch.Do(func() { go f() })
	i := 0
	for i < watchEvents {
		wev := <-wch
		i += len(wev.Events)
		bar.Add(len(wev.Events))
	}
	wg.Done()
}
<|endoftext|>


<commit_before>package db

import (
	"encoding/binary"
	"errors"

	"bazil.org/bazil/kv"
	"bazil.org/bazil/kv/kvmulti"
	"bazil.org/bazil/peer"
	"bazil.org/bazil/tokens"
	"github.com/boltdb/bolt"
)

var (
	ErrPeerNotFound      = errors.New("peer not found")
	ErrNoStorageForPeer  = errors.New("no storage offered to peer")
	ErrNoLocationForPeer = errors.New("no network location known for peer")
)

var (
	bucketPeer        = []byte(tokens.BucketPeer)
	bucketPeerID      = []byte(tokens.BucketPeerID)
	peerStateID       = []byte(tokens.PeerStateID)
	peerStateLocation = []byte(tokens.PeerStateLocation)
	peerStateStorage  = []byte(tokens.PeerStateStorage)
)

func (tx *Tx) initPeers() error {
	if _, err := tx.CreateBucketIfNotExists(bucketPeer); err != nil {
		return err
	}
	if _, err := tx.CreateBucketIfNotExists(bucketPeerID); err != nil {
		return err
	}
	return nil
}

func (tx *Tx) Peers() *Peers {
	p := &Peers{
		peers: tx.Bucket(bucketPeer),
		ids:   tx.Bucket(bucketPeerID),
	}
	return p
}

type Peers struct {
	peers *bolt.Bucket
	ids   *bolt.Bucket
}

// Get returns a Peer for the given public key.
//
// If the peer does not exist, returns ErrPeerNotFound.
func (b *Peers) Get(pub *peer.PublicKey) (*Peer, error) {
	bp := b.peers.Bucket(pub[:])
	if bp == nil {
		return nil, ErrPeerNotFound
	}
	p := &Peer{
		b:   bp,
		pub: pub,
	}
	return p, nil
}

// Make returns a Peer for the given public key, adding it if
// necessary.
func (b *Peers) Make(pub *peer.PublicKey) (*Peer, error) {
	p, err := b.Get(pub)
	if err != ErrPeerNotFound {
		// operational error or success, either is fine here
		return p, err
	}

	// really not found -> add it; first,pick a free id
	var id peer.ID
	c := b.ids.Cursor()
	if k, _ := c.Last(); k != nil {
		id = peer.ID(binary.BigEndian.Uint32(k))
	}
	id++
	if id == 0 {
		return nil, errors.New("out of peer IDs")
	}
	var idKey [4]byte
	binary.BigEndian.PutUint32(idKey[:], uint32(id))
	if err := b.ids.Put(idKey[:], pub[:]); err != nil {
		return nil, err
	}

	// create a bucket to hold information about the peer
	bp, err := b.peers.CreateBucket(pub[:])
	if err != nil {
		return nil, err
	}
	if err := bp.Put(peerStateID, idKey[:]); err != nil {
		return nil, err
	}
	if _, err := bp.CreateBucket(peerStateLocation); err != nil {
		return nil, err
	}
	if _, err := bp.CreateBucket(peerStateStorage); err != nil {
		return nil, err
	}

	p = &Peer{
		b:   bp,
		pub: pub,
	}
	return p, nil
}

type Peer struct {
	b   *bolt.Bucket
	pub *peer.PublicKey
}

func (p *Peer) Pub() *peer.PublicKey {
	return p.pub
}

func (p *Peer) ID() peer.ID {
	v := p.b.Get(peerStateID)
	if v == nil {
		panic("peer corrupt, missing id: " + p.pub.String())
	}
	return peer.ID(binary.BigEndian.Uint32(v))
}

func (p *Peer) Locations() *PeerLocations {
	b := p.b.Bucket(peerStateLocation)
	return &PeerLocations{b}
}

type PeerLocations struct {
	b *bolt.Bucket
}

// Set the network location where the peer can be contacted.
//
// TODO support multiple addresses to attempt, with some idea of
// preferring recent ones.
func (p *PeerLocations) Set(addr string) error {
	// remove all the other values, for "set"
	c := p.b.Cursor()
	for k, _ := c.First(); k != nil; k, _ = c.Next() {
		if err := p.b.Delete(k); err != nil {
			return err
		}
	}
	return p.b.Put([]byte(addr), nil)
}

// Get an address to try, to connect to the peer.
//
// Returned addr is valid after the transaction.
//
// TODO support multiple addresses to attempt, with some idea of
// preferring recent ones.
func (p *PeerLocations) Get() (addr string, err error) {
	c := p.b.Cursor()
	for k, _ := c.First(); k != nil; k, _ = c.Next() {
		return string(k), nil
	}
	return "", ErrNoLocationForPeer
}

func (p *Peer) Storage() *PeerStorage {
	b := p.b.Bucket(peerStateStorage)
	return &PeerStorage{b}
}

type PeerStorage struct {
	b *bolt.Bucket
}

func (p *PeerStorage) Allow(backend string) error {
	return p.b.Put([]byte(backend), nil)
}

// Open key-value stores as allowed for this peer. Uses the opener
// function for the actual open action.
//
// If the peer is not allowed to use any storage, returns
// ErrNoStorageForPeer.
//
// Returned KV is valid after the transaction. Strings passed to the
// opener function are valid after the transaction.
func (p *PeerStorage) Open(opener func(string) (kv.KV, error)) (kv.KV, error) {
	var kvstores []kv.KV
	c := p.b.Cursor()
	for k, _ := c.First(); k != nil; k, _ = c.Next() {
		backend := string(k)
		// later value may include quota style restrictions
		s, err := opener(backend)
		if err != nil {
			// TODO once kv.KV has Close, close all in kvstores
			return nil, err
		}
		kvstores = append(kvstores, s)
	}
	if len(kvstores) == 0 {
		return nil, ErrNoStorageForPeer
	}
	return kvmulti.New(kvstores...), nil
}
<commit_msg>db: Iterate all peers with a cursor<commit_after>package db

import (
	"encoding/binary"
	"errors"

	"bazil.org/bazil/kv"
	"bazil.org/bazil/kv/kvmulti"
	"bazil.org/bazil/peer"
	"bazil.org/bazil/tokens"
	"github.com/boltdb/bolt"
)

var (
	ErrPeerNotFound      = errors.New("peer not found")
	ErrNoStorageForPeer  = errors.New("no storage offered to peer")
	ErrNoLocationForPeer = errors.New("no network location known for peer")
)

var (
	bucketPeer        = []byte(tokens.BucketPeer)
	bucketPeerID      = []byte(tokens.BucketPeerID)
	peerStateID       = []byte(tokens.PeerStateID)
	peerStateLocation = []byte(tokens.PeerStateLocation)
	peerStateStorage  = []byte(tokens.PeerStateStorage)
)

func (tx *Tx) initPeers() error {
	if _, err := tx.CreateBucketIfNotExists(bucketPeer); err != nil {
		return err
	}
	if _, err := tx.CreateBucketIfNotExists(bucketPeerID); err != nil {
		return err
	}
	return nil
}

func (tx *Tx) Peers() *Peers {
	p := &Peers{
		peers: tx.Bucket(bucketPeer),
		ids:   tx.Bucket(bucketPeerID),
	}
	return p
}

type Peers struct {
	peers *bolt.Bucket
	ids   *bolt.Bucket
}

// Get returns a Peer for the given public key.
//
// If the peer does not exist, returns ErrPeerNotFound.
func (b *Peers) Get(pub *peer.PublicKey) (*Peer, error) {
	bp := b.peers.Bucket(pub[:])
	if bp == nil {
		return nil, ErrPeerNotFound
	}
	p := &Peer{
		b:   bp,
		pub: pub,
	}
	return p, nil
}

// Make returns a Peer for the given public key, adding it if
// necessary.
func (b *Peers) Make(pub *peer.PublicKey) (*Peer, error) {
	p, err := b.Get(pub)
	if err != ErrPeerNotFound {
		// operational error or success, either is fine here
		return p, err
	}

	// really not found -> add it; first,pick a free id
	var id peer.ID
	c := b.ids.Cursor()
	if k, _ := c.Last(); k != nil {
		id = peer.ID(binary.BigEndian.Uint32(k))
	}
	id++
	if id == 0 {
		return nil, errors.New("out of peer IDs")
	}
	var idKey [4]byte
	binary.BigEndian.PutUint32(idKey[:], uint32(id))
	if err := b.ids.Put(idKey[:], pub[:]); err != nil {
		return nil, err
	}

	// create a bucket to hold information about the peer
	bp, err := b.peers.CreateBucket(pub[:])
	if err != nil {
		return nil, err
	}
	if err := bp.Put(peerStateID, idKey[:]); err != nil {
		return nil, err
	}
	if _, err := bp.CreateBucket(peerStateLocation); err != nil {
		return nil, err
	}
	if _, err := bp.CreateBucket(peerStateStorage); err != nil {
		return nil, err
	}

	p = &Peer{
		b:   bp,
		pub: pub,
	}
	return p, nil
}

func (b *Peers) Cursor() *PeersCursor {
	return &PeersCursor{b.peers.Cursor()}
}

type PeersCursor struct {
	c *bolt.Cursor
}

func (c *PeersCursor) item(k, _ []byte) *Peer {
	if k == nil {
		return nil
	}
	bucket := c.c.Bucket().Bucket(k)
	if bucket == nil {
		panic("db peer corrupt, not a bucket")
	}
	var pub peer.PublicKey
	if err := pub.UnmarshalBinary(k); err != nil {
		panic("db peer corrupt: " + err.Error())
	}
	p := &Peer{
		b:   bucket,
		pub: &pub,
	}
	return p
}

func (c *PeersCursor) First() *Peer {
	return c.item(c.c.First())
}

func (c *PeersCursor) Next() *Peer {
	return c.item(c.c.Next())
}

type Peer struct {
	b   *bolt.Bucket
	pub *peer.PublicKey
}

func (p *Peer) Pub() *peer.PublicKey {
	return p.pub
}

func (p *Peer) ID() peer.ID {
	v := p.b.Get(peerStateID)
	if v == nil {
		panic("peer corrupt, missing id: " + p.pub.String())
	}
	return peer.ID(binary.BigEndian.Uint32(v))
}

func (p *Peer) Locations() *PeerLocations {
	b := p.b.Bucket(peerStateLocation)
	return &PeerLocations{b}
}

type PeerLocations struct {
	b *bolt.Bucket
}

// Set the network location where the peer can be contacted.
//
// TODO support multiple addresses to attempt, with some idea of
// preferring recent ones.
func (p *PeerLocations) Set(addr string) error {
	// remove all the other values, for "set"
	c := p.b.Cursor()
	for k, _ := c.First(); k != nil; k, _ = c.Next() {
		if err := p.b.Delete(k); err != nil {
			return err
		}
	}
	return p.b.Put([]byte(addr), nil)
}

// Get an address to try, to connect to the peer.
//
// Returned addr is valid after the transaction.
//
// TODO support multiple addresses to attempt, with some idea of
// preferring recent ones.
func (p *PeerLocations) Get() (addr string, err error) {
	c := p.b.Cursor()
	for k, _ := c.First(); k != nil; k, _ = c.Next() {
		return string(k), nil
	}
	return "", ErrNoLocationForPeer
}

func (p *Peer) Storage() *PeerStorage {
	b := p.b.Bucket(peerStateStorage)
	return &PeerStorage{b}
}

type PeerStorage struct {
	b *bolt.Bucket
}

func (p *PeerStorage) Allow(backend string) error {
	return p.b.Put([]byte(backend), nil)
}

// Open key-value stores as allowed for this peer. Uses the opener
// function for the actual open action.
//
// If the peer is not allowed to use any storage, returns
// ErrNoStorageForPeer.
//
// Returned KV is valid after the transaction. Strings passed to the
// opener function are valid after the transaction.
func (p *PeerStorage) Open(opener func(string) (kv.KV, error)) (kv.KV, error) {
	var kvstores []kv.KV
	c := p.b.Cursor()
	for k, _ := c.First(); k != nil; k, _ = c.Next() {
		backend := string(k)
		// later value may include quota style restrictions
		s, err := opener(backend)
		if err != nil {
			// TODO once kv.KV has Close, close all in kvstores
			return nil, err
		}
		kvstores = append(kvstores, s)
	}
	if len(kvstores) == 0 {
		return nil, ErrNoStorageForPeer
	}
	return kvmulti.New(kvstores...), nil
}
<|endoftext|>