<commit_before>// Copyright 2016 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/*
Package pubsub provides an easy way to publish and receive Google Cloud Pub/Sub
messages, hiding the details of the underlying server RPCs.  Google Cloud
Pub/Sub is a many-to-many, asynchronous messaging system that decouples senders
and receivers.

More information about Google Cloud Pub/Sub is available at
https://cloud.google.com/pubsub/docs

See https://godoc.org/cloud.google.com/go for authentication, timeouts,
connection pooling and similar aspects of this package.

Publishing

Google Cloud Pub/Sub messages are published to topics. Topics may be created
using the pubsub package like so:

 topic, err := pubsubClient.CreateTopic(context.Background(), "topic-name")

Messages may then be published to a topic:

 res := topic.Publish(ctx, &pubsub.Message{Data: []byte("payload")})

Publish queues the message for publishing and returns immediately. When enough
messages have accumulated, or enough time has elapsed, the batch of messages is
sent to the Pub/Sub service.

Publish returns a PublishResult, which behaves like a future: its Get method
blocks until the message has been sent to the service.

The first time you call Publish on a topic, goroutines are started in the
background. To clean up these goroutines, call Stop:

  topic.Stop()

Receiving

To receive messages published to a topic, clients create subscriptions
to the topic. There may be more than one subscription per topic; each message
that is published to the topic will be delivered to all of its subscriptions.

Subscriptions may be created like so:

 sub, err := pubsubClient.CreateSubscription(context.Background(), "sub-name",
	pubsub.SubscriptionConfig{Topic: topic})

Messages are then consumed from a subscription via callback.

 err := sub.Receive(context.Background(), func(ctx context.Context, m *Message) {
	log.Printf("Got message: %s", m.Data)
	m.Ack()
 })
 if err != nil {
	// Handle error.
 }

The callback is invoked concurrently by multiple goroutines, maximizing
throughput. To terminate a call to Receive, cancel its context.

Once client code has processed the message, it must call Message.Ack or
Message.Nack; otherwise the message will eventually be redelivered. Ack/Nack
MUST be called within the Receive handler function, and not from a goroutine.
Otherwise, flow control (e.g. ReceiveSettings.MaxOutstandingMessages) will
not be respected, and messages can get orphaned when cancelling Receive.

If the client cannot or doesn't want to process the message, it can call Message.Nack
to speed redelivery. For more information and configuration options, see
"Ack Deadlines" below.

Note: It is possible for Messages to be redelivered even if Message.Ack has
been called. Client code must be robust to multiple deliveries of messages.

Note: This uses pubsub's streaming pull feature. This feature has properties that
may be surprising. Please take a look at https://cloud.google.com/pubsub/docs/pull#streamingpull
for more details on how streaming pull behaves compared to the synchronous
pull method.

Streams Management

Streams used for streaming pull are managed by the gRPC connection pool setting.
By default, the number of connections in the pool is max(4,GOMAXPROCS/NumCPU).

If you have 4 or more CPU cores, the default setting allows 400 streams which is still a good default for most cases.
If you want to have more open streams (such as for low CPU core machines), you should pass in the grpc option as described below:

 opts := []option.ClientOption{
	option.WithGRPCConnectionPool(8),
 }
 client, err := pubsub.NewClient(ctx, projID, opts...)

Ack Deadlines

The default pubsub deadlines are suitable for most use cases, but may be
overridden. This section describes the tradeoffs that should be considered
when overriding the defaults.

Behind the scenes, each message returned by the Pub/Sub server has an
associated lease, known as an "ack deadline". Unless a message is
acknowledged within the ack deadline, or the client requests that
the ack deadline be extended, the message will become eligible for redelivery.

As a convenience, the pubsub client will automatically extend deadlines until
either:
  - Message.Ack or Message.Nack is called, or
  - The "MaxExtension" duration elapses from the time the message is fetched from
    the server. This defaults to 60m.

Ack deadlines are extended periodically by the client. The initial ack
deadline given to messages is based on the subscription's AckDeadline property,
which defaults to 10s. The period between extensions, as well as the
length of the extension, automatically adjusts based on the time it takes the
subscriber application to ack messages (based on the 99th percentile of ack latency).
By default, this extension period is capped at 10m, but this limit can be configured
by the "MaxExtensionPeriod" setting. This has the effect that subscribers that process
messages quickly have their message ack deadlines extended for a short amount, whereas
subscribers that process message slowly have their message ack deadlines extended
for a large amount. The net effect is fewer RPCs sent from the client library.

For example, consider a subscriber that takes 3 minutes to process each message.
Since the library has already recorded several 3-minute "ack latencies"s in a
percentile distribution, future message extensions are sent with a value of 3
minutes, every 3 minutes. Suppose the application crashes 5 seconds after the
library sends such an extension: the Pub/Sub server would wait the remaining
2m55s before re-sending the messages out to other subscribers.

Please note that by default, the client library does not use the subscription's
AckDeadline for the MaxExtension value. To enforce the subscription's AckDeadline,
set MaxExtension to the subscription's AckDeadline:

	cfg, err := sub.Config(ctx)
	if err != nil {
		// TODO: handle err
	}

	sub.ReceiveSettings.MaxExtension = cfg.AckDeadline

Slow Message Processing

For use cases where message processing exceeds 30 minutes, we recommend using
the base client in a pull model, since long-lived streams are periodically killed
by firewalls. See the example at https://godoc.org/cloud.google.com/go/pubsub/apiv1#example-SubscriberClient-Pull-LengthyClientProcessing

Emulator

To use an emulator with this library, you can set the PUBSUB_EMULATOR_HOST
environment variable to the address at which your emulator is running. This will
send requests to that address instead of to Cloud Pub/Sub. You can then create
and use a client as usual:

	// Set PUBSUB_EMULATOR_HOST environment variable.
	err := os.Setenv("PUBSUB_EMULATOR_HOST", "localhost:9000")
	if err != nil {
		// TODO: Handle error.
	}
	// Create client as usual.
	client, err := pubsub.NewClient(ctx, "my-project-id")
	if err != nil {
		// TODO: Handle error.
	}
	defer client.Close()

*/
package pubsub // import "cloud.google.com/go/pubsub"
<commit_msg>doc: fix incorrect connection pool documentation (#6450)<commit_after>// Copyright 2016 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/*
Package pubsub provides an easy way to publish and receive Google Cloud Pub/Sub
messages, hiding the details of the underlying server RPCs.  Google Cloud
Pub/Sub is a many-to-many, asynchronous messaging system that decouples senders
and receivers.

More information about Google Cloud Pub/Sub is available at
https://cloud.google.com/pubsub/docs

See https://godoc.org/cloud.google.com/go for authentication, timeouts,
connection pooling and similar aspects of this package.

Publishing

Google Cloud Pub/Sub messages are published to topics. Topics may be created
using the pubsub package like so:

 topic, err := pubsubClient.CreateTopic(context.Background(), "topic-name")

Messages may then be published to a topic:

 res := topic.Publish(ctx, &pubsub.Message{Data: []byte("payload")})

Publish queues the message for publishing and returns immediately. When enough
messages have accumulated, or enough time has elapsed, the batch of messages is
sent to the Pub/Sub service.

Publish returns a PublishResult, which behaves like a future: its Get method
blocks until the message has been sent to the service.

The first time you call Publish on a topic, goroutines are started in the
background. To clean up these goroutines, call Stop:

  topic.Stop()

Receiving

To receive messages published to a topic, clients create subscriptions
to the topic. There may be more than one subscription per topic; each message
that is published to the topic will be delivered to all of its subscriptions.

Subscriptions may be created like so:

 sub, err := pubsubClient.CreateSubscription(context.Background(), "sub-name",
	pubsub.SubscriptionConfig{Topic: topic})

Messages are then consumed from a subscription via callback.

 err := sub.Receive(context.Background(), func(ctx context.Context, m *Message) {
	log.Printf("Got message: %s", m.Data)
	m.Ack()
 })
 if err != nil {
	// Handle error.
 }

The callback is invoked concurrently by multiple goroutines, maximizing
throughput. To terminate a call to Receive, cancel its context.

Once client code has processed the message, it must call Message.Ack or
Message.Nack; otherwise the message will eventually be redelivered. Ack/Nack
MUST be called within the Receive handler function, and not from a goroutine.
Otherwise, flow control (e.g. ReceiveSettings.MaxOutstandingMessages) will
not be respected, and messages can get orphaned when cancelling Receive.

If the client cannot or doesn't want to process the message, it can call Message.Nack
to speed redelivery. For more information and configuration options, see
"Ack Deadlines" below.

Note: It is possible for Messages to be redelivered even if Message.Ack has
been called. Client code must be robust to multiple deliveries of messages.

Note: This uses pubsub's streaming pull feature. This feature has properties that
may be surprising. Please take a look at https://cloud.google.com/pubsub/docs/pull#streamingpull
for more details on how streaming pull behaves compared to the synchronous
pull method.

Streams Management

Streams used for streaming pull are managed by the gRPC connection pool setting.
By default, the number of connections in the pool is min(4,GOMAXPROCS).

If you have 4 or more CPU cores, the default setting allows 400 streams which is still a good default for most cases.
If you want to have more open streams (such as for low CPU core machines), you should pass in the grpc option as described below:

 opts := []option.ClientOption{
	option.WithGRPCConnectionPool(8),
 }
 client, err := pubsub.NewClient(ctx, projID, opts...)

Ack Deadlines

The default pubsub deadlines are suitable for most use cases, but may be
overridden. This section describes the tradeoffs that should be considered
when overriding the defaults.

Behind the scenes, each message returned by the Pub/Sub server has an
associated lease, known as an "ack deadline". Unless a message is
acknowledged within the ack deadline, or the client requests that
the ack deadline be extended, the message will become eligible for redelivery.

As a convenience, the pubsub client will automatically extend deadlines until
either:
  - Message.Ack or Message.Nack is called, or
  - The "MaxExtension" duration elapses from the time the message is fetched from
    the server. This defaults to 60m.

Ack deadlines are extended periodically by the client. The initial ack
deadline given to messages is based on the subscription's AckDeadline property,
which defaults to 10s. The period between extensions, as well as the
length of the extension, automatically adjusts based on the time it takes the
subscriber application to ack messages (based on the 99th percentile of ack latency).
By default, this extension period is capped at 10m, but this limit can be configured
by the "MaxExtensionPeriod" setting. This has the effect that subscribers that process
messages quickly have their message ack deadlines extended for a short amount, whereas
subscribers that process message slowly have their message ack deadlines extended
for a large amount. The net effect is fewer RPCs sent from the client library.

For example, consider a subscriber that takes 3 minutes to process each message.
Since the library has already recorded several 3-minute "ack latencies"s in a
percentile distribution, future message extensions are sent with a value of 3
minutes, every 3 minutes. Suppose the application crashes 5 seconds after the
library sends such an extension: the Pub/Sub server would wait the remaining
2m55s before re-sending the messages out to other subscribers.

Please note that by default, the client library does not use the subscription's
AckDeadline for the MaxExtension value. To enforce the subscription's AckDeadline,
set MaxExtension to the subscription's AckDeadline:

	cfg, err := sub.Config(ctx)
	if err != nil {
		// TODO: handle err
	}

	sub.ReceiveSettings.MaxExtension = cfg.AckDeadline

Slow Message Processing

For use cases where message processing exceeds 30 minutes, we recommend using
the base client in a pull model, since long-lived streams are periodically killed
by firewalls. See the example at https://godoc.org/cloud.google.com/go/pubsub/apiv1#example-SubscriberClient-Pull-LengthyClientProcessing

Emulator

To use an emulator with this library, you can set the PUBSUB_EMULATOR_HOST
environment variable to the address at which your emulator is running. This will
send requests to that address instead of to Cloud Pub/Sub. You can then create
and use a client as usual:

	// Set PUBSUB_EMULATOR_HOST environment variable.
	err := os.Setenv("PUBSUB_EMULATOR_HOST", "localhost:9000")
	if err != nil {
		// TODO: Handle error.
	}
	// Create client as usual.
	client, err := pubsub.NewClient(ctx, "my-project-id")
	if err != nil {
		// TODO: Handle error.
	}
	defer client.Close()

*/
package pubsub // import "cloud.google.com/go/pubsub"
<|endoftext|>


<commit_before>package countmin

import (
	"hash/fnv"
	"math"
	"math/rand"
)

const MAXINT64 = 1<<63 - 1

type CountMin struct {
	depth      int
	width      int
	size       int64
	eps        float64
	confidence float64
	table      [][]int64
	hashes     []int64
}

func New(depth, width int) *CountMin {
	cm := &CountMin{
		depth:      depth,
		width:      width,
		eps:        2.0 / float64(width),
		confidence: 1.0 - 1.0/math.Pow(2, float64(depth)),
	}
	cm.initTable()
	return cm
}

func NewWithEpsCount(confidence, eps float64) *CountMin {
	if confidence >= 1.0 {
		confidence = 0.99999
	}
	cm := &CountMin{
		eps:        eps,
		confidence: confidence,
		width:      int(math.Ceil(float64(2.0) / eps)),
		depth:      int(math.Ceil(-math.Log(1-confidence) / math.Log(2))),
	}
	cm.initTable()
	return cm
}

func (cm *CountMin) initTable() {
	cm.table = make([][]int64, cm.depth)
	for i, _ := range cm.table {
		cm.table[i] = make([]int64, cm.width)
	}
	cm.hashes = make([]int64, cm.depth)
	for i, _ := range cm.hashes {
		cm.hashes[i] = rand.Int63()
	}
}

func (cm *CountMin) RelativeError() float64 {
	return cm.eps
}

func (cm *CountMin) Confidence() float64 {
	return cm.confidence
}

func (cm *CountMin) Size() int64 {
	return cm.size
}

func (cm *CountMin) Add(item []byte, count int64) {
	if count < 0 {
		return
	}
	hashed := cm.hasher(item)
	for i := 0; i < cm.depth; i++ {
		cm.table[i][hashed[i]] += count
	}
	cm.size += count
}

func (cm *CountMin) Count(item []byte) int64 {
	var answer int64 = MAXINT64
	var val int64
	hashed := cm.hasher(item)
	for i := 0; i < cm.depth; i++ {
		val = cm.table[i][hashed[i]]
		if val < answer {
			answer = val
		}
	}
	return answer
}

func (cm *CountMin) hasher(item []byte) []int64 {
	total := cm.depth
	max := int64(cm.width)
	result := make([]int64, total)
	hash := fnv.New32a()
	hash.Write(item)
	sum := int64(hash.Sum32())
	for i := 0; i < total; i++ {
		result[i] = int64(math.Abs(float64((sum * int64(i)) % max)))
	}
	return result
}
<commit_msg>Go lint<commit_after>package countmin

import (
	"hash/fnv"
	"math"
	"math/rand"
)

// Max int 64 size
const MAXINT64 = 1<<63 - 1

// CountMin holds count table and hashes for calculating
// frequencies from a given input (usually a stream of data)
type CountMin struct {
	depth      int
	width      int
	size       int64
	eps        float64
	confidence float64
	table      [][]int64
	hashes     []int64
}

// New CountMin given a depth and width of the frequency
// table to use. Returns the newly created CountMin
func New(depth, width int) *CountMin {
	cm := &CountMin{
		depth:      depth,
		width:      width,
		eps:        2.0 / float64(width),
		confidence: 1.0 - 1.0/math.Pow(2, float64(depth)),
	}
	cm.initTable()
	return cm
}

// NewWithEpsCount initializes a CountMin given a confidence
// to ensure between 0 < 1.0 and an epsilon between 0 and 1
// with a smaller value for higher precision as it affects
// the width of the table
func NewWithEpsCount(confidence, eps float64) *CountMin {
	if confidence >= 1.0 {
		confidence = 0.99999
	}
	cm := &CountMin{
		eps:        eps,
		confidence: confidence,
		width:      int(math.Ceil(float64(2.0) / eps)),
		depth:      int(math.Ceil(-math.Log(1-confidence) / math.Log(2))),
	}
	cm.initTable()
	return cm
}

func (cm *CountMin) initTable() {
	cm.table = make([][]int64, cm.depth)
	for i := range cm.table {
		cm.table[i] = make([]int64, cm.width)
	}
	cm.hashes = make([]int64, cm.depth)
	for i := range cm.hashes {
		cm.hashes[i] = rand.Int63()
	}
}

// RelativeError returns epsilon of CountMin
func (cm *CountMin) RelativeError() float64 {
	return cm.eps
}

// Confidence returns confidence of CountMin
func (cm *CountMin) Confidence() float64 {
	return cm.confidence
}

// Size returns total size of CountMin
func (cm *CountMin) Size() int64 {
	return cm.size
}

// Add inserts an item in bytes and a count associated
// which will also increment size of CountMin
func (cm *CountMin) Add(item []byte, count int64) {
	if count < 0 {
		return
	}
	hashed := cm.hasher(item)
	for i := 0; i < cm.depth; i++ {
		cm.table[i][hashed[i]] += count
	}
	cm.size += count
}

// Count gets an associated total count for an item
// in the CountMin
func (cm *CountMin) Count(item []byte) int64 {
	var answer int64 = MAXINT64
	var val int64
	hashed := cm.hasher(item)
	for i := 0; i < cm.depth; i++ {
		val = cm.table[i][hashed[i]]
		if val < answer {
			answer = val
		}
	}
	return answer
}

func (cm *CountMin) hasher(item []byte) []int64 {
	total := cm.depth
	max := int64(cm.width)
	result := make([]int64, total)
	hash := fnv.New32a()
	hash.Write(item)
	sum := int64(hash.Sum32())
	for i := 0; i < total; i++ {
		result[i] = int64(math.Abs(float64((sum * int64(i)) % max)))
	}
	return result
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2013, 2014 Conformal Systems LLC <info@conformal.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package main

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/conformal/btcscript"
	"github.com/conformal/btcutil"
	"github.com/conformal/btcwallet/tx"
	"github.com/conformal/btcwallet/wallet"
	"github.com/conformal/btcwire"
	"sort"
	"sync"
	"time"
)

// ErrInsufficientFunds represents an error where there are not enough
// funds from unspent tx outputs for a wallet to create a transaction.
var ErrInsufficientFunds = errors.New("insufficient funds")

// ErrUnknownBitcoinNet represents an error where the parsed or
// requested bitcoin network is invalid (neither mainnet nor testnet).
var ErrUnknownBitcoinNet = errors.New("unknown bitcoin network")

// ErrNonPositiveAmount represents an error where a bitcoin amount is
// not positive (either negative, or zero).
var ErrNonPositiveAmount = errors.New("amount is not positive")

// ErrNegativeFee represents an error where a fee is erroneously
// negative.
var ErrNegativeFee = errors.New("fee is negative")

// minTxFee is the default minimum transation fee (0.0001 BTC,
// measured in satoshis) added to transactions requiring a fee.
const minTxFee = 10000

// TxFeeIncrement represents the global transaction fee per KB of Tx
// added to newly-created transactions and sent as a reward to the block
// miner.  i is measured in satoshis.
var TxFeeIncrement = struct {
	sync.Mutex
	i int64
}{
	i: minTxFee,
}

type CreatedTx struct {
	tx         *btcutil.Tx
	time       time.Time
	changeAddr btcutil.Address
}

// ByAmount defines the methods needed to satisify sort.Interface to
// sort a slice of Utxos by their amount.
type ByAmount []*tx.RecvTxOut

func (u ByAmount) Len() int {
	return len(u)
}

func (u ByAmount) Less(i, j int) bool {
	return u[i].Value() < u[j].Value()
}

func (u ByAmount) Swap(i, j int) {
	u[i], u[j] = u[j], u[i]
}

// selectInputs selects the minimum number possible of unspent
// outputs to use to create a new transaction that spends amt satoshis.
// Previous outputs with less than minconf confirmations are ignored.  btcout
// is the total number of satoshis which would be spent by the combination
// of all selected previous outputs.  err will equal ErrInsufficientFunds if there
// are not enough unspent outputs to spend amt.
func selectInputs(utxos []*tx.RecvTxOut, amt int64,
	minconf int) (selected []*tx.RecvTxOut, btcout int64, err error) {

	bs, err := GetCurBlock()
	if err != nil {
		return nil, 0, err
	}

	// Create list of eligible unspent previous outputs to use as tx
	// inputs, and sort by the amount in reverse order so a minimum number
	// of inputs is needed.
	eligible := make([]*tx.RecvTxOut, 0, len(utxos))
	for _, utxo := range utxos {
		if confirmed(minconf, utxo.Height(), bs.Height) {
			// Coinbase transactions must have 100 confirmations before
			// they may be spent.
			if utxo.IsCoinbase() && bs.Height-utxo.Height()+1 < 100 {
				continue
			}
			eligible = append(eligible, utxo)
		}
	}
	sort.Sort(sort.Reverse(ByAmount(eligible)))

	// Iterate throguh eligible transactions, appending to outputs and
	// increasing btcout.  This is finished when btcout is greater than the
	// requested amt to spend.
	for _, e := range eligible {
		selected = append(selected, e)
		btcout += e.Value()
		if btcout >= amt {
			return selected, btcout, nil
		}
	}
	if btcout < amt {
		return nil, 0, ErrInsufficientFunds
	}

	return selected, btcout, nil
}

// txToPairs creates a raw transaction sending the amounts for each
// address/amount pair and fee to each address and the miner.  minconf
// specifies the minimum number of confirmations required before an
// unspent output is eligible for spending. Leftover input funds not sent
// to addr or as a fee for the miner are sent to a newly generated
// address. If change is needed to return funds back to an owned
// address, changeUtxo will point to a unconfirmed (height = -1, zeroed
// block hash) Utxo.  ErrInsufficientFunds is returned if there are not
// enough eligible unspent outputs to create the transaction.
func (a *Account) txToPairs(pairs map[string]int64, minconf int) (*CreatedTx, error) {
	// Wallet must be unlocked to compose transaction.
	if a.IsLocked() {
		return nil, wallet.ErrWalletLocked
	}

	// Create a new transaction which will include all input scripts.
	msgtx := btcwire.NewMsgTx()

	// Calculate minimum amount needed for inputs.
	var amt int64
	for _, v := range pairs {
		// Error out if any amount is negative.
		if v <= 0 {
			return nil, ErrNonPositiveAmount
		}
		amt += v
	}

	// Add outputs to new tx.
	for addrStr, amt := range pairs {
		addr, err := btcutil.DecodeAddress(addrStr, cfg.Net())
		if err != nil {
			return nil, fmt.Errorf("cannot decode address: %s", err)
		}

		// Add output to spend amt to addr.
		pkScript, err := btcscript.PayToAddrScript(addr)
		if err != nil {
			return nil, fmt.Errorf("cannot create txout script: %s", err)
		}
		txout := btcwire.NewTxOut(int64(amt), pkScript)
		msgtx.AddTxOut(txout)
	}

	// Get current block's height and hash.
	bs, err := GetCurBlock()
	if err != nil {
		return nil, err
	}

	// Make a copy of msgtx before any inputs are added.  This will be
	// used as a starting point when trying a fee and starting over with
	// a higher fee if not enough was originally chosen.
	txNoInputs := msgtx.Copy()

	var selectedInputs []*tx.RecvTxOut
	// These are nil/zeroed until a change address is needed, and reused
	// again in case a change utxo has already been chosen.
	var changeAddr btcutil.Address

	// Get the number of satoshis to increment fee by when searching for
	// the minimum tx fee needed.
	fee := int64(0)
	for {
		msgtx = txNoInputs.Copy()

		// Select unspent outputs to be used in transaction based on the amount
		// neededing to sent, and the current fee estimation.
		inputs, btcin, err := selectInputs(a.TxStore.UnspentOutputs(),
			amt+fee, minconf)
		if err != nil {
			return nil, err
		}

		// Check if there are leftover unspent outputs, and return coins back to
		// a new address we own.
		//
		// TODO: change needs to be inserted into a random txout index, or else
		// this is a privacy risk.
		change := btcin - amt - fee
		if change > 0 {
			// Get a new change address if one has not already been found.
			if changeAddr == nil {
				changeAddr, err = a.ChangeAddress(&bs, cfg.KeypoolSize)
				if err != nil {
					return nil, fmt.Errorf("failed to get next address: %s", err)
				}

				// Mark change address as belonging to this account.
				AcctMgr.MarkAddressForAccount(changeAddr, a)
			}

			// Spend change.
			pkScript, err := btcscript.PayToAddrScript(changeAddr)
			if err != nil {
				return nil, fmt.Errorf("cannot create txout script: %s", err)
			}
			msgtx.AddTxOut(btcwire.NewTxOut(int64(change), pkScript))
		}

		// Selected unspent outputs become new transaction's inputs.
		for _, ip := range inputs {
			msgtx.AddTxIn(btcwire.NewTxIn(ip.OutPoint(), nil))
		}
		for i, input := range inputs {
			_, addrs, _, _ := input.Addresses(cfg.Net())
			if len(addrs) != 1 {
				continue
			}
			apkh, ok := addrs[0].(*btcutil.AddressPubKeyHash)
			if !ok {
				continue // don't handle inputs to this yes
			}

			ai, err := a.Address(apkh)
			if err != nil {
				return nil, fmt.Errorf("cannot get address info: %v", err)
			}

			pka := ai.(wallet.PubKeyAddress)

			privkey, err := pka.PrivKey()
			if err == wallet.ErrWalletLocked {
				return nil, wallet.ErrWalletLocked
			} else if err != nil {
				return nil, fmt.Errorf("cannot get address key: %v", err)
			}

			sigscript, err := btcscript.SignatureScript(msgtx, i,
				input.PkScript(), btcscript.SigHashAll, privkey,
				ai.Compressed())
			if err != nil {
				return nil, fmt.Errorf("cannot create sigscript: %s", err)
			}
			msgtx.TxIn[i].SignatureScript = sigscript
		}

		noFeeAllowed := false
		if !cfg.DisallowFree {
			noFeeAllowed = allowFree(bs.Height, inputs, msgtx.SerializeSize())
		}
		if minFee := minimumFee(msgtx, noFeeAllowed); fee < minFee {
			fee = minFee
		} else {
			selectedInputs = inputs
			break
		}
	}

	// Validate msgtx before returning the raw transaction.
	flags := btcscript.ScriptCanonicalSignatures
	bip16 := time.Now().After(btcscript.Bip16Activation)
	if bip16 {
		flags |= btcscript.ScriptBip16
	}
	for i, txin := range msgtx.TxIn {
		engine, err := btcscript.NewScript(txin.SignatureScript,
			selectedInputs[i].PkScript(), i, msgtx, flags)
		if err != nil {
			return nil, fmt.Errorf("cannot create script engine: %s", err)
		}
		if err = engine.Execute(); err != nil {
			return nil, fmt.Errorf("cannot validate transaction: %s", err)
		}
	}

	buf := new(bytes.Buffer)
	msgtx.BtcEncode(buf, btcwire.ProtocolVersion)
	info := &CreatedTx{
		tx:         btcutil.NewTx(msgtx),
		time:       time.Now(),
		changeAddr: changeAddr,
	}
	return info, nil
}

// minimumFee calculates the minimum fee required for a transaction.
// If allowFree is true, a fee may be zero so long as the entire
// transaction has a serialized length less than 1 kilobyte
// and none of the outputs contain a value less than 1 bitcent.
// Otherwise, the fee will be calculated using TxFeeIncrement,
// incrementing the fee for each kilobyte of transaction.
func minimumFee(tx *btcwire.MsgTx, allowFree bool) int64 {
	txLen := tx.SerializeSize()
	TxFeeIncrement.Lock()
	incr := TxFeeIncrement.i
	TxFeeIncrement.Unlock()
	fee := int64(1+txLen/1000) * incr

	if allowFree && txLen < 1000 {
		fee = 0
	}

	if fee < incr {
		for _, txOut := range tx.TxOut {
			if txOut.Value < btcutil.SatoshiPerBitcent {
				return incr
			}
		}
	}

	if fee < 0 || fee > btcutil.MaxSatoshi {
		fee = btcutil.MaxSatoshi
	}

	return fee
}

// allowFree calculates the transaction priority and checks that the
// priority reaches a certain threshhold.  If the threshhold is
// reached, a free transaction fee is allowed.
func allowFree(curHeight int32, txouts []*tx.RecvTxOut, txSize int) bool {
	const blocksPerDayEstimate = 144
	const txSizeEstimate = 250

	var weightedSum int64
	for _, txout := range txouts {
		depth := chainDepth(txout.Height(), curHeight)
		weightedSum += txout.Value() * int64(depth)
	}
	priority := float64(weightedSum) / float64(txSize)
	return priority > float64(btcutil.SatoshiPerBitcoin)*blocksPerDayEstimate/txSizeEstimate
}

// chainDepth returns the chaindepth of a target given the current
// blockchain height.
func chainDepth(target, current int32) int32 {
	if target == -1 {
		// target is not yet in a block.
		return 0
	}

	// target is in a block.
	return current - target + 1
}
<commit_msg>Use confirms func to find number of confirmations.<commit_after>/*
 * Copyright (c) 2013, 2014 Conformal Systems LLC <info@conformal.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package main

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/conformal/btcscript"
	"github.com/conformal/btcutil"
	"github.com/conformal/btcwallet/tx"
	"github.com/conformal/btcwallet/wallet"
	"github.com/conformal/btcwire"
	"sort"
	"sync"
	"time"
)

// ErrInsufficientFunds represents an error where there are not enough
// funds from unspent tx outputs for a wallet to create a transaction.
var ErrInsufficientFunds = errors.New("insufficient funds")

// ErrUnknownBitcoinNet represents an error where the parsed or
// requested bitcoin network is invalid (neither mainnet nor testnet).
var ErrUnknownBitcoinNet = errors.New("unknown bitcoin network")

// ErrNonPositiveAmount represents an error where a bitcoin amount is
// not positive (either negative, or zero).
var ErrNonPositiveAmount = errors.New("amount is not positive")

// ErrNegativeFee represents an error where a fee is erroneously
// negative.
var ErrNegativeFee = errors.New("fee is negative")

// minTxFee is the default minimum transation fee (0.0001 BTC,
// measured in satoshis) added to transactions requiring a fee.
const minTxFee = 10000

// TxFeeIncrement represents the global transaction fee per KB of Tx
// added to newly-created transactions and sent as a reward to the block
// miner.  i is measured in satoshis.
var TxFeeIncrement = struct {
	sync.Mutex
	i int64
}{
	i: minTxFee,
}

type CreatedTx struct {
	tx         *btcutil.Tx
	time       time.Time
	changeAddr btcutil.Address
}

// ByAmount defines the methods needed to satisify sort.Interface to
// sort a slice of Utxos by their amount.
type ByAmount []*tx.RecvTxOut

func (u ByAmount) Len() int {
	return len(u)
}

func (u ByAmount) Less(i, j int) bool {
	return u[i].Value() < u[j].Value()
}

func (u ByAmount) Swap(i, j int) {
	u[i], u[j] = u[j], u[i]
}

// selectInputs selects the minimum number possible of unspent
// outputs to use to create a new transaction that spends amt satoshis.
// Previous outputs with less than minconf confirmations are ignored.  btcout
// is the total number of satoshis which would be spent by the combination
// of all selected previous outputs.  err will equal ErrInsufficientFunds if there
// are not enough unspent outputs to spend amt.
func selectInputs(utxos []*tx.RecvTxOut, amt int64,
	minconf int) (selected []*tx.RecvTxOut, btcout int64, err error) {

	bs, err := GetCurBlock()
	if err != nil {
		return nil, 0, err
	}

	// Create list of eligible unspent previous outputs to use as tx
	// inputs, and sort by the amount in reverse order so a minimum number
	// of inputs is needed.
	eligible := make([]*tx.RecvTxOut, 0, len(utxos))
	for _, utxo := range utxos {
		if confirmed(minconf, utxo.Height(), bs.Height) {
			// Coinbase transactions must have 100 confirmations before
			// they may be spent.
			if utxo.IsCoinbase() && confirms(utxo.Height(), bs.Height) < 100 {
				continue
			}
			eligible = append(eligible, utxo)
		}
	}
	sort.Sort(sort.Reverse(ByAmount(eligible)))

	// Iterate throguh eligible transactions, appending to outputs and
	// increasing btcout.  This is finished when btcout is greater than the
	// requested amt to spend.
	for _, e := range eligible {
		selected = append(selected, e)
		btcout += e.Value()
		if btcout >= amt {
			return selected, btcout, nil
		}
	}
	if btcout < amt {
		return nil, 0, ErrInsufficientFunds
	}

	return selected, btcout, nil
}

// txToPairs creates a raw transaction sending the amounts for each
// address/amount pair and fee to each address and the miner.  minconf
// specifies the minimum number of confirmations required before an
// unspent output is eligible for spending. Leftover input funds not sent
// to addr or as a fee for the miner are sent to a newly generated
// address. If change is needed to return funds back to an owned
// address, changeUtxo will point to a unconfirmed (height = -1, zeroed
// block hash) Utxo.  ErrInsufficientFunds is returned if there are not
// enough eligible unspent outputs to create the transaction.
func (a *Account) txToPairs(pairs map[string]int64, minconf int) (*CreatedTx, error) {
	// Wallet must be unlocked to compose transaction.
	if a.IsLocked() {
		return nil, wallet.ErrWalletLocked
	}

	// Create a new transaction which will include all input scripts.
	msgtx := btcwire.NewMsgTx()

	// Calculate minimum amount needed for inputs.
	var amt int64
	for _, v := range pairs {
		// Error out if any amount is negative.
		if v <= 0 {
			return nil, ErrNonPositiveAmount
		}
		amt += v
	}

	// Add outputs to new tx.
	for addrStr, amt := range pairs {
		addr, err := btcutil.DecodeAddress(addrStr, cfg.Net())
		if err != nil {
			return nil, fmt.Errorf("cannot decode address: %s", err)
		}

		// Add output to spend amt to addr.
		pkScript, err := btcscript.PayToAddrScript(addr)
		if err != nil {
			return nil, fmt.Errorf("cannot create txout script: %s", err)
		}
		txout := btcwire.NewTxOut(int64(amt), pkScript)
		msgtx.AddTxOut(txout)
	}

	// Get current block's height and hash.
	bs, err := GetCurBlock()
	if err != nil {
		return nil, err
	}

	// Make a copy of msgtx before any inputs are added.  This will be
	// used as a starting point when trying a fee and starting over with
	// a higher fee if not enough was originally chosen.
	txNoInputs := msgtx.Copy()

	var selectedInputs []*tx.RecvTxOut
	// These are nil/zeroed until a change address is needed, and reused
	// again in case a change utxo has already been chosen.
	var changeAddr btcutil.Address

	// Get the number of satoshis to increment fee by when searching for
	// the minimum tx fee needed.
	fee := int64(0)
	for {
		msgtx = txNoInputs.Copy()

		// Select unspent outputs to be used in transaction based on the amount
		// neededing to sent, and the current fee estimation.
		inputs, btcin, err := selectInputs(a.TxStore.UnspentOutputs(),
			amt+fee, minconf)
		if err != nil {
			return nil, err
		}

		// Check if there are leftover unspent outputs, and return coins back to
		// a new address we own.
		//
		// TODO: change needs to be inserted into a random txout index, or else
		// this is a privacy risk.
		change := btcin - amt - fee
		if change > 0 {
			// Get a new change address if one has not already been found.
			if changeAddr == nil {
				changeAddr, err = a.ChangeAddress(&bs, cfg.KeypoolSize)
				if err != nil {
					return nil, fmt.Errorf("failed to get next address: %s", err)
				}

				// Mark change address as belonging to this account.
				AcctMgr.MarkAddressForAccount(changeAddr, a)
			}

			// Spend change.
			pkScript, err := btcscript.PayToAddrScript(changeAddr)
			if err != nil {
				return nil, fmt.Errorf("cannot create txout script: %s", err)
			}
			msgtx.AddTxOut(btcwire.NewTxOut(int64(change), pkScript))
		}

		// Selected unspent outputs become new transaction's inputs.
		for _, ip := range inputs {
			msgtx.AddTxIn(btcwire.NewTxIn(ip.OutPoint(), nil))
		}
		for i, input := range inputs {
			_, addrs, _, _ := input.Addresses(cfg.Net())
			if len(addrs) != 1 {
				continue
			}
			apkh, ok := addrs[0].(*btcutil.AddressPubKeyHash)
			if !ok {
				continue // don't handle inputs to this yes
			}

			ai, err := a.Address(apkh)
			if err != nil {
				return nil, fmt.Errorf("cannot get address info: %v", err)
			}

			pka := ai.(wallet.PubKeyAddress)

			privkey, err := pka.PrivKey()
			if err == wallet.ErrWalletLocked {
				return nil, wallet.ErrWalletLocked
			} else if err != nil {
				return nil, fmt.Errorf("cannot get address key: %v", err)
			}

			sigscript, err := btcscript.SignatureScript(msgtx, i,
				input.PkScript(), btcscript.SigHashAll, privkey,
				ai.Compressed())
			if err != nil {
				return nil, fmt.Errorf("cannot create sigscript: %s", err)
			}
			msgtx.TxIn[i].SignatureScript = sigscript
		}

		noFeeAllowed := false
		if !cfg.DisallowFree {
			noFeeAllowed = allowFree(bs.Height, inputs, msgtx.SerializeSize())
		}
		if minFee := minimumFee(msgtx, noFeeAllowed); fee < minFee {
			fee = minFee
		} else {
			selectedInputs = inputs
			break
		}
	}

	// Validate msgtx before returning the raw transaction.
	flags := btcscript.ScriptCanonicalSignatures
	bip16 := time.Now().After(btcscript.Bip16Activation)
	if bip16 {
		flags |= btcscript.ScriptBip16
	}
	for i, txin := range msgtx.TxIn {
		engine, err := btcscript.NewScript(txin.SignatureScript,
			selectedInputs[i].PkScript(), i, msgtx, flags)
		if err != nil {
			return nil, fmt.Errorf("cannot create script engine: %s", err)
		}
		if err = engine.Execute(); err != nil {
			return nil, fmt.Errorf("cannot validate transaction: %s", err)
		}
	}

	buf := new(bytes.Buffer)
	msgtx.BtcEncode(buf, btcwire.ProtocolVersion)
	info := &CreatedTx{
		tx:         btcutil.NewTx(msgtx),
		time:       time.Now(),
		changeAddr: changeAddr,
	}
	return info, nil
}

// minimumFee calculates the minimum fee required for a transaction.
// If allowFree is true, a fee may be zero so long as the entire
// transaction has a serialized length less than 1 kilobyte
// and none of the outputs contain a value less than 1 bitcent.
// Otherwise, the fee will be calculated using TxFeeIncrement,
// incrementing the fee for each kilobyte of transaction.
func minimumFee(tx *btcwire.MsgTx, allowFree bool) int64 {
	txLen := tx.SerializeSize()
	TxFeeIncrement.Lock()
	incr := TxFeeIncrement.i
	TxFeeIncrement.Unlock()
	fee := int64(1+txLen/1000) * incr

	if allowFree && txLen < 1000 {
		fee = 0
	}

	if fee < incr {
		for _, txOut := range tx.TxOut {
			if txOut.Value < btcutil.SatoshiPerBitcent {
				return incr
			}
		}
	}

	if fee < 0 || fee > btcutil.MaxSatoshi {
		fee = btcutil.MaxSatoshi
	}

	return fee
}

// allowFree calculates the transaction priority and checks that the
// priority reaches a certain threshhold.  If the threshhold is
// reached, a free transaction fee is allowed.
func allowFree(curHeight int32, txouts []*tx.RecvTxOut, txSize int) bool {
	const blocksPerDayEstimate = 144
	const txSizeEstimate = 250

	var weightedSum int64
	for _, txout := range txouts {
		depth := chainDepth(txout.Height(), curHeight)
		weightedSum += txout.Value() * int64(depth)
	}
	priority := float64(weightedSum) / float64(txSize)
	return priority > float64(btcutil.SatoshiPerBitcoin)*blocksPerDayEstimate/txSizeEstimate
}

// chainDepth returns the chaindepth of a target given the current
// blockchain height.
func chainDepth(target, current int32) int32 {
	if target == -1 {
		// target is not yet in a block.
		return 0
	}

	// target is in a block.
	return current - target + 1
}
<|endoftext|>


<commit_before>package cli

import (
	"io/ioutil"
	"os"
	"path"
	"path/filepath"

	log "github.com/Sirupsen/logrus"
	"github.com/bitrise-io/envman/envman"
	"github.com/codegangsta/cli"
)

const (
	defaultEnvStoreName string = ".envstore.yml"
)

var (
	stdinValue string
)

func isPipedData() bool {
	if stat, err := os.Stdin.Stat(); err != nil {
		return false
	} else if (stat.Mode() & os.ModeCharDevice) == 0 {
		return true
	}
	return false
}

func envStorePathInCurrentDir() (string, error) {
	return filepath.Abs(path.Join("./", defaultEnvStoreName))
}

func parseTool(c *cli.Context) bool {
	if c.IsSet(ToolKey) {
		return c.Bool(ToolKey)
	}
	return false
}

func before(c *cli.Context) error {
	// Log level
	if logLevel, err := log.ParseLevel(c.String(LogLevelKey)); err != nil {
		log.Fatal("[BITRISE_CLI] - Failed to parse log level:", err)
	} else {
		log.SetLevel(logLevel)
	}

	// Befor parsing cli, and running command
	// we need to decide wich path will be used by envman
	envman.CurrentEnvStoreFilePath = c.String(PathKey)
	if envman.CurrentEnvStoreFilePath == "" {
		if path, err := envStorePathInCurrentDir(); err != nil {
			log.Fatal("[ENVMAN] - Failed to set envman work path in current dir:", err)
		} else {
			envman.CurrentEnvStoreFilePath = path
		}
	}

	envman.ToolMode = parseTool(c)
	if envman.ToolMode {
		log.Info("[ENVMAN] - Tool mode on")
	}

	return nil
}

// Run the Envman CLI.
func Run() {
	// Read piped data
	if isPipedData() {
		if bytes, err := ioutil.ReadAll(os.Stdin); err != nil {
			log.Error("[ENVMAN] - Failed to read stdin:", err)
		} else if len(bytes) > 0 {
			stdinValue = string(bytes)
		}
	}

	// Parse cl
	app := cli.NewApp()
	app.Name = path.Base(os.Args[0])
	app.Usage = "Environment variable manager"
	app.Version = "0.0.7"

	app.Author = ""
	app.Email = ""

	app.Before = before

	app.Flags = flags
	app.Commands = commands

	if err := app.Run(os.Args); err != nil {
		log.Fatal("[ENVMAN] - Finished:", err)
	}
}
<commit_msg>version number: 0.9.1 + changed the format of the -version print (to less verbose, only print the version number)<commit_after>package cli

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"

	log "github.com/Sirupsen/logrus"
	"github.com/bitrise-io/envman/envman"
	"github.com/codegangsta/cli"
)

const (
	defaultEnvStoreName string = ".envstore.yml"
)

var (
	stdinValue string
)

func isPipedData() bool {
	if stat, err := os.Stdin.Stat(); err != nil {
		return false
	} else if (stat.Mode() & os.ModeCharDevice) == 0 {
		return true
	}
	return false
}

func envStorePathInCurrentDir() (string, error) {
	return filepath.Abs(path.Join("./", defaultEnvStoreName))
}

func parseTool(c *cli.Context) bool {
	if c.IsSet(ToolKey) {
		return c.Bool(ToolKey)
	}
	return false
}

func before(c *cli.Context) error {
	// Log level
	if logLevel, err := log.ParseLevel(c.String(LogLevelKey)); err != nil {
		log.Fatal("[BITRISE_CLI] - Failed to parse log level:", err)
	} else {
		log.SetLevel(logLevel)
	}

	// Befor parsing cli, and running command
	// we need to decide wich path will be used by envman
	envman.CurrentEnvStoreFilePath = c.String(PathKey)
	if envman.CurrentEnvStoreFilePath == "" {
		if path, err := envStorePathInCurrentDir(); err != nil {
			log.Fatal("[ENVMAN] - Failed to set envman work path in current dir:", err)
		} else {
			envman.CurrentEnvStoreFilePath = path
		}
	}

	envman.ToolMode = parseTool(c)
	if envman.ToolMode {
		log.Info("[ENVMAN] - Tool mode on")
	}

	return nil
}

func printVersion(c *cli.Context) {
	fmt.Fprintf(c.App.Writer, "%v\n", c.App.Version)
}

// Run the Envman CLI.
func Run() {
	// Read piped data
	if isPipedData() {
		if bytes, err := ioutil.ReadAll(os.Stdin); err != nil {
			log.Error("[ENVMAN] - Failed to read stdin:", err)
		} else if len(bytes) > 0 {
			stdinValue = string(bytes)
		}
	}

	// Parse cl
	cli.VersionPrinter = printVersion

	app := cli.NewApp()
	app.Name = path.Base(os.Args[0])
	app.Usage = "Environment variable manager"
	app.Version = "0.9.1"

	app.Author = ""
	app.Email = ""

	app.Before = before

	app.Flags = flags
	app.Commands = commands

	if err := app.Run(os.Args); err != nil {
		log.Fatal("[ENVMAN] - Finished:", err)
	}
}
<|endoftext|>


<commit_before>package cli

import (
	"fmt"
	"github.com/mehrdadrad/mylg/banner"
	"gopkg.in/readline.v1"
	"strings"
)

type Readline struct {
	instance  *readline.Instance
	completer *readline.PrefixCompleter
	prompt    string
	next      chan struct{}
}

func Init(prompt string) *Readline {
	var (
		r         Readline
		err       error
		completer = readline.NewPrefixCompleter(
			readline.PcItem("ping"),
			readline.PcItem("connect"),
			readline.PcItem("node"),
			readline.PcItem("local"),
			readline.PcItem("asn"),
			readline.PcItem("help"),
			readline.PcItem("exit"),
		)
	)
	r.completer = completer
	r.instance, err = readline.NewEx(&readline.Config{
		Prompt:          prompt + "> ",
		HistoryFile:     "/tmp/myping",
		InterruptPrompt: "^C",
		EOFPrompt:       "exit",
		AutoComplete:    completer,
	})
	if err != nil {
		panic(err)
	}
	banner.Println()
	return &r
}

func (r *Readline) RemoveItemCompleter(pcItem string) {
	child := []readline.PrefixCompleterInterface{}
	for _, p := range r.completer.Children {
		if strings.TrimSpace(string(p.GetName())) != pcItem {
			child = append(child, p)
		}
	}

	r.completer.Children = child

}

func (r *Readline) UpdateCompleter(pcItem string, pcSubItems []string) {
	child := []readline.PrefixCompleterInterface{}
	var pc readline.PrefixCompleter
	for _, p := range r.completer.Children {
		if strings.TrimSpace(string(p.GetName())) == pcItem {
			c := []readline.PrefixCompleterInterface{}
			for _, item := range pcSubItems {
				c = append(c, readline.PcItem(item))
			}
			pc.Name = []rune(pcItem + " ")
			pc.Children = c
			child = append(child, &pc)
		} else {
			child = append(child, p)
		}
	}

	r.completer.Children = child
}

func (r *Readline) SetPrompt(p string) {
	r.prompt = p
	r.instance.SetPrompt(p + "> ")
}

func (r *Readline) GetPrompt() string {
	return r.prompt
}

func (r *Readline) Refresh() {
	r.instance.Refresh()
}

func (r *Readline) SetVim() {
	if !r.instance.IsVimMode() {
		r.instance.SetVimMode(true)
		println("mode changed to vim")
	} else {
		println("mode already is vim")
	}
}

func (r *Readline) SetEmacs() {
	if r.instance.IsVimMode() {
		r.instance.SetVimMode(false)
		println("mode changed to emacs")
	} else {
		println("mode already is emacs")
	}
}

func (r *Readline) Next() {
	r.next <- struct{}{}
}

func (r *Readline) Run(cmd chan<- string, next chan struct{}) {
	r.next = next
	func() {
		for {
			line, err := r.instance.Readline()
			if err != nil { // io.EOF, readline.ErrInterrupt
				break
			}
			cmd <- line
			if _, ok := <-next; !ok {
				break
			}
		}
	}()
}

func (r *Readline) Close(next chan struct{}) {
	r.instance.Close()
}

func (r *Readline) Help() {
	fmt.Println(`Usage:
	The myLG tool developed to troubleshoot networking situations.
	The vi/emacs mode,almost all basic features is supported. press tab to see what options are available.

	connect <provider name>     connects to external looking glass, press tab to see the menu
	node <city/country name>    connects to specific node at current looking glass, press tab to see the available nodes
	local                       back to local
	ping                        ping ip address or domain name
	`)
}
<commit_msg>added comments<commit_after>package cli

import (
	"fmt"
	"github.com/mehrdadrad/mylg/banner"
	"gopkg.in/readline.v1"
	"strings"
)

// Readline structure
type Readline struct {
	instance  *readline.Instance
	completer *readline.PrefixCompleter
	prompt    string
	next      chan struct{}
}

// Initialize items
func Init(prompt string) *Readline {
	var (
		r         Readline
		err       error
		completer = readline.NewPrefixCompleter(
			readline.PcItem("ping"),
			readline.PcItem("connect"),
			readline.PcItem("node"),
			readline.PcItem("local"),
			readline.PcItem("asn"),
			readline.PcItem("help"),
			readline.PcItem("exit"),
		)
	)
	r.completer = completer
	r.instance, err = readline.NewEx(&readline.Config{
		Prompt:          prompt + "> ",
		HistoryFile:     "/tmp/myping",
		InterruptPrompt: "^C",
		EOFPrompt:       "exit",
		AutoComplete:    completer,
	})
	if err != nil {
		panic(err)
	}
	banner.Println()
	return &r
}

// Remove item from completer
func (r *Readline) RemoveItemCompleter(pcItem string) {
	child := []readline.PrefixCompleterInterface{}
	for _, p := range r.completer.Children {
		if strings.TrimSpace(string(p.GetName())) != pcItem {
			child = append(child, p)
		}
	}

	r.completer.Children = child

}

// Add item to completer
func (r *Readline) UpdateCompleter(pcItem string, pcSubItems []string) {
	child := []readline.PrefixCompleterInterface{}
	var pc readline.PrefixCompleter
	for _, p := range r.completer.Children {
		if strings.TrimSpace(string(p.GetName())) == pcItem {
			c := []readline.PrefixCompleterInterface{}
			for _, item := range pcSubItems {
				c = append(c, readline.PcItem(item))
			}
			pc.Name = []rune(pcItem + " ")
			pc.Children = c
			child = append(child, &pc)
		} else {
			child = append(child, p)
		}
	}

	r.completer.Children = child
}

// Set prompt and store it
func (r *Readline) SetPrompt(p string) {
	r.prompt = p
	r.instance.SetPrompt(p + "> ")
}

// Get current prompt
func (r *Readline) GetPrompt() string {
	return r.prompt
}

// Refresh prompt
func (r *Readline) Refresh() {
	r.instance.Refresh()
}

// Set mode to vim
func (r *Readline) SetVim() {
	if !r.instance.IsVimMode() {
		r.instance.SetVimMode(true)
		println("mode changed to vim")
	} else {
		println("mode already is vim")
	}
}

// Set mode to emacs
func (r *Readline) SetEmacs() {
	if r.instance.IsVimMode() {
		r.instance.SetVimMode(false)
		println("mode changed to emacs")
	} else {
		println("mode already is emacs")
	}
}

// Triger to read next line
func (r *Readline) Next() {
	r.next <- struct{}{}
}

// Run the main loop
func (r *Readline) Run(cmd chan<- string, next chan struct{}) {
	r.next = next
	func() {
		for {
			line, err := r.instance.Readline()
			if err != nil { // io.EOF, readline.ErrInterrupt
				break
			}
			cmd <- line
			if _, ok := <-next; !ok {
				break
			}
		}
	}()
}

// Close the readline instance
func (r *Readline) Close(next chan struct{}) {
	r.instance.Close()
}

// Print out the main help
func (r *Readline) Help() {
	fmt.Println(`Usage:
	The myLG tool developed to troubleshoot networking situations.
	The vi/emacs mode,almost all basic features is supported. press tab to see what options are available.

	connect <provider name>     connects to external looking glass, press tab to see the menu
	node <city/country name>    connects to specific node at current looking glass, press tab to see the available nodes
	local                       back to local
	ping                        ping ip address or domain name
	`)
}
<|endoftext|>


<commit_before>package hamming

import (
	"testing"
)

var testCases = []struct {
	expected         int
	strandA, strandB string
	description      string
}{
	{0, "", "", "no difference between empty strands"},
	{2, "AG", "CT", "complete hamming distance for small strands"},
	{0, "A", "A", "no difference between identical strands"},
	{1, "A", "G", "complete distance for single nucleotide strands"},
	{1, "AT", "CT", "small hamming distance"},
	{1, "GGACG", "GGTCG", "small hamming distance in longer strands"},
	{0, "AAAG", "AAA", "ignores extra length on first strand when longer"},
	{0, "AAA", "AAAG", "ignores extra length on second strand when longer"},
	{4, "GATACA", "GCATAA", "large hamming distance"},
	{9, "GGACGGATTCTG", "AGGACGGATTCT", "hamming distance in very long strands"},
}

func TestHamming(t *testing.T) {
	for _, tc := range testCases {

		observed := Distance(tc.strandA, tc.strandB)

		if tc.expected != observed {
			t.Fatalf(`%s:
{%v,%v}
expected: %v
observed: %v`,
				tc.description,
				tc.strandA,
				tc.strandB,
				tc.expected,
				observed,
			)
		}
	}
}
<commit_msg>Improve test suite in go/hamming<commit_after>package hamming

import (
	"testing"
)

var testCases = []struct {
	expected         int
	strandA, strandB string
	description      string
}{
	{0, "", "", "no difference between empty strands"},
	{2, "AG", "CT", "complete hamming distance for small strands"},
	{0, "A", "A", "no difference between identical strands"},
	{1, "A", "G", "complete distance for single nucleotide strands"},
	{1, "AT", "CT", "small hamming distance"},
	{1, "GGACG", "GGTCG", "small hamming distance in longer strands"},
	{1, "AATG", "AAA", "ignores extra length on first strand when longer"},
	{2, "ATA", "AGTG", "ignores extra length on second strand when longer"},
	{4, "GATACA", "GCATAA", "large hamming distance"},
	{9, "GGACGGATTCTG", "AGGACGGATTCT", "hamming distance in very long strands"},
}

func TestHamming(t *testing.T) {
	for _, tc := range testCases {

		observed := Distance(tc.strandA, tc.strandB)

		if tc.expected != observed {
			t.Fatalf(`%s:
{%v,%v}
expected: %v
observed: %v`,
				tc.description,
				tc.strandA,
				tc.strandB,
				tc.expected,
				observed,
			)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"os"
	"sync"
	"testing"
	"time"

	vault "github.com/hashicorp/vault/api"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/auth"
)

const (
	vaultAddress = "http://127.0.0.1:8200"
	pluginName   = "pachyderm"
	vaultMaxTTL  = "768h0m0s"
)

var pachClient *client.APIClient
var getPachClientOnce sync.Once

func getPachClient(t testing.TB) *client.APIClient {
	getPachClientOnce.Do(func() {
		var err error
		if addr := os.Getenv("PACHD_PORT_650_TCP_ADDR"); addr != "" {
			pachClient, err = client.NewInCluster()
		} else {
			pachClient, err = client.NewOnUserMachine(false, "user")
		}
		if err != nil {
			t.Fatalf(err.Error())
		}
	})
	return pachClient
}

func configurePlugin(t *testing.T, v *vault.Client, ttl string) error {
	c := getPachClient(t)
	resp, err := c.Authenticate(
		context.Background(),
		&auth.AuthenticateRequest{GitHubToken: "admin"})

	if err != nil {
		return err
	}

	return configurePluginHelper(c, v, resp.PachToken, c.GetAddress(), ttl)
}

func configurePluginHelper(pachClient *client.APIClient, v *vault.Client, testPachToken string, testPachdAddress string, ttl string) error {
	vl := v.Logical()
	config := make(map[string]interface{})
	config["admin_token"] = testPachToken
	config["pachd_address"] = testPachdAddress
	if ttl != "" {
		config["ttl"] = ttl
	}
	resp, err := vl.Write(
		fmt.Sprintf("/%v/config", pluginName),
		config,
	)
	if err != nil {
		return err
	}
	if respErr, ok := resp.Data["error"]; ok {
		return fmt.Errorf("error in response: %v (%T)", respErr, respErr)
	}
	return nil
}

func TestBadConfig(t *testing.T) {
	vaultClientConfig := vault.DefaultConfig()
	vaultClientConfig.Address = vaultAddress
	v, err := vault.NewClient(vaultClientConfig)
	if err != nil {
		t.Fatalf(err.Error())
	}
	v.SetToken("root")
	c := getPachClient(t)
	resp, err := c.Authenticate(
		context.Background(),
		&auth.AuthenticateRequest{GitHubToken: "admin"})

	if err != nil {
		t.Fatalf(err.Error())
	}

	// make sure we get an error for missing admin_token
	err = configurePluginHelper(c, v, "", "", "")
	if err == nil {
		t.Fatalf("expected error: missing token in config (but got none)")
	}

	// make sure we get an error for missing pachd_address (not set in configurePluginHelper)
	err = configurePluginHelper(c, v, resp.PachToken, "", "")
	if err == nil {
		t.Fatalf("expected missing address in config to error")
	}

	// make sure that missing TTL is OK
	err = configurePluginHelper(c, v, resp.PachToken, c.GetAddress(), "")
	if err != nil {
		t.Fatalf(err.Error())
	}

	// make sure that malformed TTL is not OK
	err = configurePluginHelper(c, v, resp.PachToken, c.GetAddress(), "234....^^^")
	if err == nil {
		t.Fatalf("expected bad ttl in config to error")
	}
}

func TestMinimalConfig(t *testing.T) {
	c := getPachClient(t)
	vaultClientConfig := vault.DefaultConfig()
	vaultClientConfig.Address = vaultAddress
	v, err := vault.NewClient(vaultClientConfig)
	if err != nil {
		t.Fatalf(err.Error())
	}
	v.SetToken("root")

	// Test using just defaults
	// We'll see an error if the admin token / pachd address are not set
	err = configurePlugin(t, v, "")
	if err != nil {
		t.Fatalf(err.Error())
	}

	vl := v.Logical()
	secret, err := vl.Read(
		fmt.Sprintf("/%v/config", pluginName),
	)

	if err != nil {
		t.Fatalf(err.Error())
	}

	if secret.Data["pachd_address"] != c.GetAddress() {
		t.Fatalf("pachd_address configured incorrectly")
	}
	if secret.Data["ttl"] == "0s" {
		t.Fatalf("ttl configured incorrectly")
	}
	ttlIface, ok := secret.Data["ttl"]
	if !ok {
		t.Fatalf("ttl wasn't set in config, but it should always have a default value")
	}
	ttlStr, ok := ttlIface.(string)
	if !ok {
		t.Fatalf("ttl has the wrong type (should be string but was %T)", ttlIface)
	}
	ttl, err := time.ParseDuration(ttlStr)
	if err != nil {
		t.Fatalf("could not parse duration (%s) from config: %v", ttlStr, err)
	}
	if ttl < (30 * time.Second) {
		t.Fatalf("ttl configured incorrectly; should take default vaule but is actually (%v)", secret.Data["ttl"])
	}

}

func loginHelper(t *testing.T, ttl string) (*client.APIClient, *vault.Client, *vault.Secret) {
	vaultClientConfig := vault.DefaultConfig()
	vaultClientConfig.Address = vaultAddress
	v, err := vault.NewClient(vaultClientConfig)
	if err != nil {
		t.Fatalf(err.Error())
	}
	v.SetToken("root")

	err = configurePlugin(t, v, ttl)
	if err != nil {
		t.Fatalf(err.Error())
	}

	params := make(map[string]interface{})
	params["username"] = "bogusgithubusername"
	vl := v.Logical()
	secret, err := vl.Write(
		fmt.Sprintf("/%v/login", pluginName),
		params,
	)

	if err != nil {
		t.Fatalf(err.Error())
	}

	pachToken, ok := secret.Auth.Metadata["user_token"]
	if !ok {
		t.Fatalf("vault login response did not contain user token")
	}
	reportedPachdAddress, ok := secret.Auth.Metadata["pachd_address"]
	if !ok {
		t.Fatalf("vault login response did not contain pachd address")
	}

	c, err := client.NewFromAddress(reportedPachdAddress)
	if err != nil {
		t.Fatalf(err.Error())
	}
	c.SetAuthToken(pachToken)

	return c, v, secret
}

func TestLogin(t *testing.T) {
	// Negative control: before we have a valid pach token, we should not
	// be able to list admins
	c := getPachClient(t)
	_, err := c.AuthAPIClient.GetAdmins(context.Background(), &auth.GetAdminsRequest{})
	if err == nil {
		t.Fatalf("client could list admins before using auth token. this is likely a bug")
	}

	c, _, _ = loginHelper(t, "")

	// Now do the actual test: try and list admins w a client w a valid pach token
	_, err = c.AuthAPIClient.GetAdmins(c.Ctx(), &auth.GetAdminsRequest{})
	if err != nil {
		t.Fatalf(err.Error())
	}
}

func TestLoginExpires(t *testing.T) {
	c, _, secret := loginHelper(t, "2s")

	_, err := c.AuthAPIClient.GetAdmins(c.Ctx(), &auth.GetAdminsRequest{})
	if err != nil {
		t.Fatalf(err.Error())
	}

	time.Sleep(time.Duration(secret.Auth.LeaseDuration+1) * time.Second)
	_, err = c.AuthAPIClient.GetAdmins(c.Ctx(), &auth.GetAdminsRequest{})
	if err == nil {
		t.Fatalf("API call should fail, but token did not expire")
	}
}

func TestRenewBeforeTTLExpires(t *testing.T) {
	ttl := 10
	c, vaultClient, secret := loginHelper(t, fmt.Sprintf("%vs", ttl))

	_, err := c.AuthAPIClient.GetAdmins(c.Ctx(), &auth.GetAdminsRequest{})
	if err != nil {
		t.Fatalf(err.Error())
	}

	renewer, err := vaultClient.NewRenewer(&vault.RenewerInput{
		Secret:    secret,
		Increment: ttl,
	})
	if err != nil {
		t.Fatalf(err.Error())
	}

	time.Sleep(time.Duration(ttl/2) * time.Second)
	go renewer.Renew()
	defer renewer.Stop()

	select {
	case err := <-renewer.DoneCh():
		if err != nil {
			t.Fatalf(err.Error())
		}
	case <-renewer.RenewCh():
	}
	time.Sleep(time.Duration(ttl/2+1) * time.Second)

	_, err = c.AuthAPIClient.GetAdmins(c.Ctx(), &auth.GetAdminsRequest{})
	if err != nil {
		t.Fatalf(err.Error())
	}
}

func TestRenewAfterTTLExpires(t *testing.T) {
	ttl := 2
	c, vaultClient, secret := loginHelper(t, fmt.Sprintf("%vs", ttl))

	_, err := c.AuthAPIClient.GetAdmins(c.Ctx(), &auth.GetAdminsRequest{})
	if err != nil {
		t.Fatalf(err.Error())
	}

	renewer, err := vaultClient.NewRenewer(&vault.RenewerInput{
		Secret:    secret,
		Increment: ttl,
	})
	if err != nil {
		t.Fatalf(err.Error())
	}

	time.Sleep(time.Duration(ttl+1) * time.Second)
	go renewer.Renew()
	defer renewer.Stop()

	select {
	case err := <-renewer.DoneCh():
		if err == nil {
			t.Fatalf("Expected an error renewing but got none\n")
		}
	case <-renewer.RenewCh():
		t.Fatal("Expected failed renewal, but got successful renewal\n")
	}

	_, err = c.AuthAPIClient.GetAdmins(c.Ctx(), &auth.GetAdminsRequest{})
	if err == nil {
		t.Fatalf("Expected error using pach token after expiry, but got no error\n")
	}
}

func TestRevoke(t *testing.T) {
	ttl := 2
	c, v, secret := loginHelper(t, fmt.Sprintf("%vs", ttl))

	_, err := c.AuthAPIClient.GetAdmins(c.Ctx(), &auth.GetAdminsRequest{})
	if err != nil {
		t.Fatalf(err.Error())
	}

	params := make(map[string]interface{})
	params["user_token"] = secret.Auth.Metadata["user_token"]
	vl := v.Logical()
	_, err = vl.Write(
		fmt.Sprintf("/%v/revoke", pluginName),
		params,
	)

	if err != nil {
		t.Fatalf(err.Error())
	}

	_, err = c.AuthAPIClient.GetAdmins(c.Ctx(), &auth.GetAdminsRequest{})
	if err == nil {
		t.Fatalf("expected error with revoked pach token, got none\n")
	}
}
<commit_msg>Remove unused constant from vault_test.go<commit_after>package main

import (
	"context"
	"fmt"
	"os"
	"sync"
	"testing"
	"time"

	vault "github.com/hashicorp/vault/api"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/auth"
)

const (
	vaultAddress = "http://127.0.0.1:8200"
	pluginName   = "pachyderm"
)

var pachClient *client.APIClient
var getPachClientOnce sync.Once

func getPachClient(t testing.TB) *client.APIClient {
	getPachClientOnce.Do(func() {
		var err error
		if addr := os.Getenv("PACHD_PORT_650_TCP_ADDR"); addr != "" {
			pachClient, err = client.NewInCluster()
		} else {
			pachClient, err = client.NewOnUserMachine(false, "user")
		}
		if err != nil {
			t.Fatalf(err.Error())
		}
	})
	return pachClient
}

func configurePlugin(t *testing.T, v *vault.Client, ttl string) error {
	c := getPachClient(t)
	resp, err := c.Authenticate(
		context.Background(),
		&auth.AuthenticateRequest{GitHubToken: "admin"})

	if err != nil {
		return err
	}

	return configurePluginHelper(c, v, resp.PachToken, c.GetAddress(), ttl)
}

func configurePluginHelper(pachClient *client.APIClient, v *vault.Client, testPachToken string, testPachdAddress string, ttl string) error {
	vl := v.Logical()
	config := make(map[string]interface{})
	config["admin_token"] = testPachToken
	config["pachd_address"] = testPachdAddress
	if ttl != "" {
		config["ttl"] = ttl
	}
	resp, err := vl.Write(
		fmt.Sprintf("/%v/config", pluginName),
		config,
	)
	if err != nil {
		return err
	}
	if respErr, ok := resp.Data["error"]; ok {
		return fmt.Errorf("error in response: %v (%T)", respErr, respErr)
	}
	return nil
}

func TestBadConfig(t *testing.T) {
	vaultClientConfig := vault.DefaultConfig()
	vaultClientConfig.Address = vaultAddress
	v, err := vault.NewClient(vaultClientConfig)
	if err != nil {
		t.Fatalf(err.Error())
	}
	v.SetToken("root")
	c := getPachClient(t)
	resp, err := c.Authenticate(
		context.Background(),
		&auth.AuthenticateRequest{GitHubToken: "admin"})

	if err != nil {
		t.Fatalf(err.Error())
	}

	// make sure we get an error for missing admin_token
	err = configurePluginHelper(c, v, "", "", "")
	if err == nil {
		t.Fatalf("expected error: missing token in config (but got none)")
	}

	// make sure we get an error for missing pachd_address (not set in configurePluginHelper)
	err = configurePluginHelper(c, v, resp.PachToken, "", "")
	if err == nil {
		t.Fatalf("expected missing address in config to error")
	}

	// make sure that missing TTL is OK
	err = configurePluginHelper(c, v, resp.PachToken, c.GetAddress(), "")
	if err != nil {
		t.Fatalf(err.Error())
	}

	// make sure that malformed TTL is not OK
	err = configurePluginHelper(c, v, resp.PachToken, c.GetAddress(), "234....^^^")
	if err == nil {
		t.Fatalf("expected bad ttl in config to error")
	}
}

func TestMinimalConfig(t *testing.T) {
	c := getPachClient(t)
	vaultClientConfig := vault.DefaultConfig()
	vaultClientConfig.Address = vaultAddress
	v, err := vault.NewClient(vaultClientConfig)
	if err != nil {
		t.Fatalf(err.Error())
	}
	v.SetToken("root")

	// Test using just defaults
	// We'll see an error if the admin token / pachd address are not set
	err = configurePlugin(t, v, "")
	if err != nil {
		t.Fatalf(err.Error())
	}

	vl := v.Logical()
	secret, err := vl.Read(
		fmt.Sprintf("/%v/config", pluginName),
	)

	if err != nil {
		t.Fatalf(err.Error())
	}

	if secret.Data["pachd_address"] != c.GetAddress() {
		t.Fatalf("pachd_address configured incorrectly")
	}
	if secret.Data["ttl"] == "0s" {
		t.Fatalf("ttl configured incorrectly")
	}
	ttlIface, ok := secret.Data["ttl"]
	if !ok {
		t.Fatalf("ttl wasn't set in config, but it should always have a default value")
	}
	ttlStr, ok := ttlIface.(string)
	if !ok {
		t.Fatalf("ttl has the wrong type (should be string but was %T)", ttlIface)
	}
	ttl, err := time.ParseDuration(ttlStr)
	if err != nil {
		t.Fatalf("could not parse duration (%s) from config: %v", ttlStr, err)
	}
	if ttl < (30 * time.Second) {
		t.Fatalf("ttl configured incorrectly; should take default vaule but is actually (%v)", secret.Data["ttl"])
	}

}

func loginHelper(t *testing.T, ttl string) (*client.APIClient, *vault.Client, *vault.Secret) {
	vaultClientConfig := vault.DefaultConfig()
	vaultClientConfig.Address = vaultAddress
	v, err := vault.NewClient(vaultClientConfig)
	if err != nil {
		t.Fatalf(err.Error())
	}
	v.SetToken("root")

	err = configurePlugin(t, v, ttl)
	if err != nil {
		t.Fatalf(err.Error())
	}

	params := make(map[string]interface{})
	params["username"] = "bogusgithubusername"
	vl := v.Logical()
	secret, err := vl.Write(
		fmt.Sprintf("/%v/login", pluginName),
		params,
	)

	if err != nil {
		t.Fatalf(err.Error())
	}

	pachToken, ok := secret.Auth.Metadata["user_token"]
	if !ok {
		t.Fatalf("vault login response did not contain user token")
	}
	reportedPachdAddress, ok := secret.Auth.Metadata["pachd_address"]
	if !ok {
		t.Fatalf("vault login response did not contain pachd address")
	}

	c, err := client.NewFromAddress(reportedPachdAddress)
	if err != nil {
		t.Fatalf(err.Error())
	}
	c.SetAuthToken(pachToken)

	return c, v, secret
}

func TestLogin(t *testing.T) {
	// Negative control: before we have a valid pach token, we should not
	// be able to list admins
	c := getPachClient(t)
	_, err := c.AuthAPIClient.GetAdmins(context.Background(), &auth.GetAdminsRequest{})
	if err == nil {
		t.Fatalf("client could list admins before using auth token. this is likely a bug")
	}

	c, _, _ = loginHelper(t, "")

	// Now do the actual test: try and list admins w a client w a valid pach token
	_, err = c.AuthAPIClient.GetAdmins(c.Ctx(), &auth.GetAdminsRequest{})
	if err != nil {
		t.Fatalf(err.Error())
	}
}

func TestLoginExpires(t *testing.T) {
	c, _, secret := loginHelper(t, "2s")

	_, err := c.AuthAPIClient.GetAdmins(c.Ctx(), &auth.GetAdminsRequest{})
	if err != nil {
		t.Fatalf(err.Error())
	}

	time.Sleep(time.Duration(secret.Auth.LeaseDuration+1) * time.Second)
	_, err = c.AuthAPIClient.GetAdmins(c.Ctx(), &auth.GetAdminsRequest{})
	if err == nil {
		t.Fatalf("API call should fail, but token did not expire")
	}
}

func TestRenewBeforeTTLExpires(t *testing.T) {
	ttl := 10
	c, vaultClient, secret := loginHelper(t, fmt.Sprintf("%vs", ttl))

	_, err := c.AuthAPIClient.GetAdmins(c.Ctx(), &auth.GetAdminsRequest{})
	if err != nil {
		t.Fatalf(err.Error())
	}

	renewer, err := vaultClient.NewRenewer(&vault.RenewerInput{
		Secret:    secret,
		Increment: ttl,
	})
	if err != nil {
		t.Fatalf(err.Error())
	}

	time.Sleep(time.Duration(ttl/2) * time.Second)
	go renewer.Renew()
	defer renewer.Stop()

	select {
	case err := <-renewer.DoneCh():
		if err != nil {
			t.Fatalf(err.Error())
		}
	case <-renewer.RenewCh():
	}
	time.Sleep(time.Duration(ttl/2+1) * time.Second)

	_, err = c.AuthAPIClient.GetAdmins(c.Ctx(), &auth.GetAdminsRequest{})
	if err != nil {
		t.Fatalf(err.Error())
	}
}

func TestRenewAfterTTLExpires(t *testing.T) {
	ttl := 2
	c, vaultClient, secret := loginHelper(t, fmt.Sprintf("%vs", ttl))

	_, err := c.AuthAPIClient.GetAdmins(c.Ctx(), &auth.GetAdminsRequest{})
	if err != nil {
		t.Fatalf(err.Error())
	}

	renewer, err := vaultClient.NewRenewer(&vault.RenewerInput{
		Secret:    secret,
		Increment: ttl,
	})
	if err != nil {
		t.Fatalf(err.Error())
	}

	time.Sleep(time.Duration(ttl+1) * time.Second)
	go renewer.Renew()
	defer renewer.Stop()

	select {
	case err := <-renewer.DoneCh():
		if err == nil {
			t.Fatalf("Expected an error renewing but got none\n")
		}
	case <-renewer.RenewCh():
		t.Fatal("Expected failed renewal, but got successful renewal\n")
	}

	_, err = c.AuthAPIClient.GetAdmins(c.Ctx(), &auth.GetAdminsRequest{})
	if err == nil {
		t.Fatalf("Expected error using pach token after expiry, but got no error\n")
	}
}

func TestRevoke(t *testing.T) {
	ttl := 2
	c, v, secret := loginHelper(t, fmt.Sprintf("%vs", ttl))

	_, err := c.AuthAPIClient.GetAdmins(c.Ctx(), &auth.GetAdminsRequest{})
	if err != nil {
		t.Fatalf(err.Error())
	}

	params := make(map[string]interface{})
	params["user_token"] = secret.Auth.Metadata["user_token"]
	vl := v.Logical()
	_, err = vl.Write(
		fmt.Sprintf("/%v/revoke", pluginName),
		params,
	)

	if err != nil {
		t.Fatalf(err.Error())
	}

	_, err = c.AuthAPIClient.GetAdmins(c.Ctx(), &auth.GetAdminsRequest{})
	if err == nil {
		t.Fatalf("expected error with revoked pach token, got none\n")
	}
}
<|endoftext|>


<commit_before>package clickhouse

type PingErrorEvent func(*Conn)

type Cluster struct {
	conn   []*Conn
	active []*Conn
	fail   PingErrorEvent
}

func NewCluster(conn ...*Conn) *Cluster {
	return &Cluster{
		conn: conn,
	}
}

func (c *Cluster) OnFail(f PingErrorEvent) {
	c.fail = f
}

func (c *Cluster) Active() *Conn {
	return c.active[0]
}

func (c *Cluster) CheckConnections() {
	var (
		err error
		res []*Conn
	)

	for _, conn := range c.conn {
		err = conn.Ping()
		if err == nil {
			res = append(res, conn)
		} else {
			c.fail(conn)
		}
	}

	c.active = res
}
<commit_msg>Cluster mutex<commit_after>package clickhouse

import (
	"math/rand"
	"sync"
)

type PingErrorEvent func(*Conn)

type Cluster struct {
	conn   []*Conn
	active []*Conn
	fail   PingErrorEvent
	mx     sync.Mutex
}

func NewCluster(conn ...*Conn) *Cluster {
	return &Cluster{
		conn: conn,
	}
}

func (c *Cluster) OnFail(f PingErrorEvent) {
	c.fail = f
}

func (c *Cluster) Active() *Conn {
	c.mx.Lock()
	defer c.mx.Unlock()
	l := len(c.active)
	if l < 1 {
		return nil
	}
	return c.active[rand.Intn(l)]
}

func (c *Cluster) CheckConnections() {
	var (
		err error
		res []*Conn
	)

	for _, conn := range c.conn {
		err = conn.Ping()
		if err == nil {
			res = append(res, conn)
		} else {
			c.fail(conn)
		}
	}

	c.mx.Lock()
	c.active = res
	c.mx.Unlock()
}
<|endoftext|>


<commit_before>// +build fetcherExec

package main

import (
	"encoding/base64"
	"encoding/json"
	"flag"
	"github.com/fmpwizard/owlcrawler/couchdb"
	log "github.com/golang/glog"
	"github.com/nats-io/nats"
	"io/ioutil"
	"net/http"
	"os/user"
	"path/filepath"
	"time"
)

const fetchQueue = "fetch_url"
const extractQueue = "extract_url"

type dataStore struct {
	ID        string    `json:"_id"`
	URL       string    `json:"url"`
	HTML      string    `json:"html"`
	FetchedOn time.Time `json:"fetched_on"`
}

var gnatsdCredentials gnatsdCred

type gnatsdCred struct {
	URL string
}

func fetchHTML(url string) {
	log.V(2).Infoln("Total tasks launched")

	nc, err := nats.Connect(gnatsdCredentials.URL)
	if err != nil {
		log.Fatalf("Could not connect to gnatsd, got: %s\n", err)
	}

	//Fetch url
	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		log.Errorf("Error parsing url: %s, got: %v\n", url, err)
	}
	req.Header.Set("User-Agent", "OwlCrawler - https://github.com/fmpwizard/owlcrawler")
	resp, err := client.Do(req)
	if err != nil {
		log.Errorf("Error while fetching url: %s, got error: %v\n", url, err)
		return
	}

	defer resp.Body.Close()
	htmlData, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Errorf("Error while reading html for url: %s, got error: %v\n", url, err)
		return
	}

	data := &dataStore{
		ID:        base64.URLEncoding.EncodeToString([]byte(url)),
		URL:       url,
		HTML:      string(htmlData[:]),
		FetchedOn: time.Now().UTC(),
	}

	pageData, err := json.Marshal(data)
	if err != nil {
		log.Errorf("Error generating json to save in database, got: %v\n", err)
	}

	ret, err := couchdb.AddURLData(url, pageData)
	if err == nil {
		//Send fethed url to parse queue
		err := nc.Publish(extractQueue, []byte(ret.ID))
		if err != nil {
			log.Errorf("Failed to push %s to extract queue\n", url)
		}
	}

	log.V(2).Infof("Finished getting %s", url)
}

func main() {
	flag.Parse()
	log.V(2).Infof("Starting Fetcher.")
	nc, _ := nats.Connect(gnatsdCredentials.URL)
	sub, err := nc.SubscribeSync(fetchQueue)
	if err != nil {
		log.Fatalf("Error while subscribing to fetch_url, got %s\n", err)
	}
	for {
		if payload, err := sub.NextMsg(30 * time.Second); err == nil {
			if couchdb.ShouldURLBeFetched(string(payload.Data[:])) {
				fetchHTML(string(payload.Data[:]))
			}
		}
	}
}

func init() {
	if u, err := user.Current(); err == nil {
		path := filepath.Join(u.HomeDir, ".gnatsd.json")
		content, err := ioutil.ReadFile(path)
		if err != nil {
			log.Fatalf("Error reading gnatds user file, got: %v\n", err)
		}

		err = json.Unmarshal(content, &gnatsdCredentials)
		if err != nil {
			log.Fatalf("Invalid gnatsd credentials file, got: %v\n", err)
		}
	}
}
<commit_msg>small log entry fix<commit_after>// +build fetcherExec

package main

import (
	"encoding/base64"
	"encoding/json"
	"flag"
	"github.com/fmpwizard/owlcrawler/couchdb"
	log "github.com/golang/glog"
	"github.com/nats-io/nats"
	"io/ioutil"
	"net/http"
	"os/user"
	"path/filepath"
	"time"
)

const fetchQueue = "fetch_url"
const extractQueue = "extract_url"

type dataStore struct {
	ID        string    `json:"_id"`
	URL       string    `json:"url"`
	HTML      string    `json:"html"`
	FetchedOn time.Time `json:"fetched_on"`
}

var gnatsdCredentials gnatsdCred

type gnatsdCred struct {
	URL string
}

func fetchHTML(url string) {
	log.V(2).Infof("Fetching %s\n", url)

	nc, err := nats.Connect(gnatsdCredentials.URL)
	if err != nil {
		log.Fatalf("Could not connect to gnatsd, got: %s\n", err)
	}

	//Fetch url
	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		log.Errorf("Error parsing url: %s, got: %v\n", url, err)
	}
	req.Header.Set("User-Agent", "OwlCrawler - https://github.com/fmpwizard/owlcrawler")
	resp, err := client.Do(req)
	if err != nil {
		log.Errorf("Error while fetching url: %s, got error: %v\n", url, err)
		return
	}

	defer resp.Body.Close()
	htmlData, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Errorf("Error while reading html for url: %s, got error: %v\n", url, err)
		return
	}

	data := &dataStore{
		ID:        base64.URLEncoding.EncodeToString([]byte(url)),
		URL:       url,
		HTML:      string(htmlData[:]),
		FetchedOn: time.Now().UTC(),
	}

	pageData, err := json.Marshal(data)
	if err != nil {
		log.Errorf("Error generating json to save in database, got: %v\n", err)
	}

	ret, err := couchdb.AddURLData(url, pageData)
	if err == nil {
		//Send fethed url to parse queue
		err := nc.Publish(extractQueue, []byte(ret.ID))
		if err != nil {
			log.Errorf("Failed to push %s to extract queue\n", url)
		}
	}
	log.V(2).Infof("Finished getting %s", url)
}

func main() {
	flag.Parse()
	log.V(2).Infof("Starting Fetcher.")
	nc, _ := nats.Connect(gnatsdCredentials.URL)
	sub, err := nc.SubscribeSync(fetchQueue)
	if err != nil {
		log.Fatalf("Error while subscribing to fetch_url, got %s\n", err)
	}
	for {
		if payload, err := sub.NextMsg(30 * time.Second); err == nil {
			if couchdb.ShouldURLBeFetched(string(payload.Data[:])) {
				fetchHTML(string(payload.Data[:]))
			}
		}
	}
}

func init() {
	if u, err := user.Current(); err == nil {
		path := filepath.Join(u.HomeDir, ".gnatsd.json")
		content, err := ioutil.ReadFile(path)
		if err != nil {
			log.Fatalf("Error reading gnatds user file, got: %v\n", err)
		}

		err = json.Unmarshal(content, &gnatsdCredentials)
		if err != nil {
			log.Fatalf("Invalid gnatsd credentials file, got: %v\n", err)
		}
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"log"
	"os"

	"github.com/AlecAivazis/survey"
	"github.com/spf13/cobra"
	"github.com/tunedmystic/konnect/engine"
)

// AppVersion info.
var AppVersion string

// AppBuild info.
var AppBuild string

// AppDate info.
var AppDate string

var config string
var interactive bool
var version bool

// RootCmd - Entry point to the application.
var RootCmd = &cobra.Command{
	Use:   "konnect",
	Short: "Connect to SSH hosts.",
	Long:  "Define and connect to SSH hosts.",
	Run: func(cmd *cobra.Command, args []string) {
		cmd.Help()
	},
	PreRun: func(cmd *cobra.Command, args []string) {
		if version {
			fmt.Println(AppVersion)
			os.Exit(0)
		}

		if interactive {
			InteractivePrompt(cmd)
			os.Exit(0)
		}
	},
}

func init() {
	// Config filename.
	RootCmd.PersistentFlags().StringVarP(&config, "filename", "f", "", "Specify config file")
	// Show an iteractive prompt to connect to hosts.
	RootCmd.Flags().BoolVarP(&interactive, "interactive", "i", false, "Connect to a host interactively")
	// Show version information.
	RootCmd.Flags().BoolVarP(&version, "version", "v", false, "View version information")
}

// InteractivePrompt to connect to hosts.
func InteractivePrompt(cmd *cobra.Command) {
	fmt.Println("Starting interactive prompt...")
	// Resolve filename from flags.
	filename := resolveFilename(cmd)

	// Init Konnect engine and get host names.
	engine := engine.Init(filename)
	hosts := engine.GetHosts()

	// Create survey.
	prompt := []*survey.Question{
		{
			Name:     "Hostname",
			Validate: survey.Required,
			Prompt: &survey.Select{
				Message: "Connect to host:",
				Options: hosts,
			},
		},
	}

	// Create answer.
	answer := struct {
		Hostname string
	}{}

	// Show prompt.
	if err := survey.Ask(prompt, &answer); err != nil {
		log.Fatal("No host was selected")
	}

	// Connect to host.
	engine.Connect(answer.Hostname)
}

// AddCommands - Connects subcommands to the RootCmd.
func AddCommands() {
	RootCmd.AddCommand(ArgsCmd)
	RootCmd.AddCommand(ConnectCmd)
	RootCmd.AddCommand(InitCmd)
	RootCmd.AddCommand(ListCmd)
	RootCmd.AddCommand(StatusCmd)
	RootCmd.AddCommand(VersionCmd)
}

// Execute - runs the RootCmd.
func Execute() error {
	AddCommands()
	return RootCmd.Execute()
}
<commit_msg>Better descriptions for subcommands.<commit_after>package cmd

import (
	"fmt"
	"log"
	"os"

	"github.com/AlecAivazis/survey"
	"github.com/spf13/cobra"
	"github.com/tunedmystic/konnect/engine"
)

// AppVersion info.
var AppVersion string

// AppBuild info.
var AppBuild string

// AppDate info.
var AppDate string

var config string
var interactive bool
var version bool

// RootCmd - Entry point to the application.
var RootCmd = &cobra.Command{
	Use:   "konnect",
	Short: "Connect to SSH hosts.",
	Long:  "Konnect is a tool to define and connect to SSH hosts.",
	Run: func(cmd *cobra.Command, args []string) {
		cmd.Help()
	},
	PreRun: func(cmd *cobra.Command, args []string) {
		if version {
			fmt.Println(AppVersion)
			os.Exit(0)
		}

		if interactive {
			InteractivePrompt(cmd)
			os.Exit(0)
		}
	},
}

func init() {
	// Config filename.
	RootCmd.PersistentFlags().StringVarP(&config, "filename", "f", "", "Specify config file")
	// Show an iteractive prompt to connect to hosts.
	RootCmd.Flags().BoolVarP(&interactive, "interactive", "i", false, "Connect to a host interactively")
	// Show version information.
	RootCmd.Flags().BoolVarP(&version, "version", "v", false, "View version information")
}

// InteractivePrompt to connect to hosts.
func InteractivePrompt(cmd *cobra.Command) {
	fmt.Println("Starting interactive prompt...")
	// Resolve filename from flags.
	filename := resolveFilename(cmd)

	// Init Konnect engine and get host names.
	engine := engine.Init(filename)
	hosts := engine.GetHosts()

	// Create survey.
	prompt := []*survey.Question{
		{
			Name:     "Hostname",
			Validate: survey.Required,
			Prompt: &survey.Select{
				Message: "Connect to host:",
				Options: hosts,
			},
		},
	}

	// Create answer.
	answer := struct {
		Hostname string
	}{}

	// Show prompt.
	if err := survey.Ask(prompt, &answer); err != nil {
		log.Fatal("No host was selected")
	}

	// Connect to host.
	engine.Connect(answer.Hostname)
}

// AddCommands - Connects subcommands to the RootCmd.
func AddCommands() {
	RootCmd.AddCommand(ArgsCmd)
	RootCmd.AddCommand(ConnectCmd)
	RootCmd.AddCommand(InitCmd)
	RootCmd.AddCommand(ListCmd)
	RootCmd.AddCommand(StatusCmd)
	RootCmd.AddCommand(VersionCmd)
}

// Execute - runs the RootCmd.
func Execute() error {
	AddCommands()
	return RootCmd.Execute()
}
<|endoftext|>


<commit_before>package cmd

import (
	"os"

	flag "github.com/spf13/pflag"
)

// inspired by https://github.com/constabulary/gb/blob/master/cmd/cmd.go

// Command represents a subcommand, or plugin that is executed
type Command struct {
	// Name of the command
	name string

	// UsageLine demonstrates how to use this command
	usageLine string

	// Single line description of the purpose of the command
	short string

	// Description of this command
	long string

	// Run is invoked with arguments left over after flag parsing.
	run func(args []string) error

	// FlagSet for adding flags
	fs *flag.FlagSet
}

func NewCommand(name, usageLine, short, long string, run func([]string) error) *Command {
	cmd := &Command{
		name:      name,
		usageLine: usageLine,
		short:     short,
		long:      long,
		run:       run,
		fs:        flag.NewFlagSet(name, flag.ExitOnError),
	}
	return cmd
}

// FlagSet for adding flags
func (cmd *Command) FS() *flag.FlagSet {
	return cmd.fs
}

// Runnable indicates this is a command that can be involved.
// Non runnable commands are only informational.
func (c *Command) Runnable() bool { return c.run != nil }

// RunCommand parses flags and runs the Command.
func RunCommand(cmd *Command, args []string) error {
	fs := cmd.FS()
	cmdflags, args := splitargs(args)
	if err := fs.Parse(cmdflags); err != nil {
		fs.Usage()
		os.Exit(1)
	}
	// TODO check subcommand
	return cmd.run(args)
}

func splitargs(args []string) ([]string, []string) {
	return []string{}, []string{}
}
<commit_msg>cmd/cmd.go: Command framework, with global flag set<commit_after>package cmd

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	flag "github.com/spf13/pflag"
)

var commands map[string]*Command = make(map[string]*Command)

// inspired by https://github.com/constabulary/gb/blob/master/cmd/cmd.go

// Command represents a subcommand, or plugin that is executed
type Command struct {
	// Name of the command
	name string

	// UsageLine demonstrates how to use this command
	usageLine string

	// Single line description of the purpose of the command
	short string

	// Description of this command
	long string

	// Run is invoked with arguments left over after flag parsing.
	run func(args []string) error

	// FlagSet for adding flags for that command
	fs *flag.FlagSet

	// FlagSet for adding flags for that command and any sub-Command
	gfs *flag.FlagSet

	// Parent Command
	parent parent

	// Subcommands
	subcmds map[string]*Command
}

type parent interface {
	fullCommand() string
	checkGlobalFlags(args []string) []string
}

func NewCommand(name, usageLine, short, long string, run func([]string) error, parent *Command) *Command {
	cmd := &Command{
		name:      name,
		usageLine: usageLine,
		short:     short,
		long:      long,
		run:       run,
		fs:        flag.NewFlagSet(name, flag.ExitOnError),
		gfs:       flag.NewFlagSet(name, flag.ContinueOnError),
		subcmds:   make(map[string]*Command),
	}
	if parent == nil {
		// This is a root command
		commands[name] = cmd
	} else {
		cmd.parent = *parent
	}
	return cmd
}

// FlagSet for adding flags for that command
func (cmd *Command) FS() *flag.FlagSet {
	return cmd.fs
}

// FlagSet for adding flags for that command and any sub-Command
func (cmd *Command) GFS() *flag.FlagSet {
	return cmd.gfs
}

// Runnable indicates this is a command that can be involved.
// Non runnable commands are only informational.
func (c *Command) Runnable() bool { return c.run != nil }

// RunCommand parses flags and runs the Command.
func RunCommand(args []string) error {
	cmd, err := commandFromArgs(os.Args)
	if err != nil {
		return err
	}
	args = cmd.checkGlobalFlags(args)
	fs := cmd.fs
	if err := fs.Parse(args); err != nil {
		fmt.Printf("Incorrect usage of %s:", cmd.fullCommand())
		fs.Usage()
		os.Exit(1)
	}
	if cmd.run == nil {
		return nil
	}
	return cmd.run(cmd.fs.Args()[1:])
}

func (cmd Command) fullCommand() string {
	res := cmd.name
	if cmd.parent != nil {
		res = cmd.parent.fullCommand() + " " + res
	}
	return res
}

func commandFromArgs(args []string) (*Command, error) {
	var cmd *Command
	for i, arg := range args {
		if i == 0 {
			arg = filepath.Base(arg)
			ext := filepath.Ext(arg)
			if ext != "" {
				arg = arg[:len(arg)-len(ext)]
			}
		}
		if arg == "--" {
			return cmd, nil
		}
		if strings.HasPrefix(arg, "-") {
			continue
		}
		var subcmd *Command
		if cmd == nil {
			cmd = commands[arg]
		} else {
			subcmd = cmd.subcmds[arg]
		}
		if cmd == nil {
			return nil, fmt.Errorf("Unknown command '%s'", arg)
		}
		if subcmd == nil {
			// let's consider all the rest as parameters of the current command
			return cmd, nil
		}
		cmd = subcmd
	}
	if cmd == nil {
		return nil, fmt.Errorf("Unknown command from args '%v'", args)
	}
	return cmd, nil
}

func (cmd Command) checkGlobalFlags(args []string) []string {
	if cmd.parent != nil {
		args = cmd.parent.checkGlobalFlags(args)
	}
	cmd.gfs.Parse(args)
	args = cmd.gfs.Args()
	return args
}
<|endoftext|>