<commit_before>package stripe

import (
	"encoding/json"
)

// PaymentIntentCancellationReason is the list of allowed values for the cancelation reason.
type PaymentIntentCancellationReason string

// List of values that PaymentIntentCaptureMethod can take.
const (
	PaymentIntentCancellationReasonDuplicate           PaymentIntentCancellationReason = "duplicate"
	PaymentIntentCancellationReasonFraudulent          PaymentIntentCancellationReason = "fraudulent"
	PaymentIntentCancellationReasonRequestedByCustomer PaymentIntentCancellationReason = "requested_by_customer"
)

// PaymentIntentCaptureMethod is the list of allowed values for the capture method.
type PaymentIntentCaptureMethod string

// List of values that PaymentIntentCaptureMethod can take.
const (
	PaymentIntentCaptureMethodAutomatic PaymentIntentCaptureMethod = "automatic"
	PaymentIntentCaptureMethodManual    PaymentIntentCaptureMethod = "manual"
)

// PaymentIntentConfirmationMethod is the list of allowed values for the confirmation method.
type PaymentIntentConfirmationMethod string

// List of values that PaymentIntentConfirmationMethod can take.
const (
	PaymentIntentConfirmationMethodAutomatic PaymentIntentConfirmationMethod = "automatic"
	PaymentIntentConfirmationMethodManual    PaymentIntentConfirmationMethod = "manual"
	// The following constants are considered deprecated
	PaymentIntentConfirmationMethodPublishable PaymentIntentConfirmationMethod = "publishable"
	PaymentIntentConfirmationMethodSecret      PaymentIntentConfirmationMethod = "secret"
)

// PaymentIntentNextActionType is the list of allowed values for the next action's type.
type PaymentIntentNextActionType string

// List of values that PaymentIntentNextActionType can take.
const (
	PaymentIntentNextActionTypeRedirectToURL PaymentIntentNextActionType = "redirect_to_url"
)

// PaymentIntentStatus is the list of allowed values for the payment intent's status.
type PaymentIntentStatus string

// List of values that PaymentIntentStatus can take.
const (
	PaymentIntentStatusCanceled              PaymentIntentStatus = "canceled"
	PaymentIntentStatusProcessing            PaymentIntentStatus = "processing"
	PaymentIntentStatusRequiresAction        PaymentIntentStatus = "requires_action"
	PaymentIntentStatusRequiresCapture       PaymentIntentStatus = "requires_capture"
	PaymentIntentStatusRequiresConfirmation  PaymentIntentStatus = "requires_confirmation"
	PaymentIntentStatusRequiresPaymentMethod PaymentIntentStatus = "requires_payment_method"
	PaymentIntentStatusSucceeded             PaymentIntentStatus = "succeeded"
)

// PaymentIntentCancelParams is the set of parameters that can be used when canceling a payment intent.
type PaymentIntentCancelParams struct {
	Params             `form:"*"`
	CancellationReason *string `form:"cancellation_reason"`
}

// PaymentIntentCaptureParams is the set of parameters that can be used when capturing a payment intent.
type PaymentIntentCaptureParams struct {
	Params               `form:"*"`
	AmountToCapture      *int64                           `form:"amount_to_capture"`
	ApplicationFeeAmount *int64                           `form:"application_fee_amount"`
	TransferData         *PaymentIntentTransferDataParams `form:"transfer_data"`
}

// PaymentIntentConfirmParams is the set of parameters that can be used when confirming a payment intent.
type PaymentIntentConfirmParams struct {
	Params            `form:"*"`
	PaymentMethod     *string                `form:"payment_method"`
	ReceiptEmail      *string                `form:"receipt_email"`
	ReturnURL         *string                `form:"return_url"`
	SavePaymentMethod *bool                  `form:"save_payment_method"`
	Shipping          *ShippingDetailsParams `form:"shipping"`
	Source            *string                `form:"source"`
}

// PaymentIntentTransferDataParams is the set of parameters allowed for the transfer hash.
type PaymentIntentTransferDataParams struct {
	Amount      *int64  `form:"amount"`
	Destination *string `form:"destination"`
}

// PaymentIntentParams is the set of parameters that can be used when handling a payment intent.
type PaymentIntentParams struct {
	Params               `form:"*"`
	Amount               *int64                           `form:"amount"`
	ApplicationFeeAmount *int64                           `form:"application_fee_amount"`
	CaptureMethod        *string                          `form:"capture_method"`
	Confirm              *bool                            `form:"confirm"`
	ConfirmationMethod   *string                          `form:"confirmation_method"`
	Currency             *string                          `form:"currency"`
	Customer             *string                          `form:"customer"`
	Description          *string                          `form:"description"`
	OnBehalfOf           *string                          `form:"on_behalf_of"`
	PaymentMethod        *string                          `form:"payment_method"`
	PaymentMethodTypes   []*string                        `form:"payment_method_types"`
	ReceiptEmail         *string                          `form:"receipt_email"`
	ReturnURL            *string                          `form:"return_url"`
	SavePaymentMethod    *bool                            `form:"save_payment_method"`
	Shipping             *ShippingDetailsParams           `form:"shipping"`
	Source               *string                          `form:"source"`
	StatementDescriptor  *string                          `form:"statement_descriptor"`
	TransferData         *PaymentIntentTransferDataParams `form:"transfer_data"`
	TransferGroup        *string                          `form:"transfer_group"`
}

// PaymentIntentListParams is the set of parameters that can be used when listing payment intents.
// For more details see https://stripe.com/docs/api#list_payouts.
type PaymentIntentListParams struct {
	ListParams   `form:"*"`
	Created      *int64            `form:"created"`
	CreatedRange *RangeQueryParams `form:"created"`
}

// PaymentIntentLastPaymentError represents the last error happening on a payment intent.
type PaymentIntentLastPaymentError struct {
	Charge        string         `json:"charge"`
	Code          string         `json:"code"`
	DeclineCode   string         `json:"decline_code"`
	DocURL        string         `json:"doc_url"`
	Message       string         `json:"message"`
	Param         string         `json:"param"`
	PaymentIntent *PaymentIntent `json:"payment_intent"`
	PaymentMethod *PaymentMethod `json:"payment_method"`
	Source        *PaymentSource `json:"source"`
	Type          ErrorType      `json:"type"`
}

// PaymentIntentNextActionRedirectToURL represents the resource for the next action of type
// "redirect_to_url".
type PaymentIntentNextActionRedirectToURL struct {
	ReturnURL string `json:"return_url"`
	URL       string `json:"url"`
}

// PaymentIntentNextAction represents the type of action to take on a payment intent.
type PaymentIntentNextAction struct {
	RedirectToURL *PaymentIntentNextActionRedirectToURL `json:"redirect_to_url"`
	Type          PaymentIntentNextActionType           `json:"type"`
}

// PaymentIntentTransferData represents the information for the transfer associated with a payment intent.
type PaymentIntentTransferData struct {
	Amount      int64    `json:"amount"`
	Destination *Account `json:"destination"`
}

// PaymentIntent is the resource representing a Stripe payout.
// For more details see https://stripe.com/docs/api#payment_intents.
type PaymentIntent struct {
	Amount              int64                           `json:"amount"`
	AmountCapturable    int64                           `json:"amount_capturable"`
	AmountReceived      int64                           `json:"amount_received"`
	Application         *Application                    `json:"application"`
	ApplicationFee      int64                           `json:"application_fee"`
	CanceledAt          int64                           `json:"canceled_at"`
	CaptureMethod       PaymentIntentCaptureMethod      `json:"capture_method"`
	Charges             *ChargeList                     `json:"charges"`
	ClientSecret        string                          `json:"client_secret"`
	ConfirmationMethod  PaymentIntentConfirmationMethod `json:"confirmation_method"`
	Created             int64                           `json:"created"`
	Currency            string                          `json:"currency"`
	Customer            *Customer                       `json:"customer"`
	Description         string                          `json:"description"`
	Invoice             *Invoice                        `json:"invoice"`
	LastPaymentError    *PaymentIntentLastPaymentError  `json:"last_payment_error"`
	Livemode            bool                            `json:"livemode"`
	ID                  string                          `json:"id"`
	Metadata            map[string]string               `json:"metadata"`
	NextAction          *PaymentIntentNextAction        `json:"next_action"`
	OnBehalfOf          *Account                        `json:"on_behalf_of"`
	PaymentMethod       *PaymentMethod                  `json:"payment_method"`
	PaymentMethodTypes  []string                        `json:"payment_method_types"`
	ReceiptEmail        string                          `json:"receipt_email"`
	Review              *Review                         `json:"review"`
	Shipping            ShippingDetails                 `json:"shipping"`
	Source              *PaymentSource                  `json:"source"`
	StatementDescriptor string                          `json:"statement_descriptor"`
	Status              PaymentIntentStatus             `json:"status"`
	TransferData        *PaymentIntentTransferData      `json:"transfer_data"`
	TransferGroup       string                          `json:"transfer_group"`
}

// PaymentIntentList is a list of payment intents as retrieved from a list endpoint.
type PaymentIntentList struct {
	ListMeta
	Data []*PaymentIntent `json:"data"`
}

// UnmarshalJSON handles deserialization of a Payment Intent.
// This custom unmarshaling is needed because the resulting
// property may be an id or the full struct if it was expanded.
func (p *PaymentIntent) UnmarshalJSON(data []byte) error {
	if id, ok := ParseID(data); ok {
		p.ID = id
		return nil
	}

	type paymentintent PaymentIntent
	var v paymentintent
	if err := json.Unmarshal(data, &v); err != nil {
		return err
	}

	*p = PaymentIntent(v)
	return nil
}
<commit_msg>Add support for listing PaymentIntents by customer<commit_after>package stripe

import (
	"encoding/json"
)

// PaymentIntentCancellationReason is the list of allowed values for the cancelation reason.
type PaymentIntentCancellationReason string

// List of values that PaymentIntentCaptureMethod can take.
const (
	PaymentIntentCancellationReasonDuplicate           PaymentIntentCancellationReason = "duplicate"
	PaymentIntentCancellationReasonFraudulent          PaymentIntentCancellationReason = "fraudulent"
	PaymentIntentCancellationReasonRequestedByCustomer PaymentIntentCancellationReason = "requested_by_customer"
)

// PaymentIntentCaptureMethod is the list of allowed values for the capture method.
type PaymentIntentCaptureMethod string

// List of values that PaymentIntentCaptureMethod can take.
const (
	PaymentIntentCaptureMethodAutomatic PaymentIntentCaptureMethod = "automatic"
	PaymentIntentCaptureMethodManual    PaymentIntentCaptureMethod = "manual"
)

// PaymentIntentConfirmationMethod is the list of allowed values for the confirmation method.
type PaymentIntentConfirmationMethod string

// List of values that PaymentIntentConfirmationMethod can take.
const (
	PaymentIntentConfirmationMethodAutomatic PaymentIntentConfirmationMethod = "automatic"
	PaymentIntentConfirmationMethodManual    PaymentIntentConfirmationMethod = "manual"
	// The following constants are considered deprecated
	PaymentIntentConfirmationMethodPublishable PaymentIntentConfirmationMethod = "publishable"
	PaymentIntentConfirmationMethodSecret      PaymentIntentConfirmationMethod = "secret"
)

// PaymentIntentNextActionType is the list of allowed values for the next action's type.
type PaymentIntentNextActionType string

// List of values that PaymentIntentNextActionType can take.
const (
	PaymentIntentNextActionTypeRedirectToURL PaymentIntentNextActionType = "redirect_to_url"
)

// PaymentIntentStatus is the list of allowed values for the payment intent's status.
type PaymentIntentStatus string

// List of values that PaymentIntentStatus can take.
const (
	PaymentIntentStatusCanceled              PaymentIntentStatus = "canceled"
	PaymentIntentStatusProcessing            PaymentIntentStatus = "processing"
	PaymentIntentStatusRequiresAction        PaymentIntentStatus = "requires_action"
	PaymentIntentStatusRequiresCapture       PaymentIntentStatus = "requires_capture"
	PaymentIntentStatusRequiresConfirmation  PaymentIntentStatus = "requires_confirmation"
	PaymentIntentStatusRequiresPaymentMethod PaymentIntentStatus = "requires_payment_method"
	PaymentIntentStatusSucceeded             PaymentIntentStatus = "succeeded"
)

// PaymentIntentCancelParams is the set of parameters that can be used when canceling a payment intent.
type PaymentIntentCancelParams struct {
	Params             `form:"*"`
	CancellationReason *string `form:"cancellation_reason"`
}

// PaymentIntentCaptureParams is the set of parameters that can be used when capturing a payment intent.
type PaymentIntentCaptureParams struct {
	Params               `form:"*"`
	AmountToCapture      *int64                           `form:"amount_to_capture"`
	ApplicationFeeAmount *int64                           `form:"application_fee_amount"`
	TransferData         *PaymentIntentTransferDataParams `form:"transfer_data"`
}

// PaymentIntentConfirmParams is the set of parameters that can be used when confirming a payment intent.
type PaymentIntentConfirmParams struct {
	Params            `form:"*"`
	PaymentMethod     *string                `form:"payment_method"`
	ReceiptEmail      *string                `form:"receipt_email"`
	ReturnURL         *string                `form:"return_url"`
	SavePaymentMethod *bool                  `form:"save_payment_method"`
	Shipping          *ShippingDetailsParams `form:"shipping"`
	Source            *string                `form:"source"`
}

// PaymentIntentTransferDataParams is the set of parameters allowed for the transfer hash.
type PaymentIntentTransferDataParams struct {
	Amount      *int64  `form:"amount"`
	Destination *string `form:"destination"`
}

// PaymentIntentParams is the set of parameters that can be used when handling a payment intent.
type PaymentIntentParams struct {
	Params               `form:"*"`
	Amount               *int64                           `form:"amount"`
	ApplicationFeeAmount *int64                           `form:"application_fee_amount"`
	CaptureMethod        *string                          `form:"capture_method"`
	Confirm              *bool                            `form:"confirm"`
	ConfirmationMethod   *string                          `form:"confirmation_method"`
	Currency             *string                          `form:"currency"`
	Customer             *string                          `form:"customer"`
	Description          *string                          `form:"description"`
	OnBehalfOf           *string                          `form:"on_behalf_of"`
	PaymentMethod        *string                          `form:"payment_method"`
	PaymentMethodTypes   []*string                        `form:"payment_method_types"`
	ReceiptEmail         *string                          `form:"receipt_email"`
	ReturnURL            *string                          `form:"return_url"`
	SavePaymentMethod    *bool                            `form:"save_payment_method"`
	Shipping             *ShippingDetailsParams           `form:"shipping"`
	Source               *string                          `form:"source"`
	StatementDescriptor  *string                          `form:"statement_descriptor"`
	TransferData         *PaymentIntentTransferDataParams `form:"transfer_data"`
	TransferGroup        *string                          `form:"transfer_group"`
}

// PaymentIntentListParams is the set of parameters that can be used when listing payment intents.
// For more details see https://stripe.com/docs/api#list_payouts.
type PaymentIntentListParams struct {
	ListParams   `form:"*"`
	Created      *int64            `form:"created"`
	CreatedRange *RangeQueryParams `form:"created"`
	Customer     *string           `form:"customer"`
}

// PaymentIntentLastPaymentError represents the last error happening on a payment intent.
type PaymentIntentLastPaymentError struct {
	Charge        string         `json:"charge"`
	Code          string         `json:"code"`
	DeclineCode   string         `json:"decline_code"`
	DocURL        string         `json:"doc_url"`
	Message       string         `json:"message"`
	Param         string         `json:"param"`
	PaymentIntent *PaymentIntent `json:"payment_intent"`
	PaymentMethod *PaymentMethod `json:"payment_method"`
	Source        *PaymentSource `json:"source"`
	Type          ErrorType      `json:"type"`
}

// PaymentIntentNextActionRedirectToURL represents the resource for the next action of type
// "redirect_to_url".
type PaymentIntentNextActionRedirectToURL struct {
	ReturnURL string `json:"return_url"`
	URL       string `json:"url"`
}

// PaymentIntentNextAction represents the type of action to take on a payment intent.
type PaymentIntentNextAction struct {
	RedirectToURL *PaymentIntentNextActionRedirectToURL `json:"redirect_to_url"`
	Type          PaymentIntentNextActionType           `json:"type"`
}

// PaymentIntentTransferData represents the information for the transfer associated with a payment intent.
type PaymentIntentTransferData struct {
	Amount      int64    `json:"amount"`
	Destination *Account `json:"destination"`
}

// PaymentIntent is the resource representing a Stripe payout.
// For more details see https://stripe.com/docs/api#payment_intents.
type PaymentIntent struct {
	Amount              int64                           `json:"amount"`
	AmountCapturable    int64                           `json:"amount_capturable"`
	AmountReceived      int64                           `json:"amount_received"`
	Application         *Application                    `json:"application"`
	ApplicationFee      int64                           `json:"application_fee"`
	CanceledAt          int64                           `json:"canceled_at"`
	CaptureMethod       PaymentIntentCaptureMethod      `json:"capture_method"`
	Charges             *ChargeList                     `json:"charges"`
	ClientSecret        string                          `json:"client_secret"`
	ConfirmationMethod  PaymentIntentConfirmationMethod `json:"confirmation_method"`
	Created             int64                           `json:"created"`
	Currency            string                          `json:"currency"`
	Customer            *Customer                       `json:"customer"`
	Description         string                          `json:"description"`
	Invoice             *Invoice                        `json:"invoice"`
	LastPaymentError    *PaymentIntentLastPaymentError  `json:"last_payment_error"`
	Livemode            bool                            `json:"livemode"`
	ID                  string                          `json:"id"`
	Metadata            map[string]string               `json:"metadata"`
	NextAction          *PaymentIntentNextAction        `json:"next_action"`
	OnBehalfOf          *Account                        `json:"on_behalf_of"`
	PaymentMethod       *PaymentMethod                  `json:"payment_method"`
	PaymentMethodTypes  []string                        `json:"payment_method_types"`
	ReceiptEmail        string                          `json:"receipt_email"`
	Review              *Review                         `json:"review"`
	Shipping            ShippingDetails                 `json:"shipping"`
	Source              *PaymentSource                  `json:"source"`
	StatementDescriptor string                          `json:"statement_descriptor"`
	Status              PaymentIntentStatus             `json:"status"`
	TransferData        *PaymentIntentTransferData      `json:"transfer_data"`
	TransferGroup       string                          `json:"transfer_group"`
}

// PaymentIntentList is a list of payment intents as retrieved from a list endpoint.
type PaymentIntentList struct {
	ListMeta
	Data []*PaymentIntent `json:"data"`
}

// UnmarshalJSON handles deserialization of a Payment Intent.
// This custom unmarshaling is needed because the resulting
// property may be an id or the full struct if it was expanded.
func (p *PaymentIntent) UnmarshalJSON(data []byte) error {
	if id, ok := ParseID(data); ok {
		p.ID = id
		return nil
	}

	type paymentintent PaymentIntent
	var v paymentintent
	if err := json.Unmarshal(data, &v); err != nil {
		return err
	}

	*p = PaymentIntent(v)
	return nil
}
<|endoftext|>


<commit_before>package bloom

import (
	"encoding"
	"encoding/binary"
	"errors"
	"fmt"
	"math"

	"github.com/dchest/siphash"
)

// Dynamic is a Bloom filter that doesn't need a pre-set size. The idea comes
// from http://gsd.di.uminho.pt/members/cbm/ps/dbloom.pdf
type Dynamic struct {
	fs []*Filter
}

// NewDynamic creates a new Bloom filter for an unbounded number of items with
// probability p. See New for more information.
func NewDynamic(p float64) *Dynamic {
	return &Dynamic{fs: []*Filter{New(4096, p)}}
}

func (d *Dynamic) AddBytes(key []byte) {
	f := d.fs[len(d.fs)-1]
	f.AddBytes(key)

	//fmt.Println(f.Size(), f.N, f.P, len(d.fs))
	if uint64(f.Size()) >= f.N {
		n := f.N * 2

		// Try and catch a Dynamic filter that grows too large and give a nice
		// message.
		const maxInt = uint64(int(^uint(0) >> 1)) // largest platform int
		if n < f.N || n >= maxInt {
			panic("bloom.Dynamic: too large")
		}
		d.fs = append(d.fs, New(int(n), f.P*0.85))
	}
}

func (d *Dynamic) Add(key string) { d.AddBytes(toBytes(key)) }

func (d *Dynamic) HasBytes(key []byte) bool {
	for _, f := range d.fs {
		if f.HasBytes(key) {
			return true
		}
	}
	return false
}

func (d *Dynamic) Has(key string) bool { return d.HasBytes(toBytes(key)) }

// Filter is a Bloom filter.
type Filter struct {
	N uint64  // original number of items
	P float64 // original p value

	bits     []uint64 // bit array
	nbits    uint64   // number of usable bits
	hashes   uint64   // number of hash functions
	popcount int      // popcount
}

// New creates a new Bloom filter for n items with probability p.
// p should be the between 0 and 1 and indicate the probability of false
// positives wanted. n should be a positive integer describing the number of
// items in the filter.
func New(n int, p float64) *Filter {
	if n < 0 {
		panic("bloom.New: negative count")
	}

	// Optimal number of bits, m, is
	// m = -(n log(p) / pow(log(2), 2))

	// pow(log(2), 2)
	const lnsq = 0.480453013918201424667102526326664971730552951594545586866864133623665382259834472199948263443926990932715597661358897481255128413358268503177555294880844290839184664798896404335252423673643658092881230886029639112807153031

	n0 := float64(n)
	m := -math.Ceil(n0 * math.Log(p) / lnsq)

	// Rounding up to nearest 64 simplifies our filter at the expense of using
	// slightly more space. Patches are welcome :-)
	nbits := ((uint64(m) + mod64) >> div64) * wordBits
	hashes := uint64(-int(math.Ceil(math.Log(p) / math.Ln2)))

	fmt.Println(n, p, nbits, nbits>>div64, hashes)

	return &Filter{
		N:      uint64(n),
		P:      p,
		bits:   make([]uint64, nbits>>div64),
		nbits:  nbits,
		hashes: hashes,
	}
}

const (
	k0 = 17697571051839533707
	k1 = 15128385881502100741

	wordBits  = 64               // bits per word
	div64     = 6                // division by 64
	mod64     = wordBits - 1     // remainder mod 64
	div8      = 3                // division by 8
	wordBytes = wordBits >> div8 // bytes per word
)

// Ideally the API would be reversed (e.g. AddBytes calling Add), but compiler
// optimizations and safe use of 'unsafe' dictate we should convert the string
// to []bytes instead.

// AddBytes adds a key to the filter.
func (f *Filter) AddBytes(key []byte) {
	// "Less Hashing, Same Performance: Building a Better Bloom Filter"
	// (Kirsch and Mitzenmacher) tells us gi(x) = h1(x) + ih2(x).
	a, b := siphash.Hash128(k0, k1, key)
	for h := uint64(0); h != f.hashes; h++ {
		i := (a + b*h) % f.nbits
		mask := uint64(1) << (i & mod64)
		if f.bits[i>>div64]&mask == 0 {
			f.bits[i>>div64] |= mask
			f.popcount++
		}
	}
}

// Add adds a key to the filter.
func (f *Filter) Add(key string) { f.AddBytes(toBytes(key)) }

// HasBytes returns true if the key probably exists in the filter.
func (f *Filter) HasBytes(key []byte) bool {
	a, b := siphash.Hash128(k0, k1, key)
	for h := uint64(0); h != f.hashes; h++ {
		i := (a + b*h) % f.nbits
		if (f.bits[i>>div64])&(1<<(i&mod64)) == 0 {
			return false
		}
	}
	return true
}

// Has returns true if the key probably exists in the filter.
func (f *Filter) Has(key string) bool { return f.HasBytes(toBytes(key)) }

// Size returns the approximate number of items in the filter. At most it
// should be within 3.5% of the actual amount, assuming the number of items in
// the filter is <= the original size of the filter.
//
// Full disclosure: in practice, the variance is less than 1%; 3.5% is absolute
// maximum, tested up to 1e7 elements (see bloom_test.go). The algorithm is
// from http://pubs.acs.org/doi/abs/10.1021/ci600526a, but since I do not have
// access to ACS I do not know if the authors of the authors published the
// variance of the algorithm.
func (f *Filter) Size() int {
	m := float64(f.nbits)
	k := float64(f.hashes)
	X := float64(f.popcount)
	// n* = -(m/k) ln[1 - x/m]
	return int(math.Floor(-((m / k) * math.Log(1-(X/m))) + 0.5))
}

// Stats returns basic memory information. hashes is the number of hash
// functions and nbits is the number of usable bits. The total number of bits
// allocated by the filter will be nbits rounded up to the nearest multiple of
// 64.
func (f *Filter) Stats() (hashes, nbits uint64) {
	return f.hashes, f.nbits
}

const ver = 1 // marshal version

// MarshalBinary implements encoding.BinaryMarshaler.
func (f *Filter) MarshalBinary() (data []byte, err error) {
	data = make([]byte, 0 /* formatting :-) */ +
		/* version */ 1+
		/* n       */ wordBytes+
		/* hashes  */ wordBytes+
		/* bits    */ (f.nbits>>div8),
	)
	data[0] = ver
	binary.LittleEndian.PutUint64(data[1:], f.N)
	binary.LittleEndian.PutUint64(data[1+wordBytes:], f.hashes)
	for i, w := range f.bits {
		offset := 1 + ((i + 2) * wordBytes)
		binary.LittleEndian.PutUint64(data[offset:], w)
	}
	return data, nil
}

// UnmarshalBinary implements encoding.BinaryUnmarshaler.
func (f *Filter) UnmarshalBinary(data []byte) error {
	if len(data) < 1+(wordBytes*2) {
		return errors.New("bloom.UnmarshalBinary: data too short, unknown encoding")
	}
	if data[0] != ver {
		return errors.New("bloom.UnmarshalBinary: unknown encoding")
	}
	data = data[1:]

	f.N = binary.LittleEndian.Uint64(data)
	data = data[wordBytes:]

	f.hashes = binary.LittleEndian.Uint64(data)
	data = data[wordBytes:]

	f.nbits = uint64(len(data)) << div8
	f.bits = make([]uint64, f.nbits>>div64)
	for i := range f.bits {
		f.bits[i] = binary.LittleEndian.Uint64(data[i*wordBytes:])
	}
	return nil
}

var (
	_ interface {
		encoding.BinaryMarshaler
		encoding.BinaryUnmarshaler
	} = (*Filter)(nil)
)
<commit_msg>remove missed fmt calls<commit_after>package bloom

import (
	"encoding"
	"encoding/binary"
	"errors"
	"math"

	"github.com/dchest/siphash"
)

// Dynamic is a Bloom filter that doesn't need a pre-set size. The idea comes
// from http://gsd.di.uminho.pt/members/cbm/ps/dbloom.pdf
type Dynamic struct {
	fs []*Filter
}

// NewDynamic creates a new Bloom filter for an unbounded number of items with
// probability p. See New for more information.
func NewDynamic(p float64) *Dynamic {
	return &Dynamic{fs: []*Filter{New(4096, p)}}
}

func (d *Dynamic) AddBytes(key []byte) {
	f := d.fs[len(d.fs)-1]
	f.AddBytes(key)

	if uint64(f.Size()) >= f.N {
		n := f.N * 2

		// Try and catch a Dynamic filter that grows too large and give a nice
		// message.
		const maxInt = uint64(int(^uint(0) >> 1)) // largest platform int
		if n < f.N || n >= maxInt {
			panic("bloom.Dynamic: too large")
		}
		d.fs = append(d.fs, New(int(n), f.P*0.85))
	}
}

func (d *Dynamic) Add(key string) { d.AddBytes(toBytes(key)) }

func (d *Dynamic) HasBytes(key []byte) bool {
	for _, f := range d.fs {
		if f.HasBytes(key) {
			return true
		}
	}
	return false
}

func (d *Dynamic) Has(key string) bool { return d.HasBytes(toBytes(key)) }

// Filter is a Bloom filter.
type Filter struct {
	N uint64  // original number of items
	P float64 // original p value

	bits     []uint64 // bit array
	nbits    uint64   // number of usable bits
	hashes   uint64   // number of hash functions
	popcount int      // popcount
}

// New creates a new Bloom filter for n items with probability p.
// p should be the between 0 and 1 and indicate the probability of false
// positives wanted. n should be a positive integer describing the number of
// items in the filter.
func New(n int, p float64) *Filter {
	if n < 0 {
		panic("bloom.New: negative count")
	}

	// Optimal number of bits, m, is
	// m = -(n log(p) / pow(log(2), 2))

	// pow(log(2), 2)
	const lnsq = 0.480453013918201424667102526326664971730552951594545586866864133623665382259834472199948263443926990932715597661358897481255128413358268503177555294880844290839184664798896404335252423673643658092881230886029639112807153031

	n0 := float64(n)
	m := -math.Ceil(n0 * math.Log(p) / lnsq)

	// Rounding up to nearest 64 simplifies our filter at the expense of using
	// slightly more space. Patches are welcome :-)
	nbits := ((uint64(m) + mod64) >> div64) * wordBits
	hashes := uint64(-int(math.Ceil(math.Log(p) / math.Ln2)))

	return &Filter{
		N:      uint64(n),
		P:      p,
		bits:   make([]uint64, nbits>>div64),
		nbits:  nbits,
		hashes: hashes,
	}
}

const (
	k0 = 17697571051839533707
	k1 = 15128385881502100741

	wordBits  = 64               // bits per word
	div64     = 6                // division by 64
	mod64     = wordBits - 1     // remainder mod 64
	div8      = 3                // division by 8
	wordBytes = wordBits >> div8 // bytes per word
)

// Ideally the API would be reversed (e.g. AddBytes calling Add), but compiler
// optimizations and safe use of 'unsafe' dictate we should convert the string
// to []bytes instead.

// AddBytes adds a key to the filter.
func (f *Filter) AddBytes(key []byte) {
	// "Less Hashing, Same Performance: Building a Better Bloom Filter"
	// (Kirsch and Mitzenmacher) tells us gi(x) = h1(x) + ih2(x).
	a, b := siphash.Hash128(k0, k1, key)
	for h := uint64(0); h != f.hashes; h++ {
		i := (a + b*h) % f.nbits
		mask := uint64(1) << (i & mod64)
		if f.bits[i>>div64]&mask == 0 {
			f.bits[i>>div64] |= mask
			f.popcount++
		}
	}
}

// Add adds a key to the filter.
func (f *Filter) Add(key string) { f.AddBytes(toBytes(key)) }

// HasBytes returns true if the key probably exists in the filter.
func (f *Filter) HasBytes(key []byte) bool {
	a, b := siphash.Hash128(k0, k1, key)
	for h := uint64(0); h != f.hashes; h++ {
		i := (a + b*h) % f.nbits
		if (f.bits[i>>div64])&(1<<(i&mod64)) == 0 {
			return false
		}
	}
	return true
}

// Has returns true if the key probably exists in the filter.
func (f *Filter) Has(key string) bool { return f.HasBytes(toBytes(key)) }

// Size returns the approximate number of items in the filter. At most it
// should be within 3.5% of the actual amount, assuming the number of items in
// the filter is <= the original size of the filter.
//
// Full disclosure: in practice, the variance is less than 1%; 3.5% is absolute
// maximum, tested up to 1e7 elements (see bloom_test.go). The algorithm is
// from http://pubs.acs.org/doi/abs/10.1021/ci600526a, but since I do not have
// access to ACS I do not know if the authors of the authors published the
// variance of the algorithm.
func (f *Filter) Size() int {
	m := float64(f.nbits)
	k := float64(f.hashes)
	X := float64(f.popcount)
	// n* = -(m/k) ln[1 - x/m]
	return int(math.Floor(-((m / k) * math.Log(1-(X/m))) + 0.5))
}

// Stats returns basic memory information. hashes is the number of hash
// functions and nbits is the number of usable bits. The total number of bits
// allocated by the filter will be nbits rounded up to the nearest multiple of
// 64.
func (f *Filter) Stats() (hashes, nbits uint64) {
	return f.hashes, f.nbits
}

const ver = 1 // marshal version

// MarshalBinary implements encoding.BinaryMarshaler.
func (f *Filter) MarshalBinary() (data []byte, err error) {
	data = make([]byte, 0 /* formatting :-) */ +
		/* version */ 1+
		/* n       */ wordBytes+
		/* hashes  */ wordBytes+
		/* bits    */ (f.nbits>>div8),
	)
	data[0] = ver
	binary.LittleEndian.PutUint64(data[1:], f.N)
	binary.LittleEndian.PutUint64(data[1+wordBytes:], f.hashes)
	for i, w := range f.bits {
		offset := 1 + ((i + 2) * wordBytes)
		binary.LittleEndian.PutUint64(data[offset:], w)
	}
	return data, nil
}

// UnmarshalBinary implements encoding.BinaryUnmarshaler.
func (f *Filter) UnmarshalBinary(data []byte) error {
	if len(data) < 1+(wordBytes*2) {
		return errors.New("bloom.UnmarshalBinary: data too short, unknown encoding")
	}
	if data[0] != ver {
		return errors.New("bloom.UnmarshalBinary: unknown encoding")
	}
	data = data[1:]

	f.N = binary.LittleEndian.Uint64(data)
	data = data[wordBytes:]

	f.hashes = binary.LittleEndian.Uint64(data)
	data = data[wordBytes:]

	f.nbits = uint64(len(data)) << div8
	f.bits = make([]uint64, f.nbits>>div64)
	for i := range f.bits {
		f.bits[i] = binary.LittleEndian.Uint64(data[i*wordBytes:])
	}
	return nil
}

var (
	_ interface {
		encoding.BinaryMarshaler
		encoding.BinaryUnmarshaler
	} = (*Filter)(nil)
)
<|endoftext|>


<commit_before>package tradier

import (
	"net/http"
	"testing"
	"time"

	"github.com/kylelemons/godebug/pretty"
)

func TestActivityService_List(t *testing.T) {
	setup()
	defer teardown()

	mux.HandleFunc("/user/orders", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")

		w.WriteHeader(http.StatusOK)
		w.Write(userOrdersJSON)
	})

	got, _, err := client.User.Orders()
	if err != nil {
		t.Errorf("User.Orders returned error: %v", err)
	}

	if diff := pretty.Compare(wantUserOrders, got); diff != "" {
		t.Errorf("diff: %s", diff)
	}
}

var userOrdersJSON = []byte(`{
  "accounts": {
    "account": [
      {
        "account_number": "6YA05991",
        "orders": "null"
      },
      {
        "account_number": "6YA05708",
        "orders": "null"
      }
    ]
  }
}`)

var (
	createdDate    = time.Date(2016, 8, 23, 05, 17, 37, 617000000, time.UTC)
	transitionDate = time.Date(2016, 8, 23, 12, 15, 07, 268000000, time.UTC)
)

var wantUserOrders = &User{
	Accounts: &Accounts{
		{
			AccountNumber: String("6YA05991"),
			Orders:        &Orders{},
		},
		{
			AccountNumber: String("6YA05708"),
			Orders:        &Orders{},
		},
	},
}
<commit_msg>More comprehensive user orders test<commit_after>package tradier

import (
	"net/http"
	"testing"
	"time"

	"github.com/kylelemons/godebug/pretty"
)

func TestActivityService_List(t *testing.T) {
	setup()
	defer teardown()

	mux.HandleFunc("/user/orders", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")

		w.WriteHeader(http.StatusOK)
		w.Write(userOrdersJSON)
	})

	got, _, err := client.User.Orders()
	if err != nil {
		t.Errorf("User.Orders returned error: %v", err)
	}

	if diff := pretty.Compare(wantUserOrders, got); diff != "" {
		t.Errorf("diff: %s", diff)
	}
}

var userOrdersJSON = []byte(`{
  "accounts": {
    "account": [
      {
        "account_number": "6YA05991",
        "orders": {
          "order": {
            "id": 182042,
            "type": "market",
            "symbol": "GOOGL",
            "side": "buy",
            "quantity": 1,
            "status": "pending",
            "duration": "gtc",
            "avg_fill_price": 0,
            "exec_quantity": 0,
            "last_fill_price": 0,
            "last_fill_quantity": 0,
            "remaining_quantity": 1,
            "create_date": "2016-08-23T05:17:37.617Z",
            "transaction_date": "2016-08-23T12:15:07.268Z",
            "class": "equity"
          }
        }
      },
      {
        "account_number": "6YA05708",
        "orders": "null"
      }
    ]
  }
}`)

var (
	createdDate    = time.Date(2016, 8, 23, 05, 17, 37, 617000000, time.UTC)
	transitionDate = time.Date(2016, 8, 23, 12, 15, 07, 268000000, time.UTC)
)

var wantUserOrders = &User{
	Accounts: &Accounts{
		{
			AccountNumber: String("6YA05991"),
			Orders: &Orders{
				{
					ID:                Int(182042),
					Type:              String("market"),
					Symbol:            String("GOOGL"),
					Side:              String("buy"),
					Quantity:          Float64(1),
					Status:            String("pending"),
					Duration:          String("gtc"),
					AvgFillPrice:      Float64(0),
					ExecQuantity:      Float64(0),
					LastFillPrice:     Float64(0),
					LastFillQuantity:  Float64(0),
					RemainingQuantity: Float64(1),
					CreateDate:        &createdDate,
					TransactionDate:   &transitionDate,
					Class:             String("equity"),
				},
			},
		},
		{
			AccountNumber: String("6YA05708"),
			Orders:        &Orders{},
		},
	},
}
<|endoftext|>


<commit_before>package v2

import (
	"os"

	"code.cloudfoundry.org/cli/actors/v2actions"
	oldCmd "code.cloudfoundry.org/cli/cf/cmd"
	"code.cloudfoundry.org/cli/commands"
	"code.cloudfoundry.org/cli/commands/v2/common"
)

//go:generate counterfeiter . DeleteOrphanedRoutesActor

type DeleteOrphanedRoutesActor interface {
	GetOrphanedRoutesBySpace(spaceGUID string) ([]v2actions.Route, v2actions.Warnings, error)
	DeleteRouteByGUID(routeGUID string) (v2actions.Warnings, error)
}

type DeleteOrphanedRoutesCommand struct {
	Force           bool        `short:"f" description:"Force deletion without confirmation"`
	usage           interface{} `usage:"CF_NAME delete-orphaned-routes [-f]"`
	relatedCommands interface{} `related_commands:"delete-route, routes"`

	UI     commands.UI
	Actor  DeleteOrphanedRoutesActor
	Config commands.Config
}

func (cmd *DeleteOrphanedRoutesCommand) Setup(config commands.Config, ui commands.UI) error {
	cmd.UI = ui
	cmd.Config = config

	client, err := common.NewCloudControllerClient(config)
	if err != nil {
		return err
	}
	cmd.Actor = v2actions.NewActor(client)

	return nil
}

func (cmd *DeleteOrphanedRoutesCommand) Execute(args []string) error {
	if cmd.Config.Experimental() == false {
		oldCmd.Main(os.Getenv("CF_TRACE"), os.Args)
		return nil
	}

	cmd.UI.DisplayText(ExperimentalWarning)
	cmd.UI.DisplayNewline()

	err := common.CheckTarget(cmd.Config, true, true)
	if err != nil {
		return err
	}

	if !cmd.Force {
		deleteOrphanedRoutes, err := cmd.UI.DisplayBoolPrompt("Really delete orphaned routes?", false)
		if err != nil {
			return err
		}

		if !deleteOrphanedRoutes {
			return nil
		}
	}

	user, err := cmd.Config.CurrentUser()
	if err != nil {
		return err
	}

	cmd.UI.DisplayHeaderFlavorText("Getting routes as {{.CurrentUser}} ...", map[string]interface{}{
		"CurrentUser": user.Name,
	})
	cmd.UI.DisplayNewline()

	routes, warnings, err := cmd.Actor.GetOrphanedRoutesBySpace(cmd.Config.TargetedSpace().GUID)
	cmd.UI.DisplayWarnings(warnings)
	if err != nil {
		switch err.(type) {
		case v2actions.DomainNotFoundError:
			return err // Should this error be handled differently? Note: it will 99.99% never occur.
		case v2actions.OrphanedRoutesNotFoundError:
			// Do nothing to parity the existing behavior
		default:
			return common.HandleError(err)
		}
	}

	for _, route := range routes {
		cmd.UI.DisplayHeaderFlavorText("Deleting route {{.Route}} ...", map[string]interface{}{
			"Route": route.String(),
		})

		warnings, err = cmd.Actor.DeleteRouteByGUID(route.GUID)
		cmd.UI.DisplayWarnings(warnings)
		if err != nil {
			return err
		}
	}

	cmd.UI.DisplayOK()

	return nil
}
<commit_msg>handle the error properly<commit_after>package v2

import (
	"os"

	"code.cloudfoundry.org/cli/actors/v2actions"
	oldCmd "code.cloudfoundry.org/cli/cf/cmd"
	"code.cloudfoundry.org/cli/commands"
	"code.cloudfoundry.org/cli/commands/v2/common"
)

//go:generate counterfeiter . DeleteOrphanedRoutesActor

type DeleteOrphanedRoutesActor interface {
	GetOrphanedRoutesBySpace(spaceGUID string) ([]v2actions.Route, v2actions.Warnings, error)
	DeleteRouteByGUID(routeGUID string) (v2actions.Warnings, error)
}

type DeleteOrphanedRoutesCommand struct {
	Force           bool        `short:"f" description:"Force deletion without confirmation"`
	usage           interface{} `usage:"CF_NAME delete-orphaned-routes [-f]"`
	relatedCommands interface{} `related_commands:"delete-route, routes"`

	UI     commands.UI
	Actor  DeleteOrphanedRoutesActor
	Config commands.Config
}

func (cmd *DeleteOrphanedRoutesCommand) Setup(config commands.Config, ui commands.UI) error {
	cmd.UI = ui
	cmd.Config = config

	client, err := common.NewCloudControllerClient(config)
	if err != nil {
		return err
	}
	cmd.Actor = v2actions.NewActor(client)

	return nil
}

func (cmd *DeleteOrphanedRoutesCommand) Execute(args []string) error {
	if cmd.Config.Experimental() == false {
		oldCmd.Main(os.Getenv("CF_TRACE"), os.Args)
		return nil
	}

	cmd.UI.DisplayText(ExperimentalWarning)
	cmd.UI.DisplayNewline()

	err := common.CheckTarget(cmd.Config, true, true)
	if err != nil {
		return err
	}

	if !cmd.Force {
		deleteOrphanedRoutes, err := cmd.UI.DisplayBoolPrompt("Really delete orphaned routes?", false)
		if err != nil {
			return err
		}

		if !deleteOrphanedRoutes {
			return nil
		}
	}

	user, err := cmd.Config.CurrentUser()
	if err != nil {
		return err
	}

	cmd.UI.DisplayHeaderFlavorText("Getting routes as {{.CurrentUser}} ...", map[string]interface{}{
		"CurrentUser": user.Name,
	})
	cmd.UI.DisplayNewline()

	routes, warnings, err := cmd.Actor.GetOrphanedRoutesBySpace(cmd.Config.TargetedSpace().GUID)
	cmd.UI.DisplayWarnings(warnings)
	if err != nil {
		switch err.(type) {
		case v2actions.DomainNotFoundError:
			return err // Should this error be handled differently? Note: it will 99.99% never occur.
		case v2actions.OrphanedRoutesNotFoundError:
			// Do nothing to parity the existing behavior
		default:
			return common.HandleError(err)
		}
	}

	for _, route := range routes {
		cmd.UI.DisplayHeaderFlavorText("Deleting route {{.Route}} ...", map[string]interface{}{
			"Route": route.String(),
		})

		warnings, err = cmd.Actor.DeleteRouteByGUID(route.GUID)
		cmd.UI.DisplayWarnings(warnings)
		if err != nil {
			return common.HandleError(err)
		}
	}

	cmd.UI.DisplayOK()

	return nil
}
<|endoftext|>


<commit_before>package main

import "github.com/tmrts/boilr/pkg/cmd"

func main() {
	cmd.Run()
}
<commit_msg>*: init boilr automatically<commit_after>package main

import (
  "fmt"
  "github.com/tmrts/boilr/pkg/boilr"
  "github.com/tmrts/boilr/pkg/cmd"
  "github.com/tmrts/boilr/pkg/util/exit"
  "github.com/tmrts/boilr/pkg/util/osutil"
)

func main() {
  if exists, err := osutil.DirExists(boilr.Configuration.TemplateDirPath); ! exists {
    if err := osutil.CreateDirs(boilr.Configuration.TemplateDirPath); err != nil {
      exit.Error(fmt.Errorf("Tried to initialise your template directory, but it has failed: %s", err))
    }
  } else if err != nil {
    exit.Error(fmt.Errorf("Failed to init: %s", err))
  }

  cmd.Run()
}

<|endoftext|>


<commit_before>/*
Copyright 2015 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

/*
Package bigtable is an API to Google Cloud Bigtable.

See https://cloud.google.com/bigtable/docs/ for general product documentation.

See https://godoc.org/cloud.google.com/go for authentication, timeouts,
connection pooling and similar aspects of this package.

Setup and Credentials

Use NewClient or NewAdminClient to create a client that can be used to access
the data or admin APIs respectively. Both require credentials that have
permission to access the Cloud Bigtable API.

If your program is run on Google App Engine or Google Compute Engine, using the
Application Default Credentials
(https://developers.google.com/accounts/docs/application-default-credentials)
is the simplest option. Those credentials will be used by default when NewClient
or NewAdminClient are called.

To use alternate credentials, pass them to NewClient or NewAdminClient using
option.WithTokenSource. For instance, you can use service account credentials
by visiting https://cloud.google.com/console/project/_/apiui/credential,
creating a new OAuth "Client ID", storing the JSON key somewhere accessible,
and writing

	jsonKey, err := ioutil.ReadFile(pathToKeyFile)
	if err != nil {
		// TODO: handle err.
	}
	config, err := google.JWTConfigFromJSON(jsonKey, bigtable.Scope) // or bigtable.AdminScope, etc.
	if err != nil {
		// TODO: handle err.
	}
	client, err := bigtable.NewClient(ctx, project, instance, option.WithTokenSource(config.TokenSource(ctx)))
	if err != nil {
		// TODO: handle err.
	}
	_ = client // TODO: use client.

Here, `google` means the golang.org/x/oauth2/google package
and `option` means the google.golang.org/api/option package.

Reading

The principal way to read from a Bigtable is to use the ReadRows method on
*Table. A RowRange specifies a contiguous portion of a table. A Filter may be
provided through RowFilter to limit or transform the data that is returned.

	tbl := client.Open("mytable")

	// Read all the rows starting with "com.google.", but only fetch the columns
	// in the "links" family.
	rr := bigtable.PrefixRange("com.google.")
	err := tbl.ReadRows(ctx, rr, func(r Row) bool {
		// TODO: do something with r.
		return true // Keep going.
	}, bigtable.RowFilter(bigtable.FamilyFilter("links")))
	if err != nil {
		// TODO: handle err.
	}

To read a single row, use the ReadRow helper method:

	r, err := tbl.ReadRow(ctx, "com.google.cloud") // "com.google.cloud" is the entire row key
	if err != nil {
		// TODO: handle err.
	}
	// TODO: use r.

Writing

This API exposes two distinct forms of writing to a Bigtable: a Mutation and a
ReadModifyWrite. The former expresses idempotent operations. The latter
expresses non-idempotent operations and returns the new values of updated cells.
These operations are performed by creating a Mutation or ReadModifyWrite (with
NewMutation or NewReadModifyWrite), building up one or more operations on that,
and then using the Apply or ApplyReadModifyWrite methods on a Table.

For instance, to set a couple of cells in a table:

	tbl := client.Open("mytable")
	mut := bigtable.NewMutation()
	// To use numeric values that will later be incremented,
	// they need to be big-endian encoded as 64-bit integers.
        buf := new(bytes.Buffer)
        initialLinkCount := 1 // The initial number of links.
        if err := binary.Write(buf, binary.BigEndian, initialLinkCount); err != nil {
		// TODO: handle err.
        }
	mut.Set("links", "maps.google.com", bigtable.Now(), buf.Bytes())
	mut.Set("links", "golang.org", bigtable.Now(), buf.Bytes())
	err := tbl.Apply(ctx, "com.google.cloud", mut)
	if err != nil {
		// TODO: handle err.
	}

To increment an encoded value in one cell:

	tbl := client.Open("mytable")
	rmw := bigtable.NewReadModifyWrite()
	rmw.Increment("links", "golang.org", 12) // add 12 to the cell in column "links:golang.org"
	r, err := tbl.ApplyReadModifyWrite(ctx, "com.google.cloud", rmw)
	if err != nil {
		// TODO: handle err.
	}
	// TODO: use r.

Retries

If a read or write operation encounters a transient error it will be retried
until a successful response, an unretryable error or the context deadline is
reached. Non-idempotent writes (where the timestamp is set to ServerTime) will
not be retried. In the case of ReadRows, retried calls will not re-scan rows
that have already been processed.
*/
package bigtable // import "cloud.google.com/go/bigtable"

// Scope constants for authentication credentials. These should be used when
// using credential creation functions such as oauth.NewServiceAccountFromFile.
const (
	// Scope is the OAuth scope for Cloud Bigtable data operations.
	Scope = "https://www.googleapis.com/auth/bigtable.data"
	// ReadonlyScope is the OAuth scope for Cloud Bigtable read-only data
	// operations.
	ReadonlyScope = "https://www.googleapis.com/auth/bigtable.readonly"

	// AdminScope is the OAuth scope for Cloud Bigtable table admin operations.
	AdminScope = "https://www.googleapis.com/auth/bigtable.admin.table"

	// InstanceAdminScope is the OAuth scope for Cloud Bigtable instance (and
	// cluster) admin operations.
	InstanceAdminScope = "https://www.googleapis.com/auth/bigtable.admin.cluster"
)

// clientUserAgent identifies the version of this package.
// It should be bumped upon significant changes only.
const clientUserAgent = "cbt-go/20180601"

// resourcePrefixHeader is the name of the metadata header used to indicate
// the resource being operated on.
const resourcePrefixHeader = "google-cloud-resource-prefix"
<commit_msg>bigtable: clean up docs<commit_after>/*
Copyright 2015 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

/*
Package bigtable is an API to Google Cloud Bigtable.

See https://cloud.google.com/bigtable/docs/ for general product documentation.

See https://godoc.org/cloud.google.com/go for authentication, timeouts,
connection pooling and similar aspects of this package.

Reading

The principal way to read from a Bigtable is to use the ReadRows method on
*Table. A RowRange specifies a contiguous portion of a table. A Filter may be
provided through RowFilter to limit or transform the data that is returned.

	tbl := client.Open("mytable")
	// Read all the rows starting with "com.google.", but only fetch the columns
	// in the "links" family.
	rr := bigtable.PrefixRange("com.google.")
	err := tbl.ReadRows(ctx, rr, func(r Row) bool {
		// TODO: do something with r.
		return true // Keep going.
	}, bigtable.RowFilter(bigtable.FamilyFilter("links")))
	if err != nil {
		// TODO: handle err.
	}

To read a single row, use the ReadRow helper method:

	r, err := tbl.ReadRow(ctx, "com.google.cloud") // "com.google.cloud" is the entire row key
	if err != nil {
		// TODO: handle err.
	}
	// TODO: use r.

Writing

This API exposes two distinct forms of writing to a Bigtable: a Mutation and a
ReadModifyWrite. The former expresses idempotent operations. The latter
expresses non-idempotent operations and returns the new values of updated cells.
These operations are performed by creating a Mutation or ReadModifyWrite (with
NewMutation or NewReadModifyWrite), building up one or more operations on that,
and then using the Apply or ApplyReadModifyWrite methods on a Table.

For instance, to set a couple of cells in a table:

	tbl := client.Open("mytable")
	mut := bigtable.NewMutation()
	// To use numeric values that will later be incremented,
	// they need to be big-endian encoded as 64-bit integers.
	buf := new(bytes.Buffer)
	initialLinkCount := 1 // The initial number of links.
	if err := binary.Write(buf, binary.BigEndian, initialLinkCount); err != nil {
	// TODO: handle err.
	}
	mut.Set("links", "maps.google.com", bigtable.Now(), buf.Bytes())
	mut.Set("links", "golang.org", bigtable.Now(), buf.Bytes())
	err := tbl.Apply(ctx, "com.google.cloud", mut)
	if err != nil {
		// TODO: handle err.
	}

To increment an encoded value in one cell:

	tbl := client.Open("mytable")
	rmw := bigtable.NewReadModifyWrite()
	rmw.Increment("links", "golang.org", 12) // add 12 to the cell in column "links:golang.org"
	r, err := tbl.ApplyReadModifyWrite(ctx, "com.google.cloud", rmw)
	if err != nil {
		// TODO: handle err.
	}
	// TODO: use r.

Retries

If a read or write operation encounters a transient error it will be retried
until a successful response, an unretryable error or the context deadline is
reached. Non-idempotent writes (where the timestamp is set to ServerTime) will
not be retried. In the case of ReadRows, retried calls will not re-scan rows
that have already been processed.
*/
package bigtable // import "cloud.google.com/go/bigtable"

// Scope constants for authentication credentials. These should be used when
// using credential creation functions such as oauth.NewServiceAccountFromFile.
const (
	// Scope is the OAuth scope for Cloud Bigtable data operations.
	Scope = "https://www.googleapis.com/auth/bigtable.data"
	// ReadonlyScope is the OAuth scope for Cloud Bigtable read-only data
	// operations.
	ReadonlyScope = "https://www.googleapis.com/auth/bigtable.readonly"

	// AdminScope is the OAuth scope for Cloud Bigtable table admin operations.
	AdminScope = "https://www.googleapis.com/auth/bigtable.admin.table"

	// InstanceAdminScope is the OAuth scope for Cloud Bigtable instance (and
	// cluster) admin operations.
	InstanceAdminScope = "https://www.googleapis.com/auth/bigtable.admin.cluster"
)

// clientUserAgent identifies the version of this package.
// It should be bumped upon significant changes only.
const clientUserAgent = "cbt-go/20180601"

// resourcePrefixHeader is the name of the metadata header used to indicate
// the resource being operated on.
const resourcePrefixHeader = "google-cloud-resource-prefix"
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"log"
	"net"
	"strconv"
	"sync"

	"github.com/coreos/go-etcd/etcd"
)

type service struct {
	Host string `json:"host"`
	Port int `json:"port"`
}

type host struct {
	key string
	addr string
}

type backends struct {
	path       string
	hosts      []host
	lastIndex  int
	watchIndex uint64
	lock       sync.RWMutex
}

func (b *backends) Dump() {
	for _, v := range(b.hosts) {
		log.Printf("%s -> %s", v.key, v.addr)
	}
}

func (b *backends) Remove(key string) {
	match := -1
	for k, v := range(b.hosts) {
		if v.key == key {
			match = k
		}
	}

	b.hosts = append(b.hosts[:match], b.hosts[match+1:]...)
}

func (b *backends) Update(node *etcd.Node, action string) {
	b.lock.Lock()
	defer b.lock.Unlock()

	log.Printf("key: %s action: %s value: %s", node.Key, action, string(node.Value))

	s := &service{}
	if action == "delete" {
		b.Remove(node.Key)
		return
	}

	err := json.Unmarshal([]byte(node.Value), s)
	if err != nil {
		panic(err)
	}

	addr := net.JoinHostPort(s.Host, strconv.Itoa(s.Port))

	b.hosts = append(b.hosts, host{addr: addr, key: node.Key})

	b.Dump()
}

func (b *backends) Watch(client *etcd.Client) {
	receiver := make(chan *etcd.Response)
	go client.Watch(b.path, uint64(b.watchIndex), true, receiver, nil)

	for {
		resp := <-receiver
		b.Update(resp.Node, resp.Action)
	}
}

func (b *backends) Sync(client *etcd.Client) error {
	resp, err := client.Get(b.path, false, true)

	if err != nil {
		return err
	}

	for _, n := range(resp.Node.Nodes) {
		b.Update(&n, resp.Action)
	}

	// Begin the watch after this sync from the next sync
	b.watchIndex = resp.EtcdIndex + 1

	return nil
}

func (b *backends) Next() string {
	b.lock.RLock()
	defer b.lock.RUnlock()

	if len(b.hosts) == 0 {
		return ""
	}

	index := (b.lastIndex + 1) % len(b.hosts)
	b.lastIndex = index

	return b.hosts[index].addr
}
<commit_msg>feat(backend): improve logging<commit_after>package main

import (
	"encoding/json"
	"log"
	"net"
	"strconv"
	"sync"

	"github.com/coreos/go-etcd/etcd"
)

type service struct {
	Host string `json:"host"`
	Port int `json:"port"`
}

type host struct {
	key string
	addr string
}

type backends struct {
	path       string
	hosts      []host
	lastIndex  int
	watchIndex uint64
	lock       sync.RWMutex
}

func (b *backends) Dump(action string ) {
	for _, v := range(b.hosts) {
		log.Printf("Dump after %s %s -> %s", action, v.key, v.addr)
	}
}

func (b *backends) Remove(key string) {
	match := -1
	for k, v := range(b.hosts) {
		if v.key == key {
			match = k
		}
	}

	b.hosts = append(b.hosts[:match], b.hosts[match+1:]...)
	b.Dump("remove")
}

func (b *backends) Update(node *etcd.Node, action string) {
	b.lock.Lock()
	defer b.lock.Unlock()

	log.Printf("key: %s action: %s value: %s", node.Key, action, string(node.Value))

	s := &service{}
	if action == "delete" || action == "expire" {
		b.Remove(node.Key)
		return
	}

	err := json.Unmarshal([]byte(node.Value), s)
	if err != nil {
		panic(err)
	}

	addr := net.JoinHostPort(s.Host, strconv.Itoa(s.Port))

	b.hosts = append(b.hosts, host{addr: addr, key: node.Key})

	b.Dump(action)
}

func (b *backends) Watch(client *etcd.Client) {
	receiver := make(chan *etcd.Response)
	go client.Watch(b.path, uint64(b.watchIndex), true, receiver, nil)

	for {
		resp := <-receiver
		b.Update(resp.Node, resp.Action)
	}
}

func (b *backends) Sync(client *etcd.Client) error {
	resp, err := client.Get(b.path, false, true)

	if err != nil {
		return err
	}

	for _, n := range(resp.Node.Nodes) {
		b.Update(&n, resp.Action)
	}

	// Begin the watch after this sync from the next sync
	b.watchIndex = resp.EtcdIndex + 1

	return nil
}

func (b *backends) Next() string {
	b.lock.RLock()
	defer b.lock.RUnlock()

	if len(b.hosts) == 0 {
		return ""
	}

	index := (b.lastIndex + 1) % len(b.hosts)
	b.lastIndex = index

	return b.hosts[index].addr
}
<|endoftext|>


<commit_before>package tcp

import (
	"crypto/tls"
	"net"
	"os"
	"testing"
	"time"

	"github.com/rqlite/rqlite/testdata/x509"
)

func Test_NewDialer(t *testing.T) {
	d := NewDialer(1, false, false)
	if d == nil {
		t.Fatal("failed to create a dialer")
	}
}

func Test_DialerNoConnect(t *testing.T) {
	d := NewDialer(87, false, false)
	_, err := d.Dial("127.0.0.1:0", 5*time.Second)
	if err == nil {
		t.Fatalf("no error connecting to bad address")
	}
}

func Test_DialerHeader(t *testing.T) {
	s := mustNewEchoServer()
	defer s.Close()
	go s.MustStart()

	d := NewDialer(64, false, false)
	conn, err := d.Dial(s.Addr(), 10*time.Second)
	if err != nil {
		t.Fatalf("failed to dial echo server: %s", err.Error())
	}

	buf := make([]byte, 1)
	_, err = conn.Read(buf)
	if err != nil {
		t.Fatalf("failed to read from echo server: %s", err.Error())
	}
	if exp, got := buf[0], byte(64); exp != got {
		t.Fatalf("got wrong response from echo server, exp %d, got %d", exp, got)
	}
}

func Test_DialerHeaderTLS(t *testing.T) {
	s, cert, key := mustNewEchoServerTLS()
	defer s.Close()
	defer os.Remove(cert)
	defer os.Remove(key)
	go s.MustStart()

	d := NewDialer(23, true, true)
	conn, err := d.Dial(s.Addr(), 10*time.Second)
	if err != nil {
		t.Fatalf("failed to dial TLS echo server: %s", err.Error())
	}

	buf := make([]byte, 1)
	_, err = conn.Read(buf)
	if err != nil {
		t.Fatalf("failed to read from TLS echo server: %s", err.Error())
	}
	if exp, got := buf[0], byte(23); exp != got {
		t.Fatalf("got wrong response from TLS echo server, exp %d, got %d", exp, got)
	}
}

type echoServer struct {
	ln net.Listener
}

// Addr returns the address of the echo server.
func (e *echoServer) Addr() string {
	return e.ln.Addr().String()
}

// MustStart starts the echo server.
func (e *echoServer) MustStart() {
	for {
		conn, err := e.ln.Accept()
		if err != nil {
			return
		}
		go func(c net.Conn) {
			buf := make([]byte, 1)
			_, err := c.Read(buf)
			if err != nil {
				panic("failed to read byte")
			}
			_, err = c.Write(buf)
			if err != nil {
				panic("failed to echo received byte")
			}
			c.Close()
		}(conn)
	}
}

// Close closes the echo server.
func (e *echoServer) Close() {
	e.ln.Close()
}

func mustNewEchoServer() *echoServer {
	return &echoServer{
		ln: mustTCPListener("127.0.0.1:0"),
	}
}

func mustNewEchoServerTLS() (*echoServer, string, string) {
	ln := mustTCPListener("127.0.0.1:0")
	cert := x509.CertFile("")
	key := x509.KeyFile("")

	tlsConfig, err := createTLSConfig(cert, key, "")
	if err != nil {
		panic("failed to create TLS config")
	}

	return &echoServer{
		ln: tls.NewListener(ln, tlsConfig),
	}, cert, key
}
<commit_msg>Clearer Dialer testing<commit_after>package tcp

import (
	"crypto/tls"
	"errors"
	"net"
	"os"
	"testing"
	"time"

	"github.com/rqlite/rqlite/testdata/x509"
)

func Test_NewDialer(t *testing.T) {
	d := NewDialer(1, false, false)
	if d == nil {
		t.Fatal("failed to create a dialer")
	}
}

func Test_DialerNoConnect(t *testing.T) {
	d := NewDialer(87, false, false)
	_, err := d.Dial("127.0.0.1:0", 5*time.Second)
	if err == nil {
		t.Fatalf("no error connecting to bad address")
	}
}

func Test_DialerHeader(t *testing.T) {
	s := mustNewEchoServer()
	defer s.Close()
	go s.Start(t)

	d := NewDialer(64, false, false)
	conn, err := d.Dial(s.Addr(), 10*time.Second)
	if err != nil {
		t.Fatalf("failed to dial echo server: %s", err.Error())
	}

	buf := make([]byte, 1)

	conn.SetReadDeadline(time.Now().Add(5 * time.Second))
	_, err = conn.Read(buf)
	if err != nil {
		t.Fatalf("failed to read from echo server: %s", err.Error())
	}
	if exp, got := buf[0], byte(64); exp != got {
		t.Fatalf("got wrong response from echo server, exp %d, got %d", exp, got)
	}
}

func Test_DialerHeaderTLS(t *testing.T) {
	s, cert, key := mustNewEchoServerTLS()
	defer s.Close()
	defer os.Remove(cert)
	defer os.Remove(key)
	go s.Start(t)

	d := NewDialer(23, true, true)
	conn, err := d.Dial(s.Addr(), 5*time.Second)
	if err != nil {
		t.Fatalf("failed to dial TLS echo server: %s", err.Error())
	}

	buf := make([]byte, 1)

	conn.SetReadDeadline(time.Now().Add(5 * time.Second))
	_, err = conn.Read(buf)
	if err != nil {
		t.Fatalf("failed to read from TLS echo server: %s", err.Error())
	}
	if exp, got := buf[0], byte(23); exp != got {
		t.Fatalf("got wrong response from TLS echo server, exp %d, got %d", exp, got)
	}
}

func Test_DialerHeaderTLSBadConnect(t *testing.T) {
	s, cert, key := mustNewEchoServerTLS()
	defer s.Close()
	defer os.Remove(cert)
	defer os.Remove(key)
	go s.Start(t)

	// Connect to a TLS server with a unencrypted client, to make sure
	// code can handle that misconfig.
	d := NewDialer(56, false, false)
	conn, err := d.Dial(s.Addr(), 5*time.Second)
	if err != nil {
		t.Fatalf("failed to dial TLS echo server: %s", err.Error())
	}

	buf := make([]byte, 1)

	conn.SetReadDeadline(time.Now().Add(2 * time.Second))
	_, err = conn.Read(buf)
	if err != nil && !errors.Is(err, os.ErrDeadlineExceeded) {
		t.Fatalf("unexpected error from TLS echo server: %s", err.Error())
	}
}

type echoServer struct {
	ln net.Listener
}

// Addr returns the address of the echo server.
func (e *echoServer) Addr() string {
	return e.ln.Addr().String()
}

// Start starts the echo server.
func (e *echoServer) Start(t *testing.T) {
	for {
		conn, err := e.ln.Accept()
		if err != nil {
			if errors.Is(err, net.ErrClosed) {
				// Successful testing can cause this.
				return
			}
			t.Fatalf("failed to accept a connection: %s", err.Error())
		}
		go func(c net.Conn) {
			buf := make([]byte, 1)
			_, err := c.Read(buf)
			if err != nil {
				return
			}
			_, err = c.Write(buf)
			if err != nil {
				t.Fatalf("failed to write byte: %s", err.Error())
			}
			c.Close()
		}(conn)
	}
}

// Close closes the echo server.
func (e *echoServer) Close() {
	e.ln.Close()
}

func mustNewEchoServer() *echoServer {
	return &echoServer{
		ln: mustTCPListener("127.0.0.1:0"),
	}
}

func mustNewEchoServerTLS() (*echoServer, string, string) {
	ln := mustTCPListener("127.0.0.1:0")
	cert := x509.CertFile("")
	key := x509.KeyFile("")

	tlsConfig, err := createTLSConfig(cert, key, "")
	if err != nil {
		panic("failed to create TLS config")
	}

	return &echoServer{
		ln: tls.NewListener(ln, tlsConfig),
	}, cert, key
}
<|endoftext|>


<commit_before>package tcp

import (
	"errors"
	"time"

	"github.com/hsheth2/logs"
)

func (c *TCB) packetSender() {
	// TODO: deal with data in urgSend buffers
	c.sendBufferUpdate.L.Lock()
	defer c.sendBufferUpdate.L.Lock()

	for {
		if len(c.sendBuffer) > 0 {
			sz := uint16(min(uint64(len(c.sendBuffer)), uint64(c.maxSegSize)))
			data := c.sendBuffer[:sz]
			c.sendBuffer = c.sendBuffer[sz:]
			c.sendData(data)
			continue
		}
		c.sendFinished.Broadcast(true)
		c.sendBufferUpdate.Wait()
	}
}

func (c *TCB) sendData(data []byte) (err error) {
	logs.Info.Println("Sending Data:", data)
	psh_packet := &TCP_Packet{
		header: &TCP_Header{
			seq:     c.seqNum,
			ack:     c.ackNum,
			flags:   TCP_PSH | TCP_ACK, // TODO: fix the flags
			urg:     0,
			options: []byte{},
		},
		payload: data,
	}
	c.seqNum += uint32(len(data))
	err = c.sendWithRetransmit(psh_packet)
	if err != nil {
		logs.Error.Println(err)
	}
	return err
}

func (c *TCB) sendWithRetransmit(data *TCP_Packet) error {
	// send the first packet
	c.sendPacket(data)

	// ack listeners
	ackFound := make(chan bool, 1)
	killAckListen := make(chan bool, 1)
	go c.listenForAck(ackFound, killAckListen, data.header.seq+data.getPayloadSize())

	// timers and timeouts
	resendTimerChan := make(chan bool, TCP_RESEND_LIMIT)
	timeout := make(chan bool, 1)
	killTimer := make(chan bool, 1)
	go resendTimer(resendTimerChan, timeout, killTimer, c.resendDelay)

	// resend if needed
	for {
		select {
		case <-ackFound:
			killTimer <- true
			return nil
		case <-resendTimerChan:
			c.sendPacket(data)
		case <-timeout:
			// TODO deal with a resend timeout fully
			killAckListen <- true
			logs.Error.Println("Resend of packet seq", data.header.seq, "timed out")
			return errors.New("Resend timed out")
		}
	}
}

func (c *TCB) listenForAck(successOut chan<- bool, end <-chan bool, targetAck uint32) {
	logs.Trace.Println("Listening for ack:", targetAck)
	in := c.recentAckUpdate.Register(ACK_BUF_SZ)
	defer c.recentAckUpdate.Unregister(in)
	for {
		select {
		case v := <-in:
			logs.Trace.Println("Ack listener got ack: ", v.(uint32))
			if v.(uint32) == targetAck {
				logs.Trace.Println("Killing the resender for ack:", v.(uint32))
				successOut <- true
				return
			}
		case <-end:
			return
		}
	}
}

func resendTimer(timerOutput, timeout chan<- bool, finished <-chan bool, delay time.Duration) {
	for i := 0; i < TCP_RESEND_LIMIT; i++ {
		select {
		case <-time.After(delay):
			timerOutput <- true
			delay *= 2 // increase the delay after each resend
		case <-finished:
			return
		}
	}
	timeout <- true
}

func (c *TCB) sendPacket(d *TCP_Packet) error {
	// Requires that seq, ack, flags, urg, and options are set
	// Will set everything else

	d.header.srcport = c.lport
	d.header.dstport = c.rport
	d.header.window = c.curWindow // TODO improve the window field calculation
	d.rip = c.ipAddress
	d.lip = c.srcIP

	pay, err := d.Marshal_TCP_Packet()
	if err != nil {
		logs.Error.Println(err)
		return err
	}

	err = c.writer.WriteTo(pay)

	if err != nil {
		logs.Error.Println(err)
		return err
	}

	return nil
}

func (c *TCB) sendResetFlag(seq, ack uint32, flag uint8) error {
	logs.Trace.Println("Sending RST with seq: ", seq, " and ack: ", ack)
	rst := &TCP_Packet{
		header: &TCP_Header{
			seq:     seq,
			ack:     ack,
			flags:   flag,
			urg:     0,
			options: []byte{},
		},
		payload: []byte{},
	}

	return c.sendPacket(rst)
}

func (c *TCB) sendReset(seq, ack uint32) error {
	return c.sendResetFlag(seq, ack, TCP_RST)
}

func (c *TCB) sendAck(seq, ack uint32) error {
	//logs.Trace.Println("Sending ACK with seq: ", seq, " and ack: ", ack)
	ack_packet := &TCP_Packet{
		header: &TCP_Header{
			seq:     seq,
			ack:     ack,
			flags:   TCP_ACK,
			urg:     0,
			options: []byte{},
		},
		payload: []byte{},
	}
	return c.sendPacket(ack_packet)
}

func (c *TCB) sendFin(seq, ack uint32) error {
	logs.Trace.Println("Sending FIN with seq: ", seq, " and ack: ", ack)
	fin_packet := &TCP_Packet{
		header: &TCP_Header{
			seq:     seq,
			ack:     ack,
			flags:   TCP_ACK | TCP_FIN,
			urg:     0,
			options: []byte{},
		},
		payload: []byte{},
	}
	return c.sendPacket(fin_packet)
}
<commit_msg>Fixed a problem where the registering a listener for an ACK wouldn't happen in time for the ACK reveive<commit_after>package tcp

import (
	"errors"
	"time"

	"github.com/hsheth2/logs"
)

func (c *TCB) packetSender() {
	// TODO: deal with data in urgSend buffers
	c.sendBufferUpdate.L.Lock()
	defer c.sendBufferUpdate.L.Lock()

	for {
		if len(c.sendBuffer) > 0 {
			sz := uint16(min(uint64(len(c.sendBuffer)), uint64(c.maxSegSize)))
			data := c.sendBuffer[:sz]
			c.sendBuffer = c.sendBuffer[sz:]
			c.sendData(data)
			continue
		}
		c.sendFinished.Broadcast(true)
		c.sendBufferUpdate.Wait()
	}
}

func (c *TCB) sendData(data []byte) (err error) {
	logs.Info.Println("Sending Data:", data)
	psh_packet := &TCP_Packet{
		header: &TCP_Header{
			seq:     c.seqNum,
			ack:     c.ackNum,
			flags:   TCP_PSH | TCP_ACK, // TODO: fix the flags
			urg:     0,
			options: []byte{},
		},
		payload: data,
	}
	c.seqNum += uint32(len(data))
	err = c.sendWithRetransmit(psh_packet)
	if err != nil {
		logs.Error.Println(err)
	}
	return err
}

func (c *TCB) sendWithRetransmit(data *TCP_Packet) error {
	// send the first packet
	c.sendPacket(data)

	// ack listeners
	ackFound := make(chan bool, 1)
	killAckListen := make(chan bool, 1)
	go c.listenForAck(ackFound, killAckListen, data.header.seq+data.getPayloadSize())

	// timers and timeouts
	resendTimerChan := make(chan bool, TCP_RESEND_LIMIT)
	timeout := make(chan bool, 1)
	killTimer := make(chan bool, 1)
	resendTimer(resendTimerChan, timeout, killTimer, c.resendDelay)

	// resend if needed
	for {
		select {
		case <-ackFound:
			killTimer <- true
			return nil
		case <-resendTimerChan:
			c.sendPacket(data)
		case <-timeout:
			// TODO deal with a resend timeout fully
			killAckListen <- true
			logs.Error.Println("Resend of packet seq", data.header.seq, "timed out")
			return errors.New("Resend timed out")
		}
	}
}

func (c *TCB) listenForAck(successOut chan<- bool, end <-chan bool, targetAck uint32) {
	logs.Trace.Println("Listening for ack:", targetAck)
	in := c.recentAckUpdate.Register(ACK_BUF_SZ)
	go func(in chan interface{}, successOut chan<- bool, end <-chan bool, targetAck uint32) {
		defer c.recentAckUpdate.Unregister(in)
		for {
			select {
			case v := <-in:
				logs.Trace.Println("Ack listener got ack: ", v.(uint32))
				if v.(uint32) == targetAck {
					logs.Trace.Println("Killing the resender for ack:", v.(uint32))
					successOut <- true
					return
				}
			case <-end:
				return
			}
		}
	}(in, successOut, end, targetAck)
}

func resendTimer(timerOutput, timeout chan<- bool, finished <-chan bool, delay time.Duration) {
	for i := 0; i < TCP_RESEND_LIMIT; i++ {
		select {
		case <-time.After(delay):
			timerOutput <- true
			delay *= 2 // increase the delay after each resend
		case <-finished:
			return
		}
	}
	timeout <- true
}

func (c *TCB) sendPacket(d *TCP_Packet) error {
	// Requires that seq, ack, flags, urg, and options are set
	// Will set everything else

	d.header.srcport = c.lport
	d.header.dstport = c.rport
	d.header.window = c.curWindow // TODO improve the window field calculation
	d.rip = c.ipAddress
	d.lip = c.srcIP

	pay, err := d.Marshal_TCP_Packet()
	if err != nil {
		logs.Error.Println(err)
		return err
	}

	err = c.writer.WriteTo(pay)

	if err != nil {
		logs.Error.Println(err)
		return err
	}

	return nil
}

func (c *TCB) sendResetFlag(seq, ack uint32, flag uint8) error {
	logs.Trace.Println("Sending RST with seq: ", seq, " and ack: ", ack)
	rst := &TCP_Packet{
		header: &TCP_Header{
			seq:     seq,
			ack:     ack,
			flags:   flag,
			urg:     0,
			options: []byte{},
		},
		payload: []byte{},
	}

	return c.sendPacket(rst)
}

func (c *TCB) sendReset(seq, ack uint32) error {
	return c.sendResetFlag(seq, ack, TCP_RST)
}

func (c *TCB) sendAck(seq, ack uint32) error {
	//logs.Trace.Println("Sending ACK with seq: ", seq, " and ack: ", ack)
	ack_packet := &TCP_Packet{
		header: &TCP_Header{
			seq:     seq,
			ack:     ack,
			flags:   TCP_ACK,
			urg:     0,
			options: []byte{},
		},
		payload: []byte{},
	}
	return c.sendPacket(ack_packet)
}

func (c *TCB) sendFin(seq, ack uint32) error {
	logs.Trace.Println("Sending FIN with seq: ", seq, " and ack: ", ack)
	fin_packet := &TCP_Packet{
		header: &TCP_Header{
			seq:     seq,
			ack:     ack,
			flags:   TCP_ACK | TCP_FIN,
			urg:     0,
			options: []byte{},
		},
		payload: []byte{},
	}
	return c.sendPacket(fin_packet)
}
<|endoftext|>


<commit_before>package hashstructure

import (
	"fmt"
)

func ExampleHash() {
	type ComplexStruct struct {
		Name     string
		Age      uint
		Metadata map[string]interface{}
	}

	v := ComplexStruct{
		Name: "mitchellh",
		Age:  64,
		Metadata: map[string]interface{}{
			"car":      true,
			"location": "California",
			"siblings": []string{"Bob", "John"},
		},
	}

	hash, err := Hash(v, nil)
	if err != nil {
		panic(err)
	}

	fmt.Printf("%d", hash)
	// Output:
	// 8669281753891521642
}
<commit_msg>update test<commit_after>package hashstructure

import (
	"fmt"
)

func ExampleHash() {
	type ComplexStruct struct {
		Name     string
		Age      uint
		Metadata map[string]interface{}
	}

	v := ComplexStruct{
		Name: "mitchellh",
		Age:  64,
		Metadata: map[string]interface{}{
			"car":      true,
			"location": "California",
			"siblings": []string{"Bob", "John"},
		},
	}

	hash, err := Hash(v, nil)
	if err != nil {
		panic(err)
	}

	fmt.Printf("%d", hash)
	// Output:
	// 6691276962590150517
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"runtime"
	"sync"
	"sync/atomic"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/getlantern/golog"
	"github.com/oxtoacart/tdb"
)

var (
	log = golog.LoggerFor("tdbdemo")
)

func main() {
	epoch := time.Date(2015, time.January, 1, 0, 0, 0, 0, time.UTC)

	tmpDir, err := ioutil.TempDir("", "tdbtest")
	if err != nil {
		log.Fatal(err)
	}
	defer os.RemoveAll(tmpDir)

	log.Debugf("Writing data to %v", tmpDir)

	numReporters := 5000
	uniquesPerReporter := 500
	uniquesPerPeriod := 20
	reportingPeriods := 1000
	reportingInterval := time.Millisecond
	resolution := reportingInterval * 5
	hotPeriod := resolution * 10
	retentionPeriod := time.Duration(reportingPeriods) * reportingInterval * 2
	numWriters := 4
	db := tdb.NewDB(&tdb.DBOpts{
		Dir:       tmpDir,
		BatchSize: 1000,
	})
	err = db.CreateTable("test", resolution, hotPeriod, retentionPeriod)
	if err != nil {
		log.Fatal(err)
	}

	inserts := int64(0)
	start := time.Now()

	report := func() {
		stats := db.TableStats("test")
		delta := time.Now().Sub(start)
		start = time.Now()
		i := atomic.SwapInt64(&inserts, 0)
		var ms runtime.MemStats
		runtime.ReadMemStats(&ms)
		preGC := float64(ms.HeapAlloc) / 1024.0 / 1024.0
		runtime.GC()
		runtime.ReadMemStats(&ms)
		postGC := float64(ms.HeapAlloc) / 1024.0 / 1024.0
		fmt.Printf(`
%s inserts at %s inserts per second
Hot Keys: %s     Archived Buckets: %s
HeapAlloc pre/post GC %f/%f MiB
`,
			humanize.Comma(i), humanize.Comma(i/int64(delta.Seconds())),
			humanize.Comma(stats.HotKeys), humanize.Comma(stats.ArchivedBuckets),
			preGC, postGC)
	}

	go func() {
		for {
			tk := time.NewTicker(30 * time.Second)
			for range tk.C {
				report()
			}
		}
	}()

	go func() {
		for {
			tk := time.NewTicker(10 * time.Second)
			for range tk.C {
				count := 0
				now := db.Now("test")
				q := &tdb.Query{
					Table:  "test",
					Fields: []string{"i"},
					From:   now.Add(-2 * hotPeriod),
					OnValues: func(key map[string]interface{}, field string, vals []float64) {
						count++
					},
				}
				start := time.Now()
				err := db.RunQuery(q)
				delta := time.Now().Sub(start)
				if err != nil {
					log.Errorf("Unable to run query: %v", err)
					continue
				}
				fmt.Printf("\nQuery at %v returned %v in %v\n", now, humanize.Comma(int64(count)), delta)
			}
		}
	}()

	var wg sync.WaitGroup
	wg.Add(numWriters)
	for _w := 0; _w < numWriters; _w++ {
		go func() {
			defer wg.Done()
			for i := 0; i < reportingPeriods; i++ {
				ts := epoch.Add(time.Duration(i) * reportingInterval)
				for r := 0; r < numReporters/numWriters; r++ {
					for u := 0; u < uniquesPerPeriod; u++ {
						p := &tdb.Point{
							Ts: ts,
							Dims: map[string]interface{}{
								"r": rand.Intn(numReporters),
								"u": rand.Intn(uniquesPerReporter),
								"b": rand.Float64() > 0.99,
							},
							Vals: map[string]float64{
								"i":  float64(rand.Intn(100000)),
								"ii": float64(rand.Intn(100)),
							},
						}
						ierr := db.Insert("test", p)
						if ierr != nil {
							log.Errorf("Unable to insert: %v", err)
							return
						}
						atomic.AddInt64(&inserts, 1)
					}
				}
				fmt.Print(".")
			}
		}()
	}

	wg.Wait()
	report()
}
<commit_msg>Added calculated field to tdbdemo<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"runtime"
	"sync"
	"sync/atomic"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/getlantern/golog"
	"github.com/oxtoacart/tdb"
	. "github.com/oxtoacart/tdb/expr"
)

var (
	log = golog.LoggerFor("tdbdemo")
)

func main() {
	epoch := time.Date(2015, time.January, 1, 0, 0, 0, 0, time.UTC)

	tmpDir, err := ioutil.TempDir("", "tdbtest")
	if err != nil {
		log.Fatal(err)
	}
	defer os.RemoveAll(tmpDir)

	log.Debugf("Writing data to %v", tmpDir)

	numReporters := 5000
	uniquesPerReporter := 500
	uniquesPerPeriod := 20
	reportingPeriods := 1000
	reportingInterval := time.Millisecond
	resolution := reportingInterval * 5
	hotPeriod := resolution * 10
	retentionPeriod := time.Duration(reportingPeriods) * reportingInterval * 2
	numWriters := 4
	db := tdb.NewDB(&tdb.DBOpts{
		Dir:       tmpDir,
		BatchSize: 1000,
	})
	err = db.CreateTable("test", resolution, hotPeriod, retentionPeriod, tdb.DerivedField{
		Name: "iii",
		Expr: Avg(Calc("ii / i")),
	})
	if err != nil {
		log.Fatal(err)
	}

	inserts := int64(0)
	start := time.Now()

	report := func() {
		stats := db.TableStats("test")
		delta := time.Now().Sub(start)
		start = time.Now()
		i := atomic.SwapInt64(&inserts, 0)
		var ms runtime.MemStats
		runtime.ReadMemStats(&ms)
		preGC := float64(ms.HeapAlloc) / 1024.0 / 1024.0
		runtime.GC()
		runtime.ReadMemStats(&ms)
		postGC := float64(ms.HeapAlloc) / 1024.0 / 1024.0
		fmt.Printf(`
%s inserts at %s inserts per second
Hot Keys: %s     Archived Buckets: %s
HeapAlloc pre/post GC %f/%f MiB
`,
			humanize.Comma(i), humanize.Comma(i/int64(delta.Seconds())),
			humanize.Comma(stats.HotKeys), humanize.Comma(stats.ArchivedBuckets),
			preGC, postGC)
	}

	go func() {
		for {
			tk := time.NewTicker(30 * time.Second)
			for range tk.C {
				report()
			}
		}
	}()

	go func() {
		for {
			tk := time.NewTicker(10 * time.Second)
			for range tk.C {
				count := 0
				now := db.Now("test")
				q := &tdb.Query{
					Table:  "test",
					Fields: []string{"i"},
					From:   now.Add(-2 * hotPeriod),
					OnValues: func(key map[string]interface{}, field string, vals []float64) {
						count++
					},
				}
				start := time.Now()
				err := db.RunQuery(q)
				delta := time.Now().Sub(start)
				if err != nil {
					log.Errorf("Unable to run query: %v", err)
					continue
				}
				fmt.Printf("\nQuery at %v returned %v in %v\n", now, humanize.Comma(int64(count)), delta)
			}
		}
	}()

	var wg sync.WaitGroup
	wg.Add(numWriters)
	for _w := 0; _w < numWriters; _w++ {
		go func() {
			defer wg.Done()
			for i := 0; i < reportingPeriods; i++ {
				ts := epoch.Add(time.Duration(i) * reportingInterval)
				for r := 0; r < numReporters/numWriters; r++ {
					for u := 0; u < uniquesPerPeriod; u++ {
						p := &tdb.Point{
							Ts: ts,
							Dims: map[string]interface{}{
								"r": rand.Intn(numReporters),
								"u": rand.Intn(uniquesPerReporter),
								"b": rand.Float64() > 0.99,
							},
							Vals: map[string]float64{
								"i":  float64(rand.Intn(100000)),
								"ii": float64(rand.Intn(100)),
							},
						}
						ierr := db.Insert("test", p)
						if ierr != nil {
							log.Errorf("Unable to insert: %v", err)
							return
						}
						atomic.AddInt64(&inserts, 1)
					}
				}
				fmt.Print(".")
			}
		}()
	}

	wg.Wait()
	report()
}
<|endoftext|>


<commit_before>package bagins_test

import (
	"github.com/APTrust/bagins"
	"github.com/APTrust/bagins/bagutil"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"testing"
)

func setupTestBag(bagName string) (*bagins.Bag, error) {
	algo := "sha1"
	hsh, _ := bagutil.LookupHashFunc(algo)
	cs := bagutil.NewChecksumAlgorithm(algo, hsh)

	bag, err := bagins.NewBag(os.TempDir(), bagName, cs)
	if err != nil {
		return nil, err
	}
	return bag, nil
}

func TestNewBag(t *testing.T) {

	// Use this ChecksumAlgorithm for the tests.
	algo := "sha1"
	hsh, _ := bagutil.LookupHashFunc(algo)
	cs := bagutil.NewChecksumAlgorithm(algo, hsh)

	// It should raise an error if the destination dir does not exist.
	badLocation := filepath.Join(os.TempDir(), "/GOTESTNOT_EXISTs/")
	_, err := bagins.NewBag(badLocation, "_GOFAILBAG_", cs)
	if err == nil {
		t.Error("NewBag function does not recognize when a directory does not exist!")
	}

	// It should raise an error if the bag already exists.
	os.MkdirAll(filepath.Join(badLocation, "_GOFAILBAG_"), 0766)
	defer os.RemoveAll(badLocation)

	_, err = bagins.NewBag(badLocation, "_GOFAILBAG_", cs)
	if err == nil {
		t.Error("Error not thrown when bag already exists as expected.")
	}

	// It should create a bag without any errors.
	bag, err := setupTestBag("_GOTEST_NEWBAG_")
	defer os.RemoveAll(bag.Path())

	// It should find all of the following files and directories.
	if _, err = os.Stat(filepath.Join(os.TempDir(), "_GOTESTBAG_")); os.IsNotExist(err) {
		t.Error("Bag directory does not exist!")
	}
	if data, err := os.Stat(filepath.Join(os.TempDir(), "_GOTESTBAG_", "data")); os.IsNotExist(err) || !data.IsDir() {
		t.Error("Data directory does not exist or is not a directory!")
	}
	if _, err = os.Stat(filepath.Join(bag.Path(), "bagit.txt")); os.IsNotExist(err) {
		bi, err := bag.BagInfo()
		if err != nil {
			t.Error(err)
		}
		t.Errorf("bagit.txt does not exist! %s", bi.Name())
	}
	if _, err = os.Stat(filepath.Join(os.TempDir(), "_GOTESTBAG_", "manifest-sha1.txt")); os.IsNotExist(err) {
		t.Error("manifest-sha1.txt does not exist!")
	}
}

// It should place an appropriate file in the data directory and add the fixity to the manifest.
func TestAddFile(t *testing.T) {
	// Setup the test file to add for the test.
	fi, _ := ioutil.TempFile("", "TEST_GO_ADDFILE_")
	fi.WriteString("Test the checksum")
	fi.Close()
	defer os.Remove(fi.Name())

	// Setup the Test Bag
	bag, _ := setupTestBag("_GOTEST_ADDFILE_")
	defer os.RemoveAll(bag.Path())

	// It should return an error when trying to add a file that doesn't exist.
	if err := bag.AddFile("idontexist.txt", "idontexist.txt"); err == nil {
		t.Errorf("Adding a nonexistant file did not generate an error!")
	}

	// It should and a file to the data directory and generate a fixity value.
	expFile := "my/nested/dir/mytestfile.txt"
	if err := bag.AddFile(fi.Name(), expFile); err != nil {
		t.Error(err)
	}

	// It should have created the file in the payload directory.
	_, err := os.Stat(filepath.Join(bag.Path(), "data", expFile))
	if err != nil {
		t.Error("Testing if payload file created:", err)
	}

	// It should have calulated the fixity and put it in the manifest.
	mf, _ := bag.Manifest()
	expKey := filepath.Join("data", expFile)
	fx, ok := mf.Data[expKey]
	if !ok {
		t.Error("Unable to find entry in manfest: ", expKey)
	}
	if len(fx) != 40 {
		t.Errorf("Expected %d character fixity but returned: %d", 32, len(fx))
	}
}

func TestAddDir(t *testing.T) {

	// Setup source files to test
	srcDir, _ := ioutil.TempDir("", "_GOTEST_PAYLOAD_SRC_")
	for i := 0; i < 50; i++ {
		fi, _ := ioutil.TempFile(srcDir, "TEST_GO_ADDFILE_")
		fi.WriteString("Test the checksum")
		fi.Close()
	}
	defer os.RemoveAll(srcDir)

	// Setup the test bag
	bag, _ := setupTestBag("_GOTEST_ADDDIR_")
	defer os.RemoveAll(bag.Path())

	// It should produce no errors
	if errs := bag.AddDir(srcDir); len(errs) != 0 {
		t.Error(errs)
	}

	// It should produce 50 manifest entries

	// It should generate entries in the manifest
	mf, _ := bag.Manifest()

	// It should produce 50 manifest entries
	if len(mf.Data) != 50 {
		t.Error("Expected 50 manifest entries but returned", len(mf.Data))
	}
	// It should contain the proper checksums for each file.
	for key, fx := range mf.Data {
		expFx := "da909ba395016f2a64b04d706520db6afa74fc95"
		expPfx := filepath.Join("data", "TEST_GO_ADDFILE_")

		if fx != expFx {
			t.Error("Fixity error!", fx, "does not match expected", expFx)
		}
		if !strings.HasPrefix(key, expPfx) {
			t.Error(key, "does not start with", expPfx)
		}
	}
}

func TestManifest(t *testing.T) {

	// Setup the test bag
	bag, _ := setupTestBag("_GOTEST_MANIFEST_")
	defer os.RemoveAll(bag.Path())

	// It should have the expected name and return no error.
	mf, err := bag.Manifest()
	if err != nil {
		t.Error(err)
	}
	exp := "manifest-sha1.txt"
	if filepath.Base(mf.Name()) != exp {
		t.Error("Expected manifest name", exp, "but returned", filepath.Base(mf.Name()))
	}
}

func TestAddTagFile(t *testing.T) {

	// Setup the test bag
	bag, _ := setupTestBag("_GOTEST_ADDTAGFILE_")
	defer os.RemoveAll(bag.Path())

	// It should throw an error when a bag tagfilename is passed.
	badTagName := "customtag/directory/tag"
	if err := bag.AddTagfile(badTagName); err == nil {
		t.Error("Did not generate an error when trying to add bag tagname:", badTagName)
	}

	// It should not throw an error.
	newTagName := "customtag/directory/tag.txt"
	if err := bag.AddTagfile(newTagName); err != nil {
		t.Error(err)
	}

	// It should be able to lookup the tagfile by name.
	if _, err := bag.TagFile(newTagName); err != nil {
		t.Error(err)
	}

	// Even tagfiles passed as root should be put under the bag.
	oddTagName := "/lookslikeroot/directory/tag.txt"
	if err := bag.AddTagfile(oddTagName); err != nil {
		t.Error(err)
	}

	// It should be able to lookup the tagfile by name.
	if _, err := bag.TagFile(oddTagName); err != nil {
		t.Error(err)
	}

	// It should find the file inside the bag.
	if _, err := os.Stat(filepath.Join(bag.Path(), newTagName)); err != nil {
		t.Error(err)
	}
}

func TestTagFile(t *testing.T) {

	// Setup the test bag
	bag, _ := setupTestBag("_GOTEST_TAGFILE_")
	defer os.RemoveAll(bag.Path())

	// It should find the tag file by name
	testTagName := "new/tag.txt"
	bag.AddTagfile(testTagName)
	if _, err := bag.TagFile(testTagName); err != nil {
		t.Error(err)
	}

	// It should return an error if asking for a bad tag name.
	badTagName := "/new/tag.txt"
	if _, err := bag.TagFile(badTagName); err == nil {
		t.Error("Bag.TagFile returned results for", badTagName, "when it should not exist.")
	}
}
<commit_msg>Added Bag.Path test.<commit_after>package bagins_test

import (
	"github.com/APTrust/bagins"
	"github.com/APTrust/bagins/bagutil"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"testing"
)

func setupTestBag(bagName string) (*bagins.Bag, error) {
	algo := "sha1"
	hsh, _ := bagutil.LookupHashFunc(algo)
	cs := bagutil.NewChecksumAlgorithm(algo, hsh)

	bag, err := bagins.NewBag(os.TempDir(), bagName, cs)
	if err != nil {
		return nil, err
	}
	return bag, nil
}

func TestNewBag(t *testing.T) {

	// Use this ChecksumAlgorithm for the tests.
	algo := "sha1"
	hsh, _ := bagutil.LookupHashFunc(algo)
	cs := bagutil.NewChecksumAlgorithm(algo, hsh)

	// It should raise an error if the destination dir does not exist.
	badLocation := filepath.Join(os.TempDir(), "/GOTESTNOT_EXISTs/")
	_, err := bagins.NewBag(badLocation, "_GOFAILBAG_", cs)
	if err == nil {
		t.Error("NewBag function does not recognize when a directory does not exist!")
	}

	// It should raise an error if the bag already exists.
	os.MkdirAll(filepath.Join(badLocation, "_GOFAILBAG_"), 0766)
	defer os.RemoveAll(badLocation)

	_, err = bagins.NewBag(badLocation, "_GOFAILBAG_", cs)
	if err == nil {
		t.Error("Error not thrown when bag already exists as expected.")
	}

	// It should create a bag without any errors.
	bag, err := setupTestBag("_GOTEST_NEWBAG_")
	defer os.RemoveAll(bag.Path())

	// It should find all of the following files and directories.
	if _, err = os.Stat(filepath.Join(os.TempDir(), "_GOTESTBAG_")); os.IsNotExist(err) {
		t.Error("Bag directory does not exist!")
	}
	if data, err := os.Stat(filepath.Join(os.TempDir(), "_GOTESTBAG_", "data")); os.IsNotExist(err) || !data.IsDir() {
		t.Error("Data directory does not exist or is not a directory!")
	}
	if _, err = os.Stat(filepath.Join(bag.Path(), "bagit.txt")); os.IsNotExist(err) {
		bi, err := bag.BagInfo()
		if err != nil {
			t.Error(err)
		}
		t.Errorf("bagit.txt does not exist! %s", bi.Name())
	}
	if _, err = os.Stat(filepath.Join(os.TempDir(), "_GOTESTBAG_", "manifest-sha1.txt")); os.IsNotExist(err) {
		t.Error("manifest-sha1.txt does not exist!")
	}
}

// It should place an appropriate file in the data directory and add the fixity to the manifest.
func TestAddFile(t *testing.T) {
	// Setup the test file to add for the test.
	fi, _ := ioutil.TempFile("", "TEST_GO_ADDFILE_")
	fi.WriteString("Test the checksum")
	fi.Close()
	defer os.Remove(fi.Name())

	// Setup the Test Bag
	bag, _ := setupTestBag("_GOTEST_BAG_ADDFILE_")
	defer os.RemoveAll(bag.Path())

	// It should return an error when trying to add a file that doesn't exist.
	if err := bag.AddFile("idontexist.txt", "idontexist.txt"); err == nil {
		t.Errorf("Adding a nonexistant file did not generate an error!")
	}

	// It should and a file to the data directory and generate a fixity value.
	expFile := "my/nested/dir/mytestfile.txt"
	if err := bag.AddFile(fi.Name(), expFile); err != nil {
		t.Error(err)
	}

	// It should have created the file in the payload directory.
	_, err := os.Stat(filepath.Join(bag.Path(), "data", expFile))
	if err != nil {
		t.Error("Testing if payload file created:", err)
	}

	// It should have calulated the fixity and put it in the manifest.
	mf, _ := bag.Manifest()
	expKey := filepath.Join("data", expFile)
	fx, ok := mf.Data[expKey]
	if !ok {
		t.Error("Unable to find entry in manfest: ", expKey)
	}
	if len(fx) != 40 {
		t.Errorf("Expected %d character fixity but returned: %d", 32, len(fx))
	}
}

func TestAddDir(t *testing.T) {

	// Setup source files to test
	srcDir, _ := ioutil.TempDir("", "_GOTEST_PAYLOAD_SRC_")
	for i := 0; i < 50; i++ {
		fi, _ := ioutil.TempFile(srcDir, "TEST_GO_ADDFILE_")
		fi.WriteString("Test the checksum")
		fi.Close()
	}
	defer os.RemoveAll(srcDir)

	// Setup the test bag
	bag, _ := setupTestBag("_GOTEST_BAG_ADDDIR_")
	defer os.RemoveAll(bag.Path())

	// It should produce no errors
	if errs := bag.AddDir(srcDir); len(errs) != 0 {
		t.Error(errs)
	}

	// It should produce 50 manifest entries

	// It should generate entries in the manifest
	mf, _ := bag.Manifest()

	// It should produce 50 manifest entries
	if len(mf.Data) != 50 {
		t.Error("Expected 50 manifest entries but returned", len(mf.Data))
	}
	// It should contain the proper checksums for each file.
	for key, fx := range mf.Data {
		expFx := "da909ba395016f2a64b04d706520db6afa74fc95"
		expPfx := filepath.Join("data", "TEST_GO_ADDFILE_")

		if fx != expFx {
			t.Error("Fixity error!", fx, "does not match expected", expFx)
		}
		if !strings.HasPrefix(key, expPfx) {
			t.Error(key, "does not start with", expPfx)
		}
	}
}

func TestManifest(t *testing.T) {

	// Setup the test bag
	bag, _ := setupTestBag("_GOTEST_BAG_MANIFEST_")
	defer os.RemoveAll(bag.Path())

	// It should have the expected name and return no error.
	mf, err := bag.Manifest()
	if err != nil {
		t.Error(err)
	}
	exp := "manifest-sha1.txt"
	if filepath.Base(mf.Name()) != exp {
		t.Error("Expected manifest name", exp, "but returned", filepath.Base(mf.Name()))
	}
}

func TestAddTagFile(t *testing.T) {

	// Setup the test bag
	bag, _ := setupTestBag("_GOTEST_BAG_ADDTAGFILE_")
	defer os.RemoveAll(bag.Path())

	// It should throw an error when a bag tagfilename is passed.
	badTagName := "customtag/directory/tag"
	if err := bag.AddTagfile(badTagName); err == nil {
		t.Error("Did not generate an error when trying to add bag tagname:", badTagName)
	}

	// It should not throw an error.
	newTagName := "customtag/directory/tag.txt"
	if err := bag.AddTagfile(newTagName); err != nil {
		t.Error(err)
	}

	// It should be able to lookup the tagfile by name.
	if _, err := bag.TagFile(newTagName); err != nil {
		t.Error(err)
	}

	// Even tagfiles passed as root should be put under the bag.
	oddTagName := "/lookslikeroot/directory/tag.txt"
	if err := bag.AddTagfile(oddTagName); err != nil {
		t.Error(err)
	}

	// It should be able to lookup the tagfile by name.
	if _, err := bag.TagFile(oddTagName); err != nil {
		t.Error(err)
	}

	// It should find the file inside the bag.
	if _, err := os.Stat(filepath.Join(bag.Path(), newTagName)); err != nil {
		t.Error(err)
	}
}

func TestTagFile(t *testing.T) {

	// Setup the test bag
	bag, _ := setupTestBag("_GOTEST_BAG_TAGFILE_")
	defer os.RemoveAll(bag.Path())

	// It should find the tag file by name
	testTagName := "new/tag.txt"
	bag.AddTagfile(testTagName)
	if _, err := bag.TagFile(testTagName); err != nil {
		t.Error(err)
	}

	// It should return an error if asking for a bad tag name.
	badTagName := "/new/tag.txt"
	if _, err := bag.TagFile(badTagName); err == nil {
		t.Error("Bag.TagFile returned results for", badTagName, "when it should not exist.")
	}
}

func TestPath(t *testing.T) {
	// Stup the test bag
	bagName := "_GOTEST_BAG_PATH_"
	bag, _ := setupTestBag(bagName)
	defer os.RemoveAll(bag.Path())

	expPath := filepath.Join(os.TempDir(), bagName)
	if bag.Path() != expPath {
		t.Error("Excpected", bag.Path(), "and", expPath, "to be equal!")
	}
}
<|endoftext|>