<commit_before>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Andrew Bonventre (andybons@gmail.com)
// Author: Spencer Kimball (spencer.kimball@gmail.com)
// Author: Jiang-Ming Yang (jiangming.yang@gmail.com)

package engine

import (
	"github.com/gogo/protobuf/proto"

	"github.com/cockroachdb/cockroach/pkg/roachpb"
	"github.com/cockroachdb/cockroach/pkg/storage/engine/enginepb"
	"github.com/cockroachdb/cockroach/pkg/util/protoutil"
)

// Iterator is an interface for iterating over key/value pairs in an
// engine. Iterator implementations are thread safe unless otherwise
// noted.
type Iterator interface {
	// Close frees up resources held by the iterator.
	Close()
	// Seek advances the iterator to the first key in the engine which
	// is >= the provided key.
	Seek(key MVCCKey)
	// SeekReverse advances the iterator to the first key in the engine which
	// is <= the provided key.
	SeekReverse(key MVCCKey)
	// Valid returns true if the iterator is currently valid. An iterator that
	// hasn't had Seek called on it or has gone past the end of the key range
	// is invalid.
	Valid() bool
	// Next advances the iterator to the next key/value in the
	// iteration. After this call, Valid() will be true if the
	// iterator was not positioned at the last key.
	Next()
	// Prev moves the iterator backward to the previous key/value
	// in the iteration. After this call, Valid() will be true if the
	// iterator was not positioned at the first key.
	Prev()
	// NextKey advances the iterator to the next MVCC key. This operation is
	// distinct from Next which advances to the next version of the current key
	// or the next key if the iterator is currently located at the last version
	// for a key.
	NextKey()
	// PrevKey moves the iterator backward to the previous MVCC key. This
	// operation is distinct from Prev which moves the iterator backward to the
	// prev version of the current key or the prev key if the iterator is
	// currently located at the first version for a key.
	PrevKey()
	// Key returns the current key.
	Key() MVCCKey
	// Value returns the current value as a byte slice.
	Value() []byte
	// ValueProto unmarshals the value the iterator is currently
	// pointing to using a protobuf decoder.
	ValueProto(msg proto.Message) error
	// unsafeKey returns the same value as Key, but the memory is invalidated on
	// the next call to {Next,Prev,Seek,SeekReverse,Close}.
	UnsafeKey() MVCCKey
	// unsafeKey returns the same value as Value, but the memory is invalidated
	// on the next call to {Next,Prev,Seek,SeekReverse,Close}.
	UnsafeValue() []byte
	// Less returns true if the key the iterator is currently positioned at is
	// less than the specified key.
	Less(key MVCCKey) bool
	// Error returns the error, if any, which the iterator encountered.
	Error() error
	// ComputeStats scans the underlying engine from start to end keys and
	// computes stats counters based on the values. This method is used after a
	// range is split to recompute stats for each subrange. The start key is
	// always adjusted to avoid counting local keys in the event stats are being
	// recomputed for the first range (i.e. the one with start key == KeyMin).
	// The nowNanos arg specifies the wall time in nanoseconds since the
	// epoch and is used to compute the total age of all intents.
	ComputeStats(start, end MVCCKey, nowNanos int64) (enginepb.MVCCStats, error)
}

// Reader is the read interface to an engine's data.
type Reader interface {
	// Close closes the reader, freeing up any outstanding resources. Note that
	// various implementations have slightly different behaviors. In particular,
	// Distinct() batches release their parent batch for future use while
	// Engines, Snapshots and Batches free the associated C++ resources.
	Close()
	// Closed returns true if the reader has been closed or is not usable.
	// Objects backed by this reader (e.g. Iterators) can check this to ensure
	// that they are not using a closed engine. Intended for use within package
	// engine; exported to enable wrappers to exist in other packages.
	Closed() bool
	// Get returns the value for the given key, nil otherwise.
	Get(key MVCCKey) ([]byte, error)
	// GetProto fetches the value at the specified key and unmarshals it
	// using a protobuf decoder. Returns true on success or false if the
	// key was not found. On success, returns the length in bytes of the
	// key and the value.
	GetProto(key MVCCKey, msg proto.Message) (ok bool, keyBytes, valBytes int64, err error)
	// Iterate scans from start to end keys, visiting at most max
	// key/value pairs. On each key value pair, the function f is
	// invoked. If f returns an error or if the scan itself encounters
	// an error, the iteration will stop and return the error.
	// If the first result of f is true, the iteration stops.
	Iterate(start, end MVCCKey, f func(MVCCKeyValue) (bool, error)) error
	// NewIterator returns a new instance of an Iterator over this engine. When
	// prefix is true, Seek will use the user-key prefix of the supplied MVCC key
	// to restrict which sstables are searched, but iteration (using Next) over
	// keys without the same user-key prefix will not work correctly (keys may be
	// skipped). The caller must invoke Iterator.Close() when finished with the
	// iterator to free resources.
	NewIterator(prefix bool) Iterator
}

// Writer is the write interface to an engine's data.
type Writer interface {
	// ApplyBatchRepr atomically applies a set of batched updates. Created by
	// calling Repr() on a batch. Using this method is equivalent to constructing
	// and committing a batch whose Repr() equals repr. If sync is true, the
	// batch is synchronously written to disk. It is an error to specify
	// sync=true if the Writer is a Batch.
	ApplyBatchRepr(repr []byte, sync bool) error
	// Clear removes the item from the db with the given key.
	// Note that clear actually removes entries from the storage
	// engine, rather than inserting tombstones.
	Clear(key MVCCKey) error
	// ClearRange removes a set of entries, from start (inclusive) to end
	// (exclusive). Similar to Clear, this method actually removes entries from
	// the storage engine.
	ClearRange(start, end MVCCKey) error
	// ClearIterRange removes a set of entries, from start (inclusive) to end
	// (exclusive). Similar to Clear and ClearRange, this method actually removes
	// entries from the storage engine. Unlike ClearRange, the entries to remove
	// are determined by iterating over iter and per-key tombstones are
	// generated.
	ClearIterRange(iter Iterator, start, end MVCCKey) error
	// Merge is a high-performance write operation used for values which are
	// accumulated over several writes. Multiple values can be merged
	// sequentially into a single key; a subsequent read will return a "merged"
	// value which is computed from the original merged values.
	//
	// Merge currently provides specialized behavior for three data types:
	// integers, byte slices, and time series observations. Merged integers are
	// summed, acting as a high-performance accumulator.  Byte slices are simply
	// concatenated in the order they are merged. Time series observations
	// (stored as byte slices with a special tag on the roachpb.Value) are
	// combined with specialized logic beyond that of simple byte slices.
	//
	// The logic for merges is written in db.cc in order to be compatible with RocksDB.
	Merge(key MVCCKey, value []byte) error
	// Put sets the given key to the value provided.
	Put(key MVCCKey, value []byte) error
}

// ReadWriter is the read/write interface to an engine's data.
type ReadWriter interface {
	Reader
	Writer
}

// Engine is the interface that wraps the core operations of a key/value store.
type Engine interface {
	ReadWriter
	// Attrs returns the engine/store attributes.
	Attrs() roachpb.Attributes
	// Capacity returns capacity details for the engine's available storage.
	Capacity() (roachpb.StoreCapacity, error)
	// Flush causes the engine to write all in-memory data to disk
	// immediately.
	Flush() error
	// GetStats retrieves stats from the engine.
	GetStats() (*Stats, error)
	// NewBatch returns a new instance of a batched engine which wraps
	// this engine. Batched engines accumulate all mutations and apply
	// them atomically on a call to Commit().
	NewBatch() Batch
	// NewWriteOnlyBatch returns a new instance of a batched engine which wraps
	// this engine. A write-only batch accumulates all mutations and applies them
	// atomically on a call to Commit(). Read operations return an error.
	//
	// TODO(peter): This should return a WriteBatch interface, but there are mild
	// complications in both defining that interface and implementing it. In
	// particular, Batch.Close would no longer come from Reader and we'd need to
	// refactor a bunch of code in rocksDBBatch.
	NewWriteOnlyBatch() Batch
	// NewSnapshot returns a new instance of a read-only snapshot
	// engine. Snapshots are instantaneous and, as long as they're
	// released relatively quickly, inexpensive. Snapshots are released
	// by invoking Close(). Note that snapshots must not be used after the
	// original engine has been stopped.
	NewSnapshot() Reader
}

// Batch is the interface for batch specific operations.
type Batch interface {
	ReadWriter
	// Commit atomically applies any batched updates to the underlying
	// engine. This is a noop unless the engine was created via NewBatch(). If
	// sync is true, the batch is synchronously committed to disk.
	Commit(sync bool) error
	// Distinct returns a view of the existing batch which only sees writes that
	// were performed before the Distinct batch was created. That is, the
	// returned batch will not read its own writes, but it will read writes to
	// the parent batch performed before the call to Distinct(). The returned
	// batch needs to be closed before using the parent batch again. This is used
	// as an optimization to avoid flushing mutations buffered by the batch in
	// situations where we know all of the batched operations are for distinct
	// keys.
	Distinct() ReadWriter
	// Repr returns the underlying representation of the batch and can be used to
	// reconstitute the batch on a remote node using Writer.ApplyBatchRepr().
	Repr() []byte
}

// Stats is a set of RocksDB stats. These are all described in RocksDB
//
// Currently, we collect stats from the following sources:
// 1. RocksDB's internal "tickers" (i.e. counters). They're defined in
//    rocksdb/statistics.h
// 2. DBEventListener, which implements RocksDB's EventListener interface.
// 3. rocksdb::DB::GetProperty().
//
// This is a good resource describing RocksDB's memory-related stats:
// https://github.com/facebook/rocksdb/wiki/Memory-usage-in-RocksDB
type Stats struct {
	BlockCacheHits           int64
	BlockCacheMisses         int64
	BlockCacheUsage          int64
	BlockCachePinnedUsage    int64
	BloomFilterPrefixChecked int64
	BloomFilterPrefixUseful  int64
	MemtableHits             int64
	MemtableMisses           int64
	MemtableTotalSize        int64
	Flushes                  int64
	Compactions              int64
	TableReadersMemEstimate  int64
}

// PutProto sets the given key to the protobuf-serialized byte string
// of msg and the provided timestamp. Returns the length in bytes of
// key and the value.
func PutProto(engine Writer, key MVCCKey, msg proto.Message) (keyBytes, valBytes int64, err error) {
	bytes, err := protoutil.Marshal(msg)
	if err != nil {
		return 0, 0, err
	}

	if err := engine.Put(key, bytes); err != nil {
		return 0, 0, err
	}

	return int64(key.EncodedSize()), int64(len(bytes)), nil
}

// Scan returns up to max key/value objects starting from
// start (inclusive) and ending at end (non-inclusive).
// Specify max=0 for unbounded scans.
func Scan(engine Reader, start, end MVCCKey, max int64) ([]MVCCKeyValue, error) {
	var kvs []MVCCKeyValue
	err := engine.Iterate(start, end, func(kv MVCCKeyValue) (bool, error) {
		if max != 0 && int64(len(kvs)) >= max {
			return true, nil
		}
		kvs = append(kvs, kv)
		return false, nil
	})
	return kvs, err
}
<commit_msg>engine: fix Iterator.{UnsafeKey,UnsafeValue} comments<commit_after>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Andrew Bonventre (andybons@gmail.com)
// Author: Spencer Kimball (spencer.kimball@gmail.com)
// Author: Jiang-Ming Yang (jiangming.yang@gmail.com)

package engine

import (
	"github.com/gogo/protobuf/proto"

	"github.com/cockroachdb/cockroach/pkg/roachpb"
	"github.com/cockroachdb/cockroach/pkg/storage/engine/enginepb"
	"github.com/cockroachdb/cockroach/pkg/util/protoutil"
)

// Iterator is an interface for iterating over key/value pairs in an
// engine. Iterator implementations are thread safe unless otherwise
// noted.
type Iterator interface {
	// Close frees up resources held by the iterator.
	Close()
	// Seek advances the iterator to the first key in the engine which
	// is >= the provided key.
	Seek(key MVCCKey)
	// SeekReverse advances the iterator to the first key in the engine which
	// is <= the provided key.
	SeekReverse(key MVCCKey)
	// Valid returns true if the iterator is currently valid. An iterator that
	// hasn't had Seek called on it or has gone past the end of the key range
	// is invalid.
	Valid() bool
	// Next advances the iterator to the next key/value in the
	// iteration. After this call, Valid() will be true if the
	// iterator was not positioned at the last key.
	Next()
	// Prev moves the iterator backward to the previous key/value
	// in the iteration. After this call, Valid() will be true if the
	// iterator was not positioned at the first key.
	Prev()
	// NextKey advances the iterator to the next MVCC key. This operation is
	// distinct from Next which advances to the next version of the current key
	// or the next key if the iterator is currently located at the last version
	// for a key.
	NextKey()
	// PrevKey moves the iterator backward to the previous MVCC key. This
	// operation is distinct from Prev which moves the iterator backward to the
	// prev version of the current key or the prev key if the iterator is
	// currently located at the first version for a key.
	PrevKey()
	// Key returns the current key.
	Key() MVCCKey
	// Value returns the current value as a byte slice.
	Value() []byte
	// ValueProto unmarshals the value the iterator is currently
	// pointing to using a protobuf decoder.
	ValueProto(msg proto.Message) error
	// UnsafeKey returns the same value as Key, but the memory is invalidated on
	// the next call to {Next,Prev,Seek,SeekReverse,Close}.
	UnsafeKey() MVCCKey
	// UnsafeValue returns the same value as Value, but the memory is
	// invalidated on the next call to {Next,Prev,Seek,SeekReverse,Close}.
	UnsafeValue() []byte
	// Less returns true if the key the iterator is currently positioned at is
	// less than the specified key.
	Less(key MVCCKey) bool
	// Error returns the error, if any, which the iterator encountered.
	Error() error
	// ComputeStats scans the underlying engine from start to end keys and
	// computes stats counters based on the values. This method is used after a
	// range is split to recompute stats for each subrange. The start key is
	// always adjusted to avoid counting local keys in the event stats are being
	// recomputed for the first range (i.e. the one with start key == KeyMin).
	// The nowNanos arg specifies the wall time in nanoseconds since the
	// epoch and is used to compute the total age of all intents.
	ComputeStats(start, end MVCCKey, nowNanos int64) (enginepb.MVCCStats, error)
}

// Reader is the read interface to an engine's data.
type Reader interface {
	// Close closes the reader, freeing up any outstanding resources. Note that
	// various implementations have slightly different behaviors. In particular,
	// Distinct() batches release their parent batch for future use while
	// Engines, Snapshots and Batches free the associated C++ resources.
	Close()
	// Closed returns true if the reader has been closed or is not usable.
	// Objects backed by this reader (e.g. Iterators) can check this to ensure
	// that they are not using a closed engine. Intended for use within package
	// engine; exported to enable wrappers to exist in other packages.
	Closed() bool
	// Get returns the value for the given key, nil otherwise.
	Get(key MVCCKey) ([]byte, error)
	// GetProto fetches the value at the specified key and unmarshals it
	// using a protobuf decoder. Returns true on success or false if the
	// key was not found. On success, returns the length in bytes of the
	// key and the value.
	GetProto(key MVCCKey, msg proto.Message) (ok bool, keyBytes, valBytes int64, err error)
	// Iterate scans from start to end keys, visiting at most max
	// key/value pairs. On each key value pair, the function f is
	// invoked. If f returns an error or if the scan itself encounters
	// an error, the iteration will stop and return the error.
	// If the first result of f is true, the iteration stops.
	Iterate(start, end MVCCKey, f func(MVCCKeyValue) (bool, error)) error
	// NewIterator returns a new instance of an Iterator over this engine. When
	// prefix is true, Seek will use the user-key prefix of the supplied MVCC key
	// to restrict which sstables are searched, but iteration (using Next) over
	// keys without the same user-key prefix will not work correctly (keys may be
	// skipped). The caller must invoke Iterator.Close() when finished with the
	// iterator to free resources.
	NewIterator(prefix bool) Iterator
}

// Writer is the write interface to an engine's data.
type Writer interface {
	// ApplyBatchRepr atomically applies a set of batched updates. Created by
	// calling Repr() on a batch. Using this method is equivalent to constructing
	// and committing a batch whose Repr() equals repr. If sync is true, the
	// batch is synchronously written to disk. It is an error to specify
	// sync=true if the Writer is a Batch.
	ApplyBatchRepr(repr []byte, sync bool) error
	// Clear removes the item from the db with the given key.
	// Note that clear actually removes entries from the storage
	// engine, rather than inserting tombstones.
	Clear(key MVCCKey) error
	// ClearRange removes a set of entries, from start (inclusive) to end
	// (exclusive). Similar to Clear, this method actually removes entries from
	// the storage engine.
	ClearRange(start, end MVCCKey) error
	// ClearIterRange removes a set of entries, from start (inclusive) to end
	// (exclusive). Similar to Clear and ClearRange, this method actually removes
	// entries from the storage engine. Unlike ClearRange, the entries to remove
	// are determined by iterating over iter and per-key tombstones are
	// generated.
	ClearIterRange(iter Iterator, start, end MVCCKey) error
	// Merge is a high-performance write operation used for values which are
	// accumulated over several writes. Multiple values can be merged
	// sequentially into a single key; a subsequent read will return a "merged"
	// value which is computed from the original merged values.
	//
	// Merge currently provides specialized behavior for three data types:
	// integers, byte slices, and time series observations. Merged integers are
	// summed, acting as a high-performance accumulator.  Byte slices are simply
	// concatenated in the order they are merged. Time series observations
	// (stored as byte slices with a special tag on the roachpb.Value) are
	// combined with specialized logic beyond that of simple byte slices.
	//
	// The logic for merges is written in db.cc in order to be compatible with RocksDB.
	Merge(key MVCCKey, value []byte) error
	// Put sets the given key to the value provided.
	Put(key MVCCKey, value []byte) error
}

// ReadWriter is the read/write interface to an engine's data.
type ReadWriter interface {
	Reader
	Writer
}

// Engine is the interface that wraps the core operations of a key/value store.
type Engine interface {
	ReadWriter
	// Attrs returns the engine/store attributes.
	Attrs() roachpb.Attributes
	// Capacity returns capacity details for the engine's available storage.
	Capacity() (roachpb.StoreCapacity, error)
	// Flush causes the engine to write all in-memory data to disk
	// immediately.
	Flush() error
	// GetStats retrieves stats from the engine.
	GetStats() (*Stats, error)
	// NewBatch returns a new instance of a batched engine which wraps
	// this engine. Batched engines accumulate all mutations and apply
	// them atomically on a call to Commit().
	NewBatch() Batch
	// NewWriteOnlyBatch returns a new instance of a batched engine which wraps
	// this engine. A write-only batch accumulates all mutations and applies them
	// atomically on a call to Commit(). Read operations return an error.
	//
	// TODO(peter): This should return a WriteBatch interface, but there are mild
	// complications in both defining that interface and implementing it. In
	// particular, Batch.Close would no longer come from Reader and we'd need to
	// refactor a bunch of code in rocksDBBatch.
	NewWriteOnlyBatch() Batch
	// NewSnapshot returns a new instance of a read-only snapshot
	// engine. Snapshots are instantaneous and, as long as they're
	// released relatively quickly, inexpensive. Snapshots are released
	// by invoking Close(). Note that snapshots must not be used after the
	// original engine has been stopped.
	NewSnapshot() Reader
}

// Batch is the interface for batch specific operations.
type Batch interface {
	ReadWriter
	// Commit atomically applies any batched updates to the underlying
	// engine. This is a noop unless the engine was created via NewBatch(). If
	// sync is true, the batch is synchronously committed to disk.
	Commit(sync bool) error
	// Distinct returns a view of the existing batch which only sees writes that
	// were performed before the Distinct batch was created. That is, the
	// returned batch will not read its own writes, but it will read writes to
	// the parent batch performed before the call to Distinct(). The returned
	// batch needs to be closed before using the parent batch again. This is used
	// as an optimization to avoid flushing mutations buffered by the batch in
	// situations where we know all of the batched operations are for distinct
	// keys.
	Distinct() ReadWriter
	// Repr returns the underlying representation of the batch and can be used to
	// reconstitute the batch on a remote node using Writer.ApplyBatchRepr().
	Repr() []byte
}

// Stats is a set of RocksDB stats. These are all described in RocksDB
//
// Currently, we collect stats from the following sources:
// 1. RocksDB's internal "tickers" (i.e. counters). They're defined in
//    rocksdb/statistics.h
// 2. DBEventListener, which implements RocksDB's EventListener interface.
// 3. rocksdb::DB::GetProperty().
//
// This is a good resource describing RocksDB's memory-related stats:
// https://github.com/facebook/rocksdb/wiki/Memory-usage-in-RocksDB
type Stats struct {
	BlockCacheHits           int64
	BlockCacheMisses         int64
	BlockCacheUsage          int64
	BlockCachePinnedUsage    int64
	BloomFilterPrefixChecked int64
	BloomFilterPrefixUseful  int64
	MemtableHits             int64
	MemtableMisses           int64
	MemtableTotalSize        int64
	Flushes                  int64
	Compactions              int64
	TableReadersMemEstimate  int64
}

// PutProto sets the given key to the protobuf-serialized byte string
// of msg and the provided timestamp. Returns the length in bytes of
// key and the value.
func PutProto(engine Writer, key MVCCKey, msg proto.Message) (keyBytes, valBytes int64, err error) {
	bytes, err := protoutil.Marshal(msg)
	if err != nil {
		return 0, 0, err
	}

	if err := engine.Put(key, bytes); err != nil {
		return 0, 0, err
	}

	return int64(key.EncodedSize()), int64(len(bytes)), nil
}

// Scan returns up to max key/value objects starting from
// start (inclusive) and ending at end (non-inclusive).
// Specify max=0 for unbounded scans.
func Scan(engine Reader, start, end MVCCKey, max int64) ([]MVCCKeyValue, error) {
	var kvs []MVCCKeyValue
	err := engine.Iterate(start, end, func(kv MVCCKeyValue) (bool, error) {
		if max != 0 && int64(len(kvs)) >= max {
			return true, nil
		}
		kvs = append(kvs, kv)
		return false, nil
	})
	return kvs, err
}
<|endoftext|>


<commit_before>// Copyright (c) 2018-2019 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package ticketbuyer

import (
	"context"
	"net"
	"runtime/trace"
	"sync"

	"decred.org/dcrwallet/errors"
	"decred.org/dcrwallet/wallet"
	"github.com/decred/dcrd/dcrutil/v3"
	"github.com/decred/dcrd/wire"
)

const minconf = 1

// Config modifies the behavior of TB.
type Config struct {
	BuyTickets bool

	// Account to buy tickets from
	Account uint32

	// Account to derive voting addresses from; overridden by VotingAddr
	VotingAccount uint32

	// Minimum amount to maintain in purchasing account
	Maintain dcrutil.Amount

	// Address to assign voting rights; overrides VotingAccount
	VotingAddr dcrutil.Address

	// Commitment address for stakepool fees
	PoolFeeAddr dcrutil.Address

	// Stakepool fee percentage (between 0-100)
	PoolFees float64

	// Limit maximum number of purchased tickets per block
	Limit int

	// CSPP-related options
	CSPPServer         string
	DialCSPPServer     func(ctx context.Context, network, addr string) (net.Conn, error)
	MixedAccount       uint32
	MixedAccountBranch uint32
	TicketSplitAccount uint32
	ChangeAccount      uint32
	MixChange          bool
}

// TB is an automated ticket buyer, buying as many tickets as possible given an
// account's available balance.  TB may be configured to buy tickets for any
// arbitrary voting address or (optional) stakepool.
type TB struct {
	wallet *wallet.Wallet

	cfg Config
	mu  sync.Mutex
}

// New returns a new TB to buy tickets from a wallet using the default config.
func New(w *wallet.Wallet) *TB {
	return &TB{wallet: w}
}

// Run executes the ticket buyer.  If the private passphrase is incorrect, or
// ever becomes incorrect due to a wallet passphrase change, Run exits with an
// errors.Passphrase error.
func (tb *TB) Run(ctx context.Context, passphrase []byte) error {
	err := tb.wallet.Unlock(ctx, passphrase, nil)
	if err != nil {
		return err
	}

	c := tb.wallet.NtfnServer.MainTipChangedNotifications()
	defer c.Done()

	ctx, outerCancel := context.WithCancel(ctx)
	var fatal error
	var fatalMu sync.Mutex

	var nextIntervalStart, expiry int32
	var cancels []func()
	for {
		select {
		case <-ctx.Done():
			fatalMu.Lock()
			err := fatal
			fatalMu.Unlock()
			if err != nil {
				return err
			}
			return ctx.Err()
		case n := <-c.C:
			if len(n.AttachedBlocks) == 0 {
				continue
			}

			tip := n.AttachedBlocks[len(n.AttachedBlocks)-1]
			w := tb.wallet
			tipHeader, err := w.BlockHeader(ctx, tip)
			if err != nil {
				log.Error(err)
				continue
			}
			height := int32(tipHeader.Height)

			// Cancel any ongoing ticket purchases which are buying
			// at an old ticket price or are no longer able to
			// create mined tickets the window.
			if height+2 >= nextIntervalStart {
				for i, cancel := range cancels {
					cancel()
					cancels[i] = nil
				}
				cancels = cancels[:0]

				intervalSize := int32(w.ChainParams().StakeDiffWindowSize)
				currentInterval := height / intervalSize
				nextIntervalStart = (currentInterval + 1) * intervalSize

				// Skip this purchase when no more tickets may be purchased in the interval and
				// the next sdiff is unknown.  The earliest any ticket may be mined is two
				// blocks from now, with the next block containing the split transaction
				// that the ticket purchase spends.
				if height+2 == nextIntervalStart {
					log.Debugf("Skipping purchase: next sdiff interval starts soon")
					continue
				}
				// Set expiry to prevent tickets from being mined in the next
				// sdiff interval.  When the next block begins the new interval,
				// the ticket is being purchased for the next interval; therefore
				// increment expiry by a full sdiff window size to prevent it
				// being mined in the interval after the next.
				expiry = nextIntervalStart
				if height+1 == nextIntervalStart {
					expiry += intervalSize
				}
			}

			cancelCtx, cancel := context.WithCancel(ctx)
			cancels = append(cancels, cancel)
			go func() {
				err := tb.buy(cancelCtx, passphrase, tipHeader, expiry)
				if err != nil {
					switch {
					// silence these errors
					case errors.Is(err, errors.InsufficientBalance):
					case errors.Is(err, context.Canceled):
					case errors.Is(err, context.DeadlineExceeded):
					default:
						log.Errorf("Ticket purchasing failed: %v", err)
					}
					if errors.Is(err, errors.Passphrase) {
						fatalMu.Lock()
						fatal = err
						fatalMu.Unlock()
						outerCancel()
					}
				}
			}()
			go func() {
				err := tb.mixChange(ctx)
				if err != nil {
					log.Error(err)
				}
			}()
		}
	}
}

func (tb *TB) buy(ctx context.Context, passphrase []byte, tip *wire.BlockHeader, expiry int32) error {
	ctx, task := trace.NewTask(ctx, "ticketbuyer.buy")
	defer task.End()

	tb.mu.Lock()
	buyTickets := tb.cfg.BuyTickets
	tb.mu.Unlock()
	if !buyTickets {
		return nil
	}

	w := tb.wallet

	// Don't buy tickets for this attached block when transactions are not
	// synced through the tip block.
	rp, err := w.RescanPoint(ctx)
	if err != nil {
		return err
	}
	if rp != nil {
		log.Debugf("Skipping purchase: transactions are not synced")
		return nil
	}

	// Unable to publish any transactions if the network backend is unset.
	n, err := w.NetworkBackend()
	if err != nil {
		return err
	}

	// Ensure wallet is unlocked with the current passphrase.  If the passphase
	// is changed, the Run exits and TB must be restarted with the new
	// passphrase.
	err = w.Unlock(ctx, passphrase, nil)
	if err != nil {
		return err
	}

	// Read config
	tb.mu.Lock()
	account := tb.cfg.Account
	maintain := tb.cfg.Maintain
	votingAddr := tb.cfg.VotingAddr
	poolFeeAddr := tb.cfg.PoolFeeAddr
	poolFees := tb.cfg.PoolFees
	limit := tb.cfg.Limit
	csppServer := tb.cfg.CSPPServer
	dialCSPPServer := tb.cfg.DialCSPPServer
	votingAccount := tb.cfg.VotingAccount
	mixedAccount := tb.cfg.MixedAccount
	mixedBranch := tb.cfg.MixedAccountBranch
	splitAccount := tb.cfg.TicketSplitAccount
	changeAccount := tb.cfg.ChangeAccount
	tb.mu.Unlock()

	// Determine how many tickets to buy
	bal, err := w.AccountBalance(ctx, account, minconf)
	if err != nil {
		return err
	}
	spendable := bal.Spendable
	if spendable < maintain {
		log.Debugf("Skipping purchase: low available balance")
		return nil
	}
	spendable -= maintain
	sdiff, err := w.NextStakeDifficultyAfterHeader(ctx, tip)
	if err != nil {
		return err
	}
	buy := int(spendable / sdiff)
	if buy == 0 {
		log.Debugf("Skipping purchase: low available balance")
		return nil
	}
	if max := int(w.ChainParams().MaxFreshStakePerBlock); buy > max {
		buy = max
	}
	if limit > 0 && buy > limit {
		buy = limit
	}

	tix, err := w.PurchaseTickets(ctx, n, &wallet.PurchaseTicketsRequest{
		Count:         buy,
		SourceAccount: account,
		VotingAddress: votingAddr,
		MinConf:       minconf,
		Expiry:        expiry,

		// CSPP
		CSPPServer:         csppServer,
		DialCSPPServer:     dialCSPPServer,
		VotingAccount:      votingAccount,
		MixedAccount:       mixedAccount,
		MixedAccountBranch: mixedBranch,
		MixedSplitAccount:  splitAccount,
		ChangeAccount:      changeAccount,

		// VSPs
		VSPAddress: poolFeeAddr,
		VSPFees:    poolFees,
	})
	if tix != nil {
		for _, hash := range tix.TicketHashes {
			log.Infof("Purchased ticket %v at stake difficulty %v", hash, sdiff)
		}
	}
	return err
}

// AccessConfig runs f with the current config passed as a parameter.  The
// config is protected by a mutex and this function is safe for concurrent
// access to read or modify the config.  It is unsafe to leak a pointer to the
// config, but a copy of *cfg is legal.
func (tb *TB) AccessConfig(f func(cfg *Config)) {
	tb.mu.Lock()
	f(&tb.cfg)
	tb.mu.Unlock()
}

func (tb *TB) mixChange(ctx context.Context) error {
	// Read config
	tb.mu.Lock()
	dial := tb.cfg.DialCSPPServer
	csppServer := tb.cfg.CSPPServer
	mixedAccount := tb.cfg.MixedAccount
	mixedBranch := tb.cfg.MixedAccountBranch
	changeAccount := tb.cfg.ChangeAccount
	mixChange := tb.cfg.MixChange
	tb.mu.Unlock()

	if !mixChange {
		return nil
	}

	ctx, task := trace.NewTask(ctx, "ticketbuyer.mixChange")
	defer task.End()

	return tb.wallet.MixAccount(ctx, dial, csppServer, changeAccount, mixedAccount, mixedBranch)
}
<commit_msg>Limit mixed ticketbuying to one when limit is unset.<commit_after>// Copyright (c) 2018-2019 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package ticketbuyer

import (
	"context"
	"net"
	"runtime/trace"
	"sync"

	"decred.org/dcrwallet/errors"
	"decred.org/dcrwallet/wallet"
	"github.com/decred/dcrd/dcrutil/v3"
	"github.com/decred/dcrd/wire"
)

const minconf = 1

// Config modifies the behavior of TB.
type Config struct {
	BuyTickets bool

	// Account to buy tickets from
	Account uint32

	// Account to derive voting addresses from; overridden by VotingAddr
	VotingAccount uint32

	// Minimum amount to maintain in purchasing account
	Maintain dcrutil.Amount

	// Address to assign voting rights; overrides VotingAccount
	VotingAddr dcrutil.Address

	// Commitment address for stakepool fees
	PoolFeeAddr dcrutil.Address

	// Stakepool fee percentage (between 0-100)
	PoolFees float64

	// Limit maximum number of purchased tickets per block
	Limit int

	// CSPP-related options
	CSPPServer         string
	DialCSPPServer     func(ctx context.Context, network, addr string) (net.Conn, error)
	MixedAccount       uint32
	MixedAccountBranch uint32
	TicketSplitAccount uint32
	ChangeAccount      uint32
	MixChange          bool
}

// TB is an automated ticket buyer, buying as many tickets as possible given an
// account's available balance.  TB may be configured to buy tickets for any
// arbitrary voting address or (optional) stakepool.
type TB struct {
	wallet *wallet.Wallet

	cfg Config
	mu  sync.Mutex
}

// New returns a new TB to buy tickets from a wallet using the default config.
func New(w *wallet.Wallet) *TB {
	return &TB{wallet: w}
}

// Run executes the ticket buyer.  If the private passphrase is incorrect, or
// ever becomes incorrect due to a wallet passphrase change, Run exits with an
// errors.Passphrase error.
func (tb *TB) Run(ctx context.Context, passphrase []byte) error {
	err := tb.wallet.Unlock(ctx, passphrase, nil)
	if err != nil {
		return err
	}

	c := tb.wallet.NtfnServer.MainTipChangedNotifications()
	defer c.Done()

	ctx, outerCancel := context.WithCancel(ctx)
	var fatal error
	var fatalMu sync.Mutex

	var nextIntervalStart, expiry int32
	var cancels []func()
	for {
		select {
		case <-ctx.Done():
			fatalMu.Lock()
			err := fatal
			fatalMu.Unlock()
			if err != nil {
				return err
			}
			return ctx.Err()
		case n := <-c.C:
			if len(n.AttachedBlocks) == 0 {
				continue
			}

			tip := n.AttachedBlocks[len(n.AttachedBlocks)-1]
			w := tb.wallet
			tipHeader, err := w.BlockHeader(ctx, tip)
			if err != nil {
				log.Error(err)
				continue
			}
			height := int32(tipHeader.Height)

			// Cancel any ongoing ticket purchases which are buying
			// at an old ticket price or are no longer able to
			// create mined tickets the window.
			if height+2 >= nextIntervalStart {
				for i, cancel := range cancels {
					cancel()
					cancels[i] = nil
				}
				cancels = cancels[:0]

				intervalSize := int32(w.ChainParams().StakeDiffWindowSize)
				currentInterval := height / intervalSize
				nextIntervalStart = (currentInterval + 1) * intervalSize

				// Skip this purchase when no more tickets may be purchased in the interval and
				// the next sdiff is unknown.  The earliest any ticket may be mined is two
				// blocks from now, with the next block containing the split transaction
				// that the ticket purchase spends.
				if height+2 == nextIntervalStart {
					log.Debugf("Skipping purchase: next sdiff interval starts soon")
					continue
				}
				// Set expiry to prevent tickets from being mined in the next
				// sdiff interval.  When the next block begins the new interval,
				// the ticket is being purchased for the next interval; therefore
				// increment expiry by a full sdiff window size to prevent it
				// being mined in the interval after the next.
				expiry = nextIntervalStart
				if height+1 == nextIntervalStart {
					expiry += intervalSize
				}
			}

			cancelCtx, cancel := context.WithCancel(ctx)
			cancels = append(cancels, cancel)
			go func() {
				err := tb.buy(cancelCtx, passphrase, tipHeader, expiry)
				if err != nil {
					switch {
					// silence these errors
					case errors.Is(err, errors.InsufficientBalance):
					case errors.Is(err, context.Canceled):
					case errors.Is(err, context.DeadlineExceeded):
					default:
						log.Errorf("Ticket purchasing failed: %v", err)
					}
					if errors.Is(err, errors.Passphrase) {
						fatalMu.Lock()
						fatal = err
						fatalMu.Unlock()
						outerCancel()
					}
				}
			}()
			go func() {
				err := tb.mixChange(ctx)
				if err != nil {
					log.Error(err)
				}
			}()
		}
	}
}

func (tb *TB) buy(ctx context.Context, passphrase []byte, tip *wire.BlockHeader, expiry int32) error {
	ctx, task := trace.NewTask(ctx, "ticketbuyer.buy")
	defer task.End()

	tb.mu.Lock()
	buyTickets := tb.cfg.BuyTickets
	tb.mu.Unlock()
	if !buyTickets {
		return nil
	}

	w := tb.wallet

	// Don't buy tickets for this attached block when transactions are not
	// synced through the tip block.
	rp, err := w.RescanPoint(ctx)
	if err != nil {
		return err
	}
	if rp != nil {
		log.Debugf("Skipping purchase: transactions are not synced")
		return nil
	}

	// Unable to publish any transactions if the network backend is unset.
	n, err := w.NetworkBackend()
	if err != nil {
		return err
	}

	// Ensure wallet is unlocked with the current passphrase.  If the passphase
	// is changed, the Run exits and TB must be restarted with the new
	// passphrase.
	err = w.Unlock(ctx, passphrase, nil)
	if err != nil {
		return err
	}

	// Read config
	tb.mu.Lock()
	account := tb.cfg.Account
	maintain := tb.cfg.Maintain
	votingAddr := tb.cfg.VotingAddr
	poolFeeAddr := tb.cfg.PoolFeeAddr
	poolFees := tb.cfg.PoolFees
	limit := tb.cfg.Limit
	csppServer := tb.cfg.CSPPServer
	dialCSPPServer := tb.cfg.DialCSPPServer
	votingAccount := tb.cfg.VotingAccount
	mixedAccount := tb.cfg.MixedAccount
	mixedBranch := tb.cfg.MixedAccountBranch
	splitAccount := tb.cfg.TicketSplitAccount
	changeAccount := tb.cfg.ChangeAccount
	tb.mu.Unlock()

	// Determine how many tickets to buy
	bal, err := w.AccountBalance(ctx, account, minconf)
	if err != nil {
		return err
	}
	spendable := bal.Spendable
	if spendable < maintain {
		log.Debugf("Skipping purchase: low available balance")
		return nil
	}
	spendable -= maintain
	sdiff, err := w.NextStakeDifficultyAfterHeader(ctx, tip)
	if err != nil {
		return err
	}
	buy := int(spendable / sdiff)
	if buy == 0 {
		log.Debugf("Skipping purchase: low available balance")
		return nil
	}
	if max := int(w.ChainParams().MaxFreshStakePerBlock); buy > max {
		buy = max
	}
	if limit == 0 && csppServer != "" {
		buy = 1
	} else if limit > 0 && buy > limit {
		buy = limit
	}

	tix, err := w.PurchaseTickets(ctx, n, &wallet.PurchaseTicketsRequest{
		Count:         buy,
		SourceAccount: account,
		VotingAddress: votingAddr,
		MinConf:       minconf,
		Expiry:        expiry,

		// CSPP
		CSPPServer:         csppServer,
		DialCSPPServer:     dialCSPPServer,
		VotingAccount:      votingAccount,
		MixedAccount:       mixedAccount,
		MixedAccountBranch: mixedBranch,
		MixedSplitAccount:  splitAccount,
		ChangeAccount:      changeAccount,

		// VSPs
		VSPAddress: poolFeeAddr,
		VSPFees:    poolFees,
	})
	if tix != nil {
		for _, hash := range tix.TicketHashes {
			log.Infof("Purchased ticket %v at stake difficulty %v", hash, sdiff)
		}
	}
	return err
}

// AccessConfig runs f with the current config passed as a parameter.  The
// config is protected by a mutex and this function is safe for concurrent
// access to read or modify the config.  It is unsafe to leak a pointer to the
// config, but a copy of *cfg is legal.
func (tb *TB) AccessConfig(f func(cfg *Config)) {
	tb.mu.Lock()
	f(&tb.cfg)
	tb.mu.Unlock()
}

func (tb *TB) mixChange(ctx context.Context) error {
	// Read config
	tb.mu.Lock()
	dial := tb.cfg.DialCSPPServer
	csppServer := tb.cfg.CSPPServer
	mixedAccount := tb.cfg.MixedAccount
	mixedBranch := tb.cfg.MixedAccountBranch
	changeAccount := tb.cfg.ChangeAccount
	mixChange := tb.cfg.MixChange
	tb.mu.Unlock()

	if !mixChange {
		return nil
	}

	ctx, task := trace.NewTask(ctx, "ticketbuyer.mixChange")
	defer task.End()

	return tb.wallet.MixAccount(ctx, dial, csppServer, changeAccount, mixedAccount, mixedBranch)
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"log"

	"github.com/streadway/amqp"
)

// Worker does shoveling.
type Worker struct {
	ShovelConfig
	sourceConnection *amqp.Connection
	sourceChannel    *amqp.Channel
	sinkConnection   *amqp.Connection
	sinkChannel      *amqp.Channel
}

func (w *Worker) initSource() {
	connection, err := amqp.Dial(w.Source.URI())
	if err != nil {
		log.Fatal(err)
	}

	channel, err := connection.Channel()
	if err != nil {
		log.Fatal(err)
	}

	if _, err := channel.QueueDeclare(w.Source.Queue, true, false, false, false, nil); err != nil {
		log.Fatal(err)
	}

	for _, binding := range w.Source.Bindings {
		if binding.Exchange == "" {
			log.Fatal("exchange missing from source binding for: ", w.Name)
		}
		if err := channel.QueueBind(w.Source.Queue, binding.RoutingKey, binding.Exchange, false, nil); err != nil {
			log.Fatal(err)
		}
	}

	if err := channel.Qos(w.Source.Prefetch, 0, false); err != nil {
		log.Fatal(err)
	}

	w.sourceConnection = connection
	w.sourceChannel = channel
}

func (w *Worker) initSink() {
	connection, err := amqp.Dial(w.Sink.URI())
	if err != nil {
		log.Fatal(err)
	}

	channel, err := connection.Channel()
	if err != nil {
		log.Fatal(err)
	}

	if err := channel.ExchangeDeclare(w.Sink.Exchange, "topic", true, false, false, false, nil); err != nil {
		log.Fatal(err)
	}

	w.sinkConnection = connection
	w.sinkChannel = channel
}

// Init initializes the worker's source and connections, and establishes bindings.
func (w *Worker) Init() {
	w.initSource()
	w.initSink()
}

// Work does the shoveling and handles reconnecting as needed.
func (w *Worker) Work() {
	for {
		if w.sourceConnection == nil || w.sinkConnection == nil {
			w.Init()
		}

		err := w.doShoveling()
		if err != nil {
			log.Fatal(err)
		}

		w.sourceConnection.Close()
		w.sourceConnection = nil
		w.sinkConnection.Close()
		w.sinkConnection = nil
	}
}

func (w *Worker) doShoveling() error {
	// see https://godoc.org/github.com/streadway/amqp#example-Channel-Confirm-Bridge

	source := w.sourceChannel
	sink := w.sinkChannel

	shovel, err := source.Consume(w.Source.Queue, w.Name, false, false, false, false, nil)
	if err != nil {
		log.Panic(err)
		return err
	}

	confirms := sink.NotifyPublish(make(chan amqp.Confirmation, 1))
	if err := sink.Confirm(false); err != nil {
		log.Fatal(err)
	}

	for {
		msg, ok := <-shovel
		if !ok {
			return errors.New("source channel closed")
		}

		routingKey := msg.RoutingKey
		if w.Sink.RoutingKey != "" {
			routingKey = w.Sink.RoutingKey
		}

		err := sink.Publish(w.Sink.Exchange, routingKey, false, false, amqp.Publishing{
			ContentType:     msg.ContentType,
			ContentEncoding: msg.ContentEncoding,
			DeliveryMode:    msg.DeliveryMode,
			Priority:        msg.Priority,
			CorrelationId:   msg.CorrelationId,
			ReplyTo:         msg.ReplyTo,
			Expiration:      msg.Expiration,
			MessageId:       msg.MessageId,
			Timestamp:       msg.Timestamp,
			Type:            msg.Type,
			UserId:          msg.UserId,
			AppId:           msg.AppId,
			Headers:         msg.Headers,
			Body:            msg.Body})

		if err != nil {
			msg.Nack(false, true)
			log.Panic(err)
		}

		if confirmed := <-confirms; confirmed.Ack {
			msg.Ack(false)
		} else {
			msg.Nack(false, true)
		}
	}
}
<commit_msg>support exchange type option for sink<commit_after>package main

import (
	"errors"
	"log"

	"github.com/streadway/amqp"
)

// Worker does shoveling.
type Worker struct {
	ShovelConfig
	sourceConnection *amqp.Connection
	sourceChannel    *amqp.Channel
	sinkConnection   *amqp.Connection
	sinkChannel      *amqp.Channel
}

func (w *Worker) initSource() {
	connection, err := amqp.Dial(w.Source.URI())
	if err != nil {
		log.Fatal(err)
	}

	channel, err := connection.Channel()
	if err != nil {
		log.Fatal(err)
	}

	if _, err := channel.QueueDeclare(w.Source.Queue, true, false, false, false, nil); err != nil {
		log.Fatal(err)
	}

	for _, binding := range w.Source.Bindings {
		if binding.Exchange == "" {
			log.Fatal("exchange missing from source binding for: ", w.Name)
		}
		if err := channel.QueueBind(w.Source.Queue, binding.RoutingKey, binding.Exchange, false, nil); err != nil {
			log.Fatal(err)
		}
	}

	if err := channel.Qos(w.Source.Prefetch, 0, false); err != nil {
		log.Fatal(err)
	}

	w.sourceConnection = connection
	w.sourceChannel = channel
}

func (w *Worker) initSink() {
	connection, err := amqp.Dial(w.Sink.URI())
	if err != nil {
		log.Fatal(err)
	}

	channel, err := connection.Channel()
	if err != nil {
		log.Fatal(err)
	}

	if err := channel.ExchangeDeclare(w.Sink.Exchange, w.Sink.ExchangeType, true, false, false, false, nil); err != nil {
		log.Fatal(err)
	}

	w.sinkConnection = connection
	w.sinkChannel = channel
}

// Init initializes the worker's source and connections, and establishes bindings.
func (w *Worker) Init() {
	w.initSource()
	w.initSink()
}

// Work does the shoveling and handles reconnecting as needed.
func (w *Worker) Work() {
	for {
		if w.sourceConnection == nil || w.sinkConnection == nil {
			w.Init()
		}

		err := w.doShoveling()
		if err != nil {
			log.Fatal(err)
		}

		w.sourceConnection.Close()
		w.sourceConnection = nil
		w.sinkConnection.Close()
		w.sinkConnection = nil
	}
}

func (w *Worker) doShoveling() error {
	// see https://godoc.org/github.com/streadway/amqp#example-Channel-Confirm-Bridge

	source := w.sourceChannel
	sink := w.sinkChannel

	shovel, err := source.Consume(w.Source.Queue, w.Name, false, false, false, false, nil)
	if err != nil {
		log.Panic(err)
		return err
	}

	confirms := sink.NotifyPublish(make(chan amqp.Confirmation, 1))
	if err := sink.Confirm(false); err != nil {
		log.Fatal(err)
	}

	for {
		msg, ok := <-shovel
		if !ok {
			return errors.New("source channel closed")
		}

		routingKey := msg.RoutingKey
		if w.Sink.RoutingKey != "" {
			routingKey = w.Sink.RoutingKey
		}

		err := sink.Publish(w.Sink.Exchange, routingKey, false, false, amqp.Publishing{
			ContentType:     msg.ContentType,
			ContentEncoding: msg.ContentEncoding,
			DeliveryMode:    msg.DeliveryMode,
			Priority:        msg.Priority,
			CorrelationId:   msg.CorrelationId,
			ReplyTo:         msg.ReplyTo,
			Expiration:      msg.Expiration,
			MessageId:       msg.MessageId,
			Timestamp:       msg.Timestamp,
			Type:            msg.Type,
			UserId:          msg.UserId,
			AppId:           msg.AppId,
			Headers:         msg.Headers,
			Body:            msg.Body})

		if err != nil {
			msg.Nack(false, true)
			log.Panic(err)
		}

		if confirmed := <-confirms; confirmed.Ack {
			msg.Ack(false)
		} else {
			msg.Nack(false, true)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2003-2005 Maxim Sobolev. All rights reserved.
// Copyright (c) 2006-2015 Sippy Software, Inc. All rights reserved.
// Copyright (c) 2015 Andrii Pylypenko. All rights reserved.
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation and/or
// other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package sippy

import (
    "errors"
    "fmt"
    "strings"

    "sippy/net"
    "sippy/types"
)

type msgBody struct {
    mtype                   string
    sdp                     sippy_types.Sdp
    string_content          string
    needs_update            bool
    parsed                  bool
}

func NewMsgBody(content, mtype string) *msgBody {
    return &msgBody{
        mtype                   : mtype,
        sdp                     : nil,
        string_content          : content,
        needs_update            : true,
        parsed                  : false,
    }
}

func (self *msgBody) GetSdp() (sippy_types.Sdp, error) {
    err := self.parse()
    if err != nil {
        return nil, err
    }
    if self.sdp == nil {
        return nil, errors.New("Not an SDP message")
    }
    return self.sdp, nil
}

func (self *msgBody) parse() error {
    if self.parsed {
        return nil
    }
    if strings.HasPrefix(self.mtype, "multipart/mixed;") {
        arr := strings.SplitN(self.mtype, ";", 2)
        mtheaders := arr[1]
        var mth_boundary *string = nil
        for _, s := range strings.Split(mtheaders, ";") {
            arr = strings.SplitN(s, "=", 2)
            if arr[0] == "boundary" && len(arr) == 2 {
                mth_boundary = &arr[1]
                break
            }
        }
        if mth_boundary == nil {
            return errors.New("Error parsing the multipart message")
        }
        boundary := "--" + *mth_boundary
        for _, subsection := range strings.Split(self.string_content, boundary) {
            subsection = strings.TrimSpace(subsection)
            if subsection == "" { continue }
            boff, bdel := -1, ""
            for _, bdel = range []string{ "\r\n\r\n", "\r\r", "\n\n" } {
                boff = strings.Index(subsection, bdel)
                if boff != -1 {
                    break
                }
            }
            if boff == -1 {
                continue
            }
            mbody := subsection[boff + len(bdel):]
            mtype := ""
            for _, line := range strings.FieldsFunc(subsection[:boff], func(c rune) bool { return c == '\n' || c == '\r' }) {
                tmp := strings.ToLower(strings.TrimSpace(line))
                if strings.HasPrefix(tmp, "content-type:") {
                    arr = strings.SplitN(tmp, ":", 2)
                    mtype = strings.TrimSpace(arr[1])
                }
            }
            if mtype == "" {
                continue
            }
            if mtype == "application/sdp" {
                self.mtype = mtype
                self.string_content = mbody
                break
            }
        }
    }
    if self.mtype == "application/sdp" {
        sdp, err := ParseSdpBody(self.string_content)
        if err == nil {
            self.sdp = sdp
        } else {
            return fmt.Errorf("error parsing the SDP: %s", err.Error())
        }
    }
    self.parsed = true
    return nil
}

func (self *msgBody) String() string {
    if self.sdp != nil {
        self.string_content = self.sdp.String()
    }
    return self.string_content
}

func (self *msgBody) LocalStr(local_hostport *sippy_net.HostPort) string {
    if self.sdp != nil {
        return self.sdp.LocalStr(local_hostport)
    }
    return self.String()
}

func (self *msgBody) GetCopy() sippy_types.MsgBody {
    if self == nil {
        return nil
    }
    var sdp sippy_types.Sdp
    if self.sdp != nil {
        sdp = self.sdp.GetCopy()
    }
    return &msgBody{
        mtype                   : self.mtype,
        sdp                     : sdp,
        string_content          : self.string_content,
        needs_update            : true,
    }
}

func (self *msgBody) GetMtype() string {
    return self.mtype
}

func (self *msgBody) NeedsUpdate() bool {
    return self.needs_update
}

func (self *msgBody) SetNeedsUpdate(v bool) {
    self.needs_update = v
}

func (self *msgBody) AppendAHeader(hdr string) {
    if self.sdp != nil {
        self.sdp.AppendAHeader(hdr)
    } else {
        self.string_content += "a=" + hdr + "\r\n"
    }
}
<commit_msg>Do not forget to copy the parsed attribute too.<commit_after>// Copyright (c) 2003-2005 Maxim Sobolev. All rights reserved.
// Copyright (c) 2006-2015 Sippy Software, Inc. All rights reserved.
// Copyright (c) 2015 Andrii Pylypenko. All rights reserved.
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation and/or
// other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package sippy

import (
    "errors"
    "fmt"
    "strings"

    "sippy/net"
    "sippy/types"
)

type msgBody struct {
    mtype                   string
    sdp                     sippy_types.Sdp
    string_content          string
    needs_update            bool
    parsed                  bool
}

func NewMsgBody(content, mtype string) *msgBody {
    return &msgBody{
        mtype                   : mtype,
        sdp                     : nil,
        string_content          : content,
        needs_update            : true,
        parsed                  : false,
    }
}

func (self *msgBody) GetSdp() (sippy_types.Sdp, error) {
    err := self.parse()
    if err != nil {
        return nil, err
    }
    if self.sdp == nil {
        return nil, errors.New("Not an SDP message")
    }
    return self.sdp, nil
}

func (self *msgBody) parse() error {
    if self.parsed {
        return nil
    }
    if strings.HasPrefix(self.mtype, "multipart/mixed;") {
        arr := strings.SplitN(self.mtype, ";", 2)
        mtheaders := arr[1]
        var mth_boundary *string = nil
        for _, s := range strings.Split(mtheaders, ";") {
            arr = strings.SplitN(s, "=", 2)
            if arr[0] == "boundary" && len(arr) == 2 {
                mth_boundary = &arr[1]
                break
            }
        }
        if mth_boundary == nil {
            return errors.New("Error parsing the multipart message")
        }
        boundary := "--" + *mth_boundary
        for _, subsection := range strings.Split(self.string_content, boundary) {
            subsection = strings.TrimSpace(subsection)
            if subsection == "" { continue }
            boff, bdel := -1, ""
            for _, bdel = range []string{ "\r\n\r\n", "\r\r", "\n\n" } {
                boff = strings.Index(subsection, bdel)
                if boff != -1 {
                    break
                }
            }
            if boff == -1 {
                continue
            }
            mbody := subsection[boff + len(bdel):]
            mtype := ""
            for _, line := range strings.FieldsFunc(subsection[:boff], func(c rune) bool { return c == '\n' || c == '\r' }) {
                tmp := strings.ToLower(strings.TrimSpace(line))
                if strings.HasPrefix(tmp, "content-type:") {
                    arr = strings.SplitN(tmp, ":", 2)
                    mtype = strings.TrimSpace(arr[1])
                }
            }
            if mtype == "" {
                continue
            }
            if mtype == "application/sdp" {
                self.mtype = mtype
                self.string_content = mbody
                break
            }
        }
    }
    if self.mtype == "application/sdp" {
        sdp, err := ParseSdpBody(self.string_content)
        if err == nil {
            self.sdp = sdp
        } else {
            return fmt.Errorf("error parsing the SDP: %s", err.Error())
        }
    }
    self.parsed = true
    return nil
}

func (self *msgBody) String() string {
    if self.sdp != nil {
        self.string_content = self.sdp.String()
    }
    return self.string_content
}

func (self *msgBody) LocalStr(local_hostport *sippy_net.HostPort) string {
    if self.sdp != nil {
        return self.sdp.LocalStr(local_hostport)
    }
    return self.String()
}

func (self *msgBody) GetCopy() sippy_types.MsgBody {
    if self == nil {
        return nil
    }
    var sdp sippy_types.Sdp
    if self.sdp != nil {
        sdp = self.sdp.GetCopy()
    }
    return &msgBody{
        mtype                   : self.mtype,
        sdp                     : sdp,
        string_content          : self.string_content,
        needs_update            : true,
        parsed                  : self.parsed,
    }
}

func (self *msgBody) GetMtype() string {
    return self.mtype
}

func (self *msgBody) NeedsUpdate() bool {
    return self.needs_update
}

func (self *msgBody) SetNeedsUpdate(v bool) {
    self.needs_update = v
}

func (self *msgBody) AppendAHeader(hdr string) {
    if self.sdp != nil {
        self.sdp.AppendAHeader(hdr)
    } else {
        self.string_content += "a=" + hdr + "\r\n"
    }
}
<|endoftext|>


<commit_before>package expreduce

import "log"

type Rule struct {
	Lhs string
	Rhs string
}

type Definition struct {
	// The symbol name, like "Mean", and "Total"
	Name  string
	Usage string
	// Currently used for SetDelayed, since other definitions depend on
	// SetDelayed, we define it first.
	Bootstrap         bool
	OmitDocumentation bool
	ExpreduceSpecific bool
	Details           string

	// Regular rules to define. This should never include a map, as maps have
	// indeterminate iteration.
	Rules []Rule
	// Map symbol to Eval() function
	legacyEvalFn    (func(*Expression, *EvalState) Ex)
	SimpleExamples  []TestInstruction
	FurtherExamples []TestInstruction
	Tests           []TestInstruction
	KnownFailures   []TestInstruction
	KnownDangerous  []TestInstruction

	toString ToStringFnType

	Attributes []string
	Default    string
}

func ToTestInstructions(tc *Expression) []TestInstruction {
	instructions := []TestInstruction{}
	for _, tiEx := range tc.Parts[1:] {
		if st, isSt := HeadAssertion(tiEx, "System`ESameTest"); isSt {
			if len(st.Parts) != 3 {
				log.Fatalf("Invalid test case: %v\n", tiEx)
				continue
			}
			instructions = append(instructions, &SameTestEx{
				st.Parts[1], st.Parts[2]})
			continue
		}
		if st, isSt := HeadAssertion(tiEx, "System`EStringTest"); isSt {
			if len(st.Parts) != 3 {
				log.Fatalf("Invalid test case: %v\n", tiEx)
				continue
			}
			instructions = append(instructions, &StringTest{
				st.Parts[1].(*String).Val, st.Parts[2].(*String).Val})
			continue
		}
		if st, isSt := HeadAssertion(tiEx, "System`EExampleOnlyInstruction"); isSt {
			if len(st.Parts) != 3 {
				log.Fatalf("Invalid test case: %v\n", tiEx)
				continue
			}
			instructions = append(instructions, &ExampleOnlyInstruction{
				st.Parts[1].(*String).Val, st.Parts[2].(*String).Val})
			continue
		}
		if st, isSt := HeadAssertion(tiEx, "System`EComment"); isSt {
			if len(st.Parts) != 2 {
				log.Fatalf("Invalid test case: %v\n", tiEx)
				continue
			}
			comStr, comIsStr := st.Parts[1].(*String)
			if !comIsStr {
				log.Fatalf("Invalid test case: %v\n", tiEx)
				continue
			}
			instructions = append(instructions, &TestComment{
				comStr.Val})
			continue
		}
		if st, isSt := HeadAssertion(tiEx, "System`EResetState"); isSt {
			if len(st.Parts) != 1 {
				log.Fatalf("Invalid test case: %v\n", tiEx)
				continue
			}
			instructions = append(instructions, &ResetState{})
			continue
		}
		log.Fatalf("Invalid test case: %v\n", tiEx)
	}
	return instructions
}

func (def *Definition) AnnotateWithDynamicTests(es *EvalState) {
	tests, testsDef := es.GetSymDef("Tests`" + def.Name)
	if !testsDef {
		return
	}
	testsList, testsIsList := HeadAssertion(tests, "System`List")
	if !testsIsList {
		return
	}
	for _, testCol := range testsList.Parts[1:] {
		testColExpr, testColIsExpr := testCol.(*Expression)
		if !testColIsExpr {
			continue
		}
		headSym, headIsSym := testColExpr.Parts[0].(*Symbol)
		if !headIsSym {
			continue
		}
		if headSym.Name == "System`ESimpleExamples" {
			def.SimpleExamples = append(
				def.SimpleExamples,
				ToTestInstructions(testColExpr)...)
		} else if headSym.Name == "System`EFurtherExamples" {
			def.FurtherExamples = append(
				def.FurtherExamples,
				ToTestInstructions(testColExpr)...)
		} else if headSym.Name == "System`ETests" {
			def.Tests = append(
				def.Tests,
				ToTestInstructions(testColExpr)...)
		} else if headSym.Name == "System`EKnownFailures" {
			def.KnownFailures = append(
				def.KnownFailures,
				ToTestInstructions(testColExpr)...)
		} else if headSym.Name == "System`EKnownDangerous" {
			def.KnownDangerous = append(
				def.KnownDangerous,
				ToTestInstructions(testColExpr)...)
		} else {
			log.Fatalf("Invalid test collection: %v\n", testColExpr)
		}
	}
}

func (def *Definition) AnnotateWithDynamicUsage(es *EvalState) {
	if len(def.Usage) > 0 {
		return
	}
	lhs := NewExpression([]Ex{
		&Symbol{"System`MessageName"},
		&Symbol{def.Name},
		&String{"usage"},
	})
	usage, usageIsDef, _ := es.GetDef("MessageName", lhs)
	if !usageIsDef {
		return
	}
	if usageStr, usageIsStr := usage.(*String); usageIsStr {
		def.Usage = usageStr.Val
	}
}

func (def *Definition) AnnotateWithDynamic(es *EvalState) {
	def.AnnotateWithDynamicTests(es)
	def.AnnotateWithDynamicUsage(es)
}

type NamedDefSet struct {
	Name string
	Defs []Definition
}

func GetAllDefinitions() (defs []NamedDefSet) {
	defs = append(defs, NamedDefSet{"combinatorics", getCombinatoricsDefinitions()})
	defs = append(defs, NamedDefSet{"calculus", getCalculusDefinitions()})
	defs = append(defs, NamedDefSet{"comparison", getComparisonDefinitions()})
	defs = append(defs, NamedDefSet{"atoms", getAtomsDefinitions()})
	defs = append(defs, NamedDefSet{"functional", getFunctionalDefinitions()})
	defs = append(defs, NamedDefSet{"expression", GetExpressionDefinitions()})
	defs = append(defs, NamedDefSet{"solve", GetSolveDefinitions()})
	defs = append(defs, NamedDefSet{"flowcontrol", GetFlowControlDefinitions()})
	defs = append(defs, NamedDefSet{"list", GetListDefinitions()})
	defs = append(defs, NamedDefSet{"matrix", GetMatrixDefinitions()})
	defs = append(defs, NamedDefSet{"arithmetic", getArithmeticDefinitions()})
	defs = append(defs, NamedDefSet{"specialsyms", getSpecialSymsDefinitions()})
	defs = append(defs, NamedDefSet{"power", GetPowerDefinitions()})
	defs = append(defs, NamedDefSet{"random", GetRandomDefinitions()})
	defs = append(defs, NamedDefSet{"replacement", getReplacementDefinitions()})
	defs = append(defs, NamedDefSet{"sort", GetSortDefinitions()})
	defs = append(defs, NamedDefSet{"system", GetSystemDefinitions()})
	defs = append(defs, NamedDefSet{"trig", GetTrigDefinitions()})
	defs = append(defs, NamedDefSet{"string", GetStringDefinitions()})
	defs = append(defs, NamedDefSet{"time", GetTimeDefinitions()})
	defs = append(defs, NamedDefSet{"pattern", GetPatternDefinitions()})
	defs = append(defs, NamedDefSet{"boolean", GetBooleanDefinitions()})
	defs = append(defs, NamedDefSet{"simplify", GetSimplifyDefinitions()})
	defs = append(defs, NamedDefSet{"numbertheory", GetNumberTheoryDefinitions()})
	defs = append(defs, NamedDefSet{"manip", GetManipDefinitions()})

	// Check for duplicate definitions
	definedNames := make(map[string]bool)
	for _, defSet := range defs {
		for _, def := range defSet.Defs {
			_, alreadyDefined := definedNames[def.Name]
			if alreadyDefined {
				log.Fatalf("Found duplicate definition: %v\n", def.Name)
			}
			definedNames[def.Name] = true
		}
	}
	return
}
<commit_msg>Update usage lookup after context change.<commit_after>package expreduce

import "log"

type Rule struct {
	Lhs string
	Rhs string
}

type Definition struct {
	// The symbol name, like "Mean", and "Total"
	Name  string
	Usage string
	// Currently used for SetDelayed, since other definitions depend on
	// SetDelayed, we define it first.
	Bootstrap         bool
	OmitDocumentation bool
	ExpreduceSpecific bool
	Details           string

	// Regular rules to define. This should never include a map, as maps have
	// indeterminate iteration.
	Rules []Rule
	// Map symbol to Eval() function
	legacyEvalFn    (func(*Expression, *EvalState) Ex)
	SimpleExamples  []TestInstruction
	FurtherExamples []TestInstruction
	Tests           []TestInstruction
	KnownFailures   []TestInstruction
	KnownDangerous  []TestInstruction

	toString ToStringFnType

	Attributes []string
	Default    string
}

func ToTestInstructions(tc *Expression) []TestInstruction {
	instructions := []TestInstruction{}
	for _, tiEx := range tc.Parts[1:] {
		if st, isSt := HeadAssertion(tiEx, "System`ESameTest"); isSt {
			if len(st.Parts) != 3 {
				log.Fatalf("Invalid test case: %v\n", tiEx)
				continue
			}
			instructions = append(instructions, &SameTestEx{
				st.Parts[1], st.Parts[2]})
			continue
		}
		if st, isSt := HeadAssertion(tiEx, "System`EStringTest"); isSt {
			if len(st.Parts) != 3 {
				log.Fatalf("Invalid test case: %v\n", tiEx)
				continue
			}
			instructions = append(instructions, &StringTest{
				st.Parts[1].(*String).Val, st.Parts[2].(*String).Val})
			continue
		}
		if st, isSt := HeadAssertion(tiEx, "System`EExampleOnlyInstruction"); isSt {
			if len(st.Parts) != 3 {
				log.Fatalf("Invalid test case: %v\n", tiEx)
				continue
			}
			instructions = append(instructions, &ExampleOnlyInstruction{
				st.Parts[1].(*String).Val, st.Parts[2].(*String).Val})
			continue
		}
		if st, isSt := HeadAssertion(tiEx, "System`EComment"); isSt {
			if len(st.Parts) != 2 {
				log.Fatalf("Invalid test case: %v\n", tiEx)
				continue
			}
			comStr, comIsStr := st.Parts[1].(*String)
			if !comIsStr {
				log.Fatalf("Invalid test case: %v\n", tiEx)
				continue
			}
			instructions = append(instructions, &TestComment{
				comStr.Val})
			continue
		}
		if st, isSt := HeadAssertion(tiEx, "System`EResetState"); isSt {
			if len(st.Parts) != 1 {
				log.Fatalf("Invalid test case: %v\n", tiEx)
				continue
			}
			instructions = append(instructions, &ResetState{})
			continue
		}
		log.Fatalf("Invalid test case: %v\n", tiEx)
	}
	return instructions
}

func (def *Definition) AnnotateWithDynamicTests(es *EvalState) {
	tests, testsDef := es.GetSymDef("Tests`" + def.Name)
	if !testsDef {
		return
	}
	testsList, testsIsList := HeadAssertion(tests, "System`List")
	if !testsIsList {
		return
	}
	for _, testCol := range testsList.Parts[1:] {
		testColExpr, testColIsExpr := testCol.(*Expression)
		if !testColIsExpr {
			continue
		}
		headSym, headIsSym := testColExpr.Parts[0].(*Symbol)
		if !headIsSym {
			continue
		}
		if headSym.Name == "System`ESimpleExamples" {
			def.SimpleExamples = append(
				def.SimpleExamples,
				ToTestInstructions(testColExpr)...)
		} else if headSym.Name == "System`EFurtherExamples" {
			def.FurtherExamples = append(
				def.FurtherExamples,
				ToTestInstructions(testColExpr)...)
		} else if headSym.Name == "System`ETests" {
			def.Tests = append(
				def.Tests,
				ToTestInstructions(testColExpr)...)
		} else if headSym.Name == "System`EKnownFailures" {
			def.KnownFailures = append(
				def.KnownFailures,
				ToTestInstructions(testColExpr)...)
		} else if headSym.Name == "System`EKnownDangerous" {
			def.KnownDangerous = append(
				def.KnownDangerous,
				ToTestInstructions(testColExpr)...)
		} else {
			log.Fatalf("Invalid test collection: %v\n", testColExpr)
		}
	}
}

func (def *Definition) AnnotateWithDynamicUsage(es *EvalState) {
	if len(def.Usage) > 0 {
		return
	}
	lhs := NewExpression([]Ex{
		&Symbol{"System`MessageName"},
		&Symbol{"System`" + def.Name},
		&String{"usage"},
	})
	usage, usageIsDef, _ := es.GetDef("System`MessageName", lhs)
	if !usageIsDef {
		return
	}
	if usageStr, usageIsStr := usage.(*String); usageIsStr {
		def.Usage = usageStr.Val
	}
}

func (def *Definition) AnnotateWithDynamic(es *EvalState) {
	def.AnnotateWithDynamicTests(es)
	def.AnnotateWithDynamicUsage(es)
}

type NamedDefSet struct {
	Name string
	Defs []Definition
}

func GetAllDefinitions() (defs []NamedDefSet) {
	defs = append(defs, NamedDefSet{"combinatorics", getCombinatoricsDefinitions()})
	defs = append(defs, NamedDefSet{"calculus", getCalculusDefinitions()})
	defs = append(defs, NamedDefSet{"comparison", getComparisonDefinitions()})
	defs = append(defs, NamedDefSet{"atoms", getAtomsDefinitions()})
	defs = append(defs, NamedDefSet{"functional", getFunctionalDefinitions()})
	defs = append(defs, NamedDefSet{"expression", GetExpressionDefinitions()})
	defs = append(defs, NamedDefSet{"solve", GetSolveDefinitions()})
	defs = append(defs, NamedDefSet{"flowcontrol", GetFlowControlDefinitions()})
	defs = append(defs, NamedDefSet{"list", GetListDefinitions()})
	defs = append(defs, NamedDefSet{"matrix", GetMatrixDefinitions()})
	defs = append(defs, NamedDefSet{"arithmetic", getArithmeticDefinitions()})
	defs = append(defs, NamedDefSet{"specialsyms", getSpecialSymsDefinitions()})
	defs = append(defs, NamedDefSet{"power", GetPowerDefinitions()})
	defs = append(defs, NamedDefSet{"random", GetRandomDefinitions()})
	defs = append(defs, NamedDefSet{"replacement", getReplacementDefinitions()})
	defs = append(defs, NamedDefSet{"sort", GetSortDefinitions()})
	defs = append(defs, NamedDefSet{"system", GetSystemDefinitions()})
	defs = append(defs, NamedDefSet{"trig", GetTrigDefinitions()})
	defs = append(defs, NamedDefSet{"string", GetStringDefinitions()})
	defs = append(defs, NamedDefSet{"time", GetTimeDefinitions()})
	defs = append(defs, NamedDefSet{"pattern", GetPatternDefinitions()})
	defs = append(defs, NamedDefSet{"boolean", GetBooleanDefinitions()})
	defs = append(defs, NamedDefSet{"simplify", GetSimplifyDefinitions()})
	defs = append(defs, NamedDefSet{"numbertheory", GetNumberTheoryDefinitions()})
	defs = append(defs, NamedDefSet{"manip", GetManipDefinitions()})

	// Check for duplicate definitions
	definedNames := make(map[string]bool)
	for _, defSet := range defs {
		for _, def := range defSet.Defs {
			_, alreadyDefined := definedNames[def.Name]
			if alreadyDefined {
				log.Fatalf("Found duplicate definition: %v\n", def.Name)
			}
			definedNames[def.Name] = true
		}
	}
	return
}
<|endoftext|>


<commit_before>package parser_test

import (
	"bytes"
	"errors"
	"fmt"
	"net/http"
	"net/http/httptest"
	"reflect"
	"strings"
	"testing"
	"time"

	"cloud.google.com/go/bigquery"

	p "github.com/m-lab/etl/parser"
	"github.com/m-lab/etl/schema"
)

func TestAddMetaDataNDTConnSpec(t *testing.T) {
	tests := []struct {
		spec      schema.Web100ValueMap
		timestamp time.Time
		url       string
		res       schema.Web100ValueMap
	}{
		{
			spec: func() schema.Web100ValueMap {
				spec := schema.EmptyConnectionSpec()
				spec["client_ip"] = "127.0.0.1"
				spec["server_ip"] = "1.0.0.127"
				return spec
			}(),
			timestamp: time.Now(),
			url:       "/10583?",
			res: func() schema.Web100ValueMap {
				spec := schema.EmptyConnectionSpec()
				spec["client_ip"] = "127.0.0.1"
				spec["server_ip"] = "1.0.0.127"
				geoc := spec.Get("client_geolocation")
				geoc["continent_code"] = ""
				geoc["country_code"] = "US"
				geoc["country_code3"] = "USA"
				geoc["country_name"] = "United States of America"
				geoc["region"] = "NY"
				geoc["metro_code"] = int64(0)
				geoc["city"] = "Scarsdale"
				geoc["area_code"] = int64(10583)
				geoc["postal_code"] = "10583"
				geoc["latitude"] = float64(0.0)
				geoc["longitude"] = float64(0.0)
				geos := spec.Get("server_geolocation")
				geos["continent_code"] = ""
				geos["country_code"] = "US"
				geos["country_code3"] = "USA"
				geos["country_name"] = "United States of America"
				geos["region"] = "NY"
				geos["metro_code"] = int64(0)
				geos["city"] = "Scarsdale"
				geos["area_code"] = int64(10583)
				geos["postal_code"] = "10583"
				geos["latitude"] = float64(0.0)
				geos["longitude"] = float64(0.0)
				return spec
			}(),
		},
	}
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprint(w, `{"Geo":{"continent_code":"","country_code":"US","country_code3":"USA","country_name":"United States of America","region":"NY","metro_code":0,"city":"Scarsdale","area_code":10583,"postal_code":"10583","latitude":0,"longitude":0},"ASN":{}}`)
	}))
	for _, test := range tests {
		p.BaseURL = ts.URL + test.url
		p.AddMetaDataNDTConnSpec(test.spec, test.timestamp)
		if !reflect.DeepEqual(test.spec, test.res) {
			t.Errorf("Expected %+v, got %+v from data %s", test.res, test.spec, test.url)
		}
	}
}

func TestGetAndInsertNDT(t *testing.T) {
	tests := []struct {
		side      string
		spec      schema.Web100ValueMap
		timestamp time.Time
		url       string
		res       schema.Web100ValueMap
	}{
		{
			side:      "broken",
			spec:      schema.EmptyConnectionSpec(),
			timestamp: time.Now(),
			url:       "/notUsed?",
			res:       schema.EmptyConnectionSpec(),
		},
		{
			side:      "client",
			spec:      schema.EmptyConnectionSpec(),
			timestamp: time.Now(),
			url:       "portGarbage",
			res:       schema.EmptyConnectionSpec(),
		},
		{
			side: "client",
			spec: func() schema.Web100ValueMap {
				spec := schema.EmptyConnectionSpec()
				spec["client_ip"] = "127.0.0.1"
				return spec
			}(),
			timestamp: time.Now(),
			url:       "/10583?",
			res: func() schema.Web100ValueMap {
				spec := schema.EmptyConnectionSpec()
				spec["client_ip"] = "127.0.0.1"
				geo := spec.Get("client_geolocation")
				geo["continent_code"] = ""
				geo["country_code"] = "US"
				geo["country_code3"] = "USA"
				geo["country_name"] = "United States of America"
				geo["region"] = "NY"
				geo["metro_code"] = int64(0)
				geo["city"] = "Scarsdale"
				geo["area_code"] = int64(10583)
				geo["postal_code"] = "10583"
				geo["latitude"] = float64(0.0)
				geo["longitude"] = float64(0.0)
				return spec
			}(),
		},
	}
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprint(w, `{"Geo":{"continent_code":"","country_code":"US","country_code3":"USA","country_name":"United States of America","region":"NY","metro_code":0,"city":"Scarsdale","area_code":10583,"postal_code":"10583","latitude":0,"longitude":0},"ASN":{}}`)
	}))
	for _, test := range tests {
		p.BaseURL = ts.URL + test.url
		p.GetAndInsertNDT(test.side, test.spec, test.timestamp)
		if !reflect.DeepEqual(test.spec, test.res) {
			t.Errorf("Expected %+v, got %+v from data %s", test.res, test.spec, test.url)
		}
	}

}

func TestCopyStructToMap(t *testing.T) {
	tests := []struct {
		source interface{}
		dest   map[string]bigquery.Value
		res    map[string]bigquery.Value
	}{
		{
			source: &struct {
				A   int
				Bee string
			}{A: 1, Bee: "2"},
			dest: make(map[string]bigquery.Value),
			res:  map[string]bigquery.Value{"a": 1, "bee": "2"},
		},
		{
			source: &struct{}{},
			dest:   make(map[string]bigquery.Value),
			res:    map[string]bigquery.Value{},
		},
	}
	for _, test := range tests {
		p.CopyStructToMap(test.source, test.dest)
		if !reflect.DeepEqual(test.dest, test.res) {
			t.Errorf("Expected %+v, got %+v for data: %+v\n", test.res, test.dest, test.source)
		}
	}

}

func TestGetMetaData(t *testing.T) {
	tests := []struct {
		url string
		res *schema.MetaData
	}{
		{
			url: "portGarbage",
			res: nil,
		},
		{
			url: "/badJson",
			res: nil,
		},
		{
			url: "/goodJson",
			res: &schema.MetaData{},
		},
	}
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		path := r.URL.Path
		if strings.HasSuffix(path, "goodJson") {
			fmt.Fprint(w, `{"Geo":null,"ASN":null}`)
			return
		}
		fmt.Fprint(w, "{jngngfsljngsljngfsljngsljn")
	}))
	for _, test := range tests {
		res := p.GetMetaData(ts.URL + test.url)
		if res != test.res && *res != *test.res {
			t.Errorf("Expected %+v, got %+v for data: %s\n", test.res, res, test.url)
		}

	}

}

func TestQueryAnnotationService(t *testing.T) {
	tests := []struct {
		url string
		res []byte
		err error
	}{
		{
			url: "portGarbage",
			res: nil,
			err: errors.New("HTTP Protocol Error"),
		},
		{
			url: "/error",
			res: nil,
			err: errors.New("HTTP 404 Error"),
		},
		{
			url: "/Echo",
			res: []byte("Echo"),
			err: nil,
		},
	}
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		path := r.URL.Path
		if strings.HasSuffix(path, "error") {
			http.Error(w, "Test Error", 404)
			return
		}
		fmt.Fprint(w, "Echo")
	}))
	for _, test := range tests {
		json, err := p.QueryAnnotationService(ts.URL + test.url)
		if err != nil && test.err == nil || err == nil && test.err != nil {
			t.Errorf("Expected %s, got %s, for %s", test.err, err, test.url)
		}
		if !bytes.Equal(json, test.res) {
			t.Errorf("Expected %s, got %s, for %s", string(test.res), string(json), test.url)
		}

	}
}

func TestParseJSONMetaDataResponse(t *testing.T) {
	tests := []struct {
		testBuffer  []byte
		resultData  *schema.MetaData
		resultError error
	}{
		{
			testBuffer:  []byte(`{"Geo":null,"ASN":null}`),
			resultData:  &schema.MetaData{Geo: nil, ASN: nil},
			resultError: nil,
		},
		{
			testBuffer:  []byte(`"Geo":{},"ASN":{`),
			resultData:  nil,
			resultError: errors.New("Couldn't Parse JSON"),
		},
	}
	for _, test := range tests {
		res, err := p.ParseJSONMetaDataResponse(test.testBuffer)
		// This big mishmash of if statements is simply
		// checking that if one err is nil, that the other is
		// too. Because error messages can vary, this is less
		// brittle than doing just err == test.resultError. If
		// that is okay, then we just use DeepEqual to compare
		// the structs.
		if err == nil && test.resultError != nil ||
			err != nil && test.resultError == nil {
			t.Errorf("Expected %s, got %s for data: %s\n", test.resultError, err, string(test.testBuffer))
		} else if !reflect.DeepEqual(res, test.resultData) {
			t.Errorf("Expected %+v, got %+v, for data %s\n", test.resultData, res, string(test.testBuffer))
		}
	}
}
<commit_msg>I forgot to fix a broken test.<commit_after>package parser_test

import (
	"bytes"
	"errors"
	"fmt"
	"net/http"
	"net/http/httptest"
	"reflect"
	"strings"
	"testing"
	"time"

	"cloud.google.com/go/bigquery"

	p "github.com/m-lab/etl/parser"
	"github.com/m-lab/etl/schema"
)

func TestAddMetaDataNDTConnSpec(t *testing.T) {
	tests := []struct {
		spec      schema.Web100ValueMap
		timestamp time.Time
		url       string
		res       schema.Web100ValueMap
	}{
		{
			spec: func() schema.Web100ValueMap {
				spec := schema.EmptyConnectionSpec()
				spec["client_ip"] = "127.0.0.1"
				spec["server_ip"] = "1.0.0.127"
				return spec
			}(),
			timestamp: time.Now(),
			url:       "/10583?",
			res: func() schema.Web100ValueMap {
				spec := schema.EmptyConnectionSpec()
				spec["client_ip"] = "127.0.0.1"
				spec["server_ip"] = "1.0.0.127"
				geoc := spec.Get("client_geolocation")
				geoc["continent_code"] = ""
				geoc["country_code"] = "US"
				geoc["country_code3"] = "USA"
				geoc["country_name"] = "United States of America"
				geoc["region"] = "NY"
				geoc["metro_code"] = int64(0)
				geoc["city"] = "Scarsdale"
				geoc["area_code"] = int64(10583)
				geoc["postal_code"] = "10583"
				geoc["latitude"] = float64(0.0)
				geoc["longitude"] = float64(0.0)
				geos := spec.Get("server_geolocation")
				geos["continent_code"] = ""
				geos["country_code"] = "US"
				geos["country_code3"] = "USA"
				geos["country_name"] = "United States of America"
				geos["region"] = "NY"
				geos["metro_code"] = int64(0)
				geos["city"] = "Scarsdale"
				geos["area_code"] = int64(10583)
				geos["postal_code"] = "10583"
				geos["latitude"] = float64(0.0)
				geos["longitude"] = float64(0.0)
				return spec
			}(),
		},
	}
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprint(w, `{"Geo":{"continent_code":"","country_code":"US","country_code3":"USA","country_name":"United States of America","region":"NY","metro_code":0,"city":"Scarsdale","area_code":10583,"postal_code":"10583","latitude":0,"longitude":0},"ASN":{}}`)
	}))
	for _, test := range tests {
		p.BaseURL = ts.URL + test.url
		p.AddMetaDataNDTConnSpec(test.spec, test.timestamp)
		if !reflect.DeepEqual(test.spec, test.res) {
			t.Errorf("Expected %+v, got %+v from data %s", test.res, test.spec, test.url)
		}
	}
}

func TestGetAndInsertMetaIntoNDTConnSpec(t *testing.T) {
	tests := []struct {
		side      string
		spec      schema.Web100ValueMap
		timestamp time.Time
		url       string
		res       schema.Web100ValueMap
	}{
		{
			side:      "broken",
			spec:      schema.EmptyConnectionSpec(),
			timestamp: time.Now(),
			url:       "/notUsed?",
			res:       schema.EmptyConnectionSpec(),
		},
		{
			side:      "client",
			spec:      schema.EmptyConnectionSpec(),
			timestamp: time.Now(),
			url:       "portGarbage",
			res:       schema.EmptyConnectionSpec(),
		},
		{
			side: "client",
			spec: func() schema.Web100ValueMap {
				spec := schema.EmptyConnectionSpec()
				spec["client_ip"] = "127.0.0.1"
				return spec
			}(),
			timestamp: time.Now(),
			url:       "/10583?",
			res: func() schema.Web100ValueMap {
				spec := schema.EmptyConnectionSpec()
				spec["client_ip"] = "127.0.0.1"
				geo := spec.Get("client_geolocation")
				geo["continent_code"] = ""
				geo["country_code"] = "US"
				geo["country_code3"] = "USA"
				geo["country_name"] = "United States of America"
				geo["region"] = "NY"
				geo["metro_code"] = int64(0)
				geo["city"] = "Scarsdale"
				geo["area_code"] = int64(10583)
				geo["postal_code"] = "10583"
				geo["latitude"] = float64(0.0)
				geo["longitude"] = float64(0.0)
				return spec
			}(),
		},
	}
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprint(w, `{"Geo":{"continent_code":"","country_code":"US","country_code3":"USA","country_name":"United States of America","region":"NY","metro_code":0,"city":"Scarsdale","area_code":10583,"postal_code":"10583","latitude":0,"longitude":0},"ASN":{}}`)
	}))
	for _, test := range tests {
		p.BaseURL = ts.URL + test.url
		p.GetAndInsertMetaIntoNDTConnSpec(test.side, test.spec, test.timestamp)
		if !reflect.DeepEqual(test.spec, test.res) {
			t.Errorf("Expected %+v, got %+v from data %s", test.res, test.spec, test.url)
		}
	}

}

func TestCopyStructToMap(t *testing.T) {
	tests := []struct {
		source interface{}
		dest   map[string]bigquery.Value
		res    map[string]bigquery.Value
	}{
		{
			source: &struct {
				A   int
				Bee string
			}{A: 1, Bee: "2"},
			dest: make(map[string]bigquery.Value),
			res:  map[string]bigquery.Value{"a": 1, "bee": "2"},
		},
		{
			source: &struct{}{},
			dest:   make(map[string]bigquery.Value),
			res:    map[string]bigquery.Value{},
		},
	}
	for _, test := range tests {
		p.CopyStructToMap(test.source, test.dest)
		if !reflect.DeepEqual(test.dest, test.res) {
			t.Errorf("Expected %+v, got %+v for data: %+v\n", test.res, test.dest, test.source)
		}
	}

}

func TestGetMetaData(t *testing.T) {
	tests := []struct {
		url string
		res *schema.MetaData
	}{
		{
			url: "portGarbage",
			res: nil,
		},
		{
			url: "/badJson",
			res: nil,
		},
		{
			url: "/goodJson",
			res: &schema.MetaData{},
		},
	}
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		path := r.URL.Path
		if strings.HasSuffix(path, "goodJson") {
			fmt.Fprint(w, `{"Geo":null,"ASN":null}`)
			return
		}
		fmt.Fprint(w, "{jngngfsljngsljngfsljngsljn")
	}))
	for _, test := range tests {
		res := p.GetMetaData(ts.URL + test.url)
		if res != test.res && *res != *test.res {
			t.Errorf("Expected %+v, got %+v for data: %s\n", test.res, res, test.url)
		}

	}

}

func TestQueryAnnotationService(t *testing.T) {
	tests := []struct {
		url string
		res []byte
		err error
	}{
		{
			url: "portGarbage",
			res: nil,
			err: errors.New("HTTP Protocol Error"),
		},
		{
			url: "/error",
			res: nil,
			err: errors.New("HTTP 404 Error"),
		},
		{
			url: "/Echo",
			res: []byte("Echo"),
			err: nil,
		},
	}
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		path := r.URL.Path
		if strings.HasSuffix(path, "error") {
			http.Error(w, "Test Error", 404)
			return
		}
		fmt.Fprint(w, "Echo")
	}))
	for _, test := range tests {
		json, err := p.QueryAnnotationService(ts.URL + test.url)
		if err != nil && test.err == nil || err == nil && test.err != nil {
			t.Errorf("Expected %s, got %s, for %s", test.err, err, test.url)
		}
		if !bytes.Equal(json, test.res) {
			t.Errorf("Expected %s, got %s, for %s", string(test.res), string(json), test.url)
		}

	}
}

func TestParseJSONMetaDataResponse(t *testing.T) {
	tests := []struct {
		testBuffer  []byte
		resultData  *schema.MetaData
		resultError error
	}{
		{
			testBuffer:  []byte(`{"Geo":null,"ASN":null}`),
			resultData:  &schema.MetaData{Geo: nil, ASN: nil},
			resultError: nil,
		},
		{
			testBuffer:  []byte(`"Geo":{},"ASN":{`),
			resultData:  nil,
			resultError: errors.New("Couldn't Parse JSON"),
		},
	}
	for _, test := range tests {
		res, err := p.ParseJSONMetaDataResponse(test.testBuffer)
		// This big mishmash of if statements is simply
		// checking that if one err is nil, that the other is
		// too. Because error messages can vary, this is less
		// brittle than doing just err == test.resultError. If
		// that is okay, then we just use DeepEqual to compare
		// the structs.
		if err == nil && test.resultError != nil ||
			err != nil && test.resultError == nil {
			t.Errorf("Expected %s, got %s for data: %s\n", test.resultError, err, string(test.testBuffer))
		} else if !reflect.DeepEqual(res, test.resultData) {
			t.Errorf("Expected %+v, got %+v, for data %s\n", test.resultData, res, string(test.testBuffer))
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/ian-kent/gotcha/http"
	"strings"
	"sync"
	"time"
	"github.com/ian-kent/gopan/gopan"
	"sort"
	"html/template"
)

type SearchResult struct {
	Name string
	Type string
	URL string
	Obj interface{}
	Glyph string
}

type ByName []*SearchResult
func (a ByName) Len() int { return len(a) }
func (a ByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a ByName) Less(i, j int) bool { return a[i].Name < a[j].Name }

func search(session *http.Session) {
	if q, ok := session.Request.Form()["q"]; ok {
		query := strings.ToLower(q[0])
		session.Stash["Query"] = q[0]
		results := make([]*SearchResult, 0)
		var lock sync.Mutex

		tStart := time.Now().UnixNano()

		var wg sync.WaitGroup

		for _, idx := range indexes {
			wg.Add(1)
			go func(idx *gopan.Source) {
				defer wg.Done()

				if strings.Contains(strings.ToLower(idx.Name), query) {
					lock.Lock()
					results = append(results, &SearchResult{
						Name: idx.Name,
						Type: "Index",
						URL: idx.Name,
						Obj: idx,
						Glyph: "list",
					})
					lock.Unlock()
				}

				for _, auth := range idx.Authors {
					wg.Add(1)
					go func(idx *gopan.Source, auth *gopan.Author) {
						defer wg.Done()

						if strings.Contains(strings.ToLower(auth.Name), query) {
							lock.Lock()
							results = append(results, &SearchResult{
								Name: auth.Name,
								Type: "Author",
								URL: idx.Name + "/authors/id/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name,
								Obj: auth,
								Glyph: "user",
							})
							lock.Unlock()
						}
						for _, pkg := range auth.Packages {
							if strings.Contains(strings.ToLower(pkg.Name), query) {
								lock.Lock()
								results = append(results, &SearchResult{
									Name: pkg.Name,
									Type: "Module",
									URL: idx.Name + "/authors/id/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name + "/" + pkg.Name,
									Obj: pkg,
									Glyph: "compressed",
								})
								lock.Unlock()
							}
							for _, prov := range pkg.Provides {
								if strings.Contains(strings.ToLower(prov.Name), query) {
									lock.Lock()
									results = append(results, &SearchResult{
										Name: prov.Name,
										Type: "Package",
										URL: idx.Name + "/authors/id/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name + "/" + pkg.Name,
										Obj: prov,
										Glyph: "briefcase",
									})
									lock.Unlock()
								}
							}
						}
					}(idx, auth)
				}
			}(idx)
		}

		wg.Wait()

		t := float64(time.Now().UnixNano()-tStart) / 100000 // ms

		sort.Sort(ByName(results))

		session.Stash["Results"] = results
		session.Stash["Duration"] = t
	}

	session.Stash["Title"] = "SmartPAN"
	html, _ := session.RenderTemplate("search.html")

	session.Stash["Page"] = "Search"
	session.Stash["Content"] = template.HTML(html)
	session.Render("layout.html")
}
<commit_msg>Fix #2, packages now point to modules index<commit_after>package main

import (
	"github.com/ian-kent/gotcha/http"
	"strings"
	"sync"
	"time"
	"github.com/ian-kent/gopan/gopan"
	"sort"
	"html/template"
)

type SearchResult struct {
	Name string
	Type string
	URL string
	Obj interface{}
	Glyph string
}

type ByName []*SearchResult
func (a ByName) Len() int { return len(a) }
func (a ByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a ByName) Less(i, j int) bool { return a[i].Name < a[j].Name }

func search(session *http.Session) {
	if q, ok := session.Request.Form()["q"]; ok {
		query := strings.ToLower(q[0])
		session.Stash["Query"] = q[0]
		results := make([]*SearchResult, 0)
		var lock sync.Mutex

		tStart := time.Now().UnixNano()

		var wg sync.WaitGroup

		for _, idx := range indexes {
			wg.Add(1)
			go func(idx *gopan.Source) {
				defer wg.Done()

				if strings.Contains(strings.ToLower(idx.Name), query) {
					lock.Lock()
					results = append(results, &SearchResult{
						Name: idx.Name,
						Type: "Index",
						URL: idx.Name,
						Obj: idx,
						Glyph: "list",
					})
					lock.Unlock()
				}

				for _, auth := range idx.Authors {
					wg.Add(1)
					go func(idx *gopan.Source, auth *gopan.Author) {
						defer wg.Done()

						if strings.Contains(strings.ToLower(auth.Name), query) {
							lock.Lock()
							results = append(results, &SearchResult{
								Name: auth.Name,
								Type: "Author",
								URL: idx.Name + "/authors/id/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name,
								Obj: auth,
								Glyph: "user",
							})
							lock.Unlock()
						}
						for _, pkg := range auth.Packages {
							if strings.Contains(strings.ToLower(pkg.Name), query) {
								lock.Lock()
								results = append(results, &SearchResult{
									Name: pkg.Name,
									Type: "Module",
									URL: idx.Name + "/authors/id/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name + "/" + pkg.Name,
									Obj: pkg,
									Glyph: "compressed",
								})
								lock.Unlock()
							}
							for _, prov := range pkg.Provides {
								if strings.Contains(strings.ToLower(prov.Name), query) {
									lock.Lock()
									results = append(results, &SearchResult{
										Name: prov.Name,
										Type: "Package",
										URL: idx.Name + "/modules/" + strings.Replace(prov.Name, "::", "/", -1),
										Obj: prov,
										Glyph: "briefcase",
									})
									lock.Unlock()
								}
							}
						}
					}(idx, auth)
				}
			}(idx)
		}

		wg.Wait()

		t := float64(time.Now().UnixNano()-tStart) / 100000 // ms

		sort.Sort(ByName(results))

		session.Stash["Results"] = results
		session.Stash["Duration"] = t
	}

	session.Stash["Title"] = "SmartPAN"
	html, _ := session.RenderTemplate("search.html")

	session.Stash["Page"] = "Search"
	session.Stash["Content"] = template.HTML(html)
	session.Render("layout.html")
}
<|endoftext|>