<commit_before>// Copyright (c) 2013-2016 The btcsuite developers
// Copyright (c) 2015-2016 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package blockchain

import (
	"fmt"

	"github.com/decred/dcrd/blockchain/stake"
	"github.com/decred/dcrd/chaincfg/chainhash"
	"github.com/decred/dcrd/database"
)

// nodeAtHeightFromTopNode goes backwards through a node until it a reaches
// the node with a desired block height; it returns this block.  The benefit is
// this works for both the main chain and the side chain.
func (b *BlockChain) nodeAtHeightFromTopNode(node *blockNode,
	toTraverse int64) (*blockNode, error) {
	oldNode := node
	var err error

	for i := 0; i < int(toTraverse); i++ {
		// Get the previous block node.
		oldNode, err = b.getPrevNodeFromNode(oldNode)
		if err != nil {
			return nil, err
		}

		if oldNode == nil {
			return nil, fmt.Errorf("unable to obtain previous node; " +
				"ancestor is genesis block")
		}
	}

	return oldNode, nil
}

// fetchNewTicketsForNode fetches the list of newly maturing tickets for a
// given node by traversing backwards through its parents until it finds the
// block that contains the original tickets to mature.
//
// This function is NOT safe for concurrent access and must be called with
// the chainLock held for writes.
func (b *BlockChain) fetchNewTicketsForNode(node *blockNode) ([]chainhash.Hash, error) {
	// If we're before the stake enabled height, there can be no
	// tickets in the live ticket pool.
	if node.height < b.chainParams.StakeEnabledHeight {
		return []chainhash.Hash{}, nil
	}

	// If we already cached the tickets, simply return the cached list.
	// It's important to make the distinction here that nil means the
	// value was never looked up, while an empty slice of pointers means
	// that there were no new tickets at this height.
	if node.newTickets != nil {
		return node.newTickets, nil
	}

	// Calculate block number for where new tickets matured from and retrieve
	// this block from DB or in memory if it's a sidechain.
	matureNode, err := b.nodeAtHeightFromTopNode(node,
		int64(b.chainParams.TicketMaturity))
	if err != nil {
		return nil, err
	}

	matureBlock, errBlock := b.fetchBlockFromHash(&matureNode.hash)
	if errBlock != nil {
		return nil, errBlock
	}

	tickets := []chainhash.Hash{}
	for _, stx := range matureBlock.STransactions() {
		if is, _ := stake.IsSStx(stx); is {
			h := stx.Sha()
			tickets = append(tickets, *h)
		}
	}

	// Set the new tickets in memory so that they exist for future
	// reference in the node.
	node.newTickets = tickets

	return tickets, nil
}

// fetchStakeNode will scour the blockchain from the best block, for which we
// know that there is valid stake node.  The first step is finding a path to the
// ancestor, or, if on a side chain, the path to the common ancestor, followed
// by the path to the sidechain node.  After this path is established, the
// algorithm walks along the path, regenerating and storing intermediate nodes
// as it does so, until the final stake node of interest is populated with the
// correct data.
//
// This function MUST be called with the chain state lock held (for writes).
func (b *BlockChain) fetchStakeNode(node *blockNode) (*stake.Node, error) {
	// If we already have the stake node fetched, returned the cached result.
	// Stake nodes are immutable.
	if node.stakeNode != nil {
		return node.stakeNode, nil
	}

	// If the parent stake node is cached, connect the stake node
	// from there.
	if node.parent != nil {
		if node.stakeNode == nil && node.parent.stakeNode != nil {
			var err error
			if node.newTickets == nil {
				node.newTickets, err = b.fetchNewTicketsForNode(node)
				if err != nil {
					return nil, err
				}
			}

			node.stakeNode, err = node.parent.stakeNode.ConnectNode(node.header,
				node.ticketsSpent,
				node.ticketsRevoked,
				node.newTickets)
			if err != nil {
				return nil, err
			}

			return node.stakeNode, nil
		}
	}

	// We need to generate a path to the stake node and restore it
	// it through the entire path.  The bestNode stake node must
	// always be filled in, so assume it is safe to begin working
	// backwards from there.
	detachNodes, attachNodes, err := b.getReorganizeNodes(node)
	if err != nil {
		return nil, err
	}
	current := b.bestNode

	// Move backwards through the main chain, undoing the ticket
	// treaps for each block.  The database is passed because the
	// undo data and new tickets data for each block may not yet
	// be filled in and may require the database to look up.
	err = b.db.View(func(dbTx database.Tx) error {
		for e := detachNodes.Front(); e != nil; e = e.Next() {
			n := e.Value.(*blockNode)
			var errLocal error
			if n.stakeNode == nil {
				n.stakeNode, errLocal =
					current.stakeNode.DisconnectNode(n.header,
						n.stakeUndoData, n.newTickets, dbTx)
			}
			if errLocal != nil {
				return errLocal
			}
			current = n
		}

		return nil
	})
	if err != nil {
		return nil, err
	}

	// Detach the final block and get the filled in node for the fork
	// point.
	err = b.db.View(func(dbTx database.Tx) error {
		var errLocal error
		if current.parent.stakeNode == nil {
			current.parent.stakeNode, errLocal =
				current.stakeNode.DisconnectNode(current.header,
					current.stakeUndoData, current.newTickets, dbTx)
		}
		if errLocal != nil {
			return errLocal
		}

		current = current.parent
		return nil
	})
	if err != nil {
		return nil, err
	}

	// The node is at a fork point in the block chain, so just return
	// this stake node.
	if attachNodes.Len() == 0 {
		if current.hash != node.hash ||
			current.height != node.height {
			return nil, AssertError("failed to restore stake node to " +
				"fork point when fetching")
		}

		return current.stakeNode, nil
	}

	// The requested node is on a side chain, so we need to apply the
	// transactions and spend information from each of the nodes to attach.
	// Not that side chain ticket data and undo data is always stored
	// in memory, so there is not need to use the database here.
	for e := attachNodes.Front(); e != nil; e = e.Next() {
		n := e.Value.(*blockNode)

		if n.stakeNode == nil {
			if n.newTickets == nil {
				n.newTickets, err = b.fetchNewTicketsForNode(n)
				if err != nil {
					return nil, err
				}
			}

			n.stakeNode, err = current.stakeNode.ConnectNode(n.header,
				n.ticketsSpent, n.ticketsRevoked, n.newTickets)
			if err != nil {
				return nil, err
			}
		}

		current = n
	}

	return current.stakeNode, nil
}
<commit_msg>Fix a bug reloading the blockchain<commit_after>// Copyright (c) 2013-2016 The btcsuite developers
// Copyright (c) 2015-2016 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package blockchain

import (
	"fmt"

	"github.com/decred/dcrd/blockchain/stake"
	"github.com/decred/dcrd/chaincfg/chainhash"
	"github.com/decred/dcrd/database"
)

// nodeAtHeightFromTopNode goes backwards through a node until it a reaches
// the node with a desired block height; it returns this block.  The benefit is
// this works for both the main chain and the side chain.
func (b *BlockChain) nodeAtHeightFromTopNode(node *blockNode,
	toTraverse int64) (*blockNode, error) {
	oldNode := node
	var err error

	for i := 0; i < int(toTraverse); i++ {
		// Get the previous block node.
		oldNode, err = b.getPrevNodeFromNode(oldNode)
		if err != nil {
			return nil, err
		}

		if oldNode == nil {
			return nil, fmt.Errorf("unable to obtain previous node; " +
				"ancestor is genesis block")
		}
	}

	return oldNode, nil
}

// fetchNewTicketsForNode fetches the list of newly maturing tickets for a
// given node by traversing backwards through its parents until it finds the
// block that contains the original tickets to mature.
//
// This function is NOT safe for concurrent access and must be called with
// the chainLock held for writes.
func (b *BlockChain) fetchNewTicketsForNode(node *blockNode) ([]chainhash.Hash, error) {
	// If we're before the stake enabled height, there can be no
	// tickets in the live ticket pool.
	if node.height < b.chainParams.StakeEnabledHeight {
		return []chainhash.Hash{}, nil
	}

	// If we already cached the tickets, simply return the cached list.
	// It's important to make the distinction here that nil means the
	// value was never looked up, while an empty slice of pointers means
	// that there were no new tickets at this height.
	if node.newTickets != nil {
		return node.newTickets, nil
	}

	// Calculate block number for where new tickets matured from and retrieve
	// this block from DB or in memory if it's a sidechain.
	matureNode, err := b.nodeAtHeightFromTopNode(node,
		int64(b.chainParams.TicketMaturity))
	if err != nil {
		return nil, err
	}

	matureBlock, errBlock := b.fetchBlockFromHash(&matureNode.hash)
	if errBlock != nil {
		return nil, errBlock
	}

	tickets := []chainhash.Hash{}
	for _, stx := range matureBlock.STransactions() {
		if is, _ := stake.IsSStx(stx); is {
			h := stx.Sha()
			tickets = append(tickets, *h)
		}
	}

	// Set the new tickets in memory so that they exist for future
	// reference in the node.
	node.newTickets = tickets

	return tickets, nil
}

// fetchStakeNode will scour the blockchain from the best block, for which we
// know that there is valid stake node.  The first step is finding a path to the
// ancestor, or, if on a side chain, the path to the common ancestor, followed
// by the path to the sidechain node.  After this path is established, the
// algorithm walks along the path, regenerating and storing intermediate nodes
// as it does so, until the final stake node of interest is populated with the
// correct data.
//
// This function MUST be called with the chain state lock held (for writes).
func (b *BlockChain) fetchStakeNode(node *blockNode) (*stake.Node, error) {
	// If we already have the stake node fetched, returned the cached result.
	// Stake nodes are immutable.
	if node.stakeNode != nil {
		return node.stakeNode, nil
	}

	// If the parent stake node is cached, connect the stake node
	// from there.
	if node.parent != nil {
		if node.stakeNode == nil && node.parent.stakeNode != nil {
			var err error
			if node.newTickets == nil {
				node.newTickets, err = b.fetchNewTicketsForNode(node)
				if err != nil {
					return nil, err
				}
			}

			node.stakeNode, err = node.parent.stakeNode.ConnectNode(node.header,
				node.ticketsSpent,
				node.ticketsRevoked,
				node.newTickets)
			if err != nil {
				return nil, err
			}

			return node.stakeNode, nil
		}
	}

	// We need to generate a path to the stake node and restore it
	// it through the entire path.  The bestNode stake node must
	// always be filled in, so assume it is safe to begin working
	// backwards from there.
	detachNodes, attachNodes, err := b.getReorganizeNodes(node)
	if err != nil {
		return nil, err
	}
	current := b.bestNode

	// Move backwards through the main chain, undoing the ticket
	// treaps for each block.  The database is passed because the
	// undo data and new tickets data for each block may not yet
	// be filled in and may require the database to look up.
	err = b.db.View(func(dbTx database.Tx) error {
		for e := detachNodes.Front(); e != nil; e = e.Next() {
			n := e.Value.(*blockNode)
			var errLocal error
			if n.stakeNode == nil {
				n.stakeNode, errLocal =
					current.stakeNode.DisconnectNode(n.header,
						n.stakeUndoData, n.newTickets, dbTx)
			}
			if errLocal != nil {
				return errLocal
			}
			current = n
		}

		return nil
	})
	if err != nil {
		return nil, err
	}

	// Detach the final block and get the filled in node for the fork
	// point.
	err = b.db.View(func(dbTx database.Tx) error {
		var errLocal error
		if current.parent.stakeNode == nil {
			current.parent.stakeNode, errLocal =
				current.stakeNode.DisconnectNode(current.parent.header,
					current.parent.stakeUndoData, current.parent.newTickets, dbTx)
		}
		if errLocal != nil {
			return errLocal
		}

		current = current.parent
		return nil
	})
	if err != nil {
		return nil, err
	}

	// The node is at a fork point in the block chain, so just return
	// this stake node.
	if attachNodes.Len() == 0 {
		if current.hash != node.hash ||
			current.height != node.height {
			return nil, AssertError("failed to restore stake node to " +
				"fork point when fetching")
		}

		return current.stakeNode, nil
	}

	// The requested node is on a side chain, so we need to apply the
	// transactions and spend information from each of the nodes to attach.
	// Not that side chain ticket data and undo data is always stored
	// in memory, so there is not need to use the database here.
	for e := attachNodes.Front(); e != nil; e = e.Next() {
		n := e.Value.(*blockNode)

		if n.stakeNode == nil {
			if n.newTickets == nil {
				n.newTickets, err = b.fetchNewTicketsForNode(n)
				if err != nil {
					return nil, err
				}
			}

			n.stakeNode, err = current.stakeNode.ConnectNode(n.header,
				n.ticketsSpent, n.ticketsRevoked, n.newTickets)
			if err != nil {
				return nil, err
			}
		}

		current = n
	}

	return current.stakeNode, nil
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The code is the original Go implementation of rand.Prime optimized for
// generating safe (Sophie Germain) primes.
// A safe prime is a prime number of the form 2p + 1, where p is also a prime.

package paillier

import (
	"context"
	"errors"
	"fmt"
	"io"
	"math/big"
	"sync"
	"time"
)

// smallPrimes is a list of small, prime numbers that allows us to rapidly
// exclude some fraction of composite candidates when searching for a random
// prime. This list is truncated at the point where smallPrimesProduct exceeds
// a uint64. It does not include two because we ensure that the candidates are
// odd by construction.
var smallPrimes = []uint8{
	3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53,
}

// smallPrimesProduct is the product of the values in smallPrimes and allows us
// to reduce a candidate prime by this number and then determine whether it's
// coprime to all the elements of smallPrimes without further big.Int
// operations.
var smallPrimesProduct = new(big.Int).SetUint64(16294579238595022365)

// GenerateSafePrime tries to find a safe prime concurrently.
// The returned result is a safe prime `p` and prime `q` such that `p=2q+1`.
// Concurrency level can be controlled with the `concurrencyLevel` parameter.
// If a safe prime could not be found in the specified `timeout`, the error
// is returned. Also, if at least one search process failed, error is returned
// as well.
//
// How fast we generate a prime number is mostly a matter of luck and it depends
// on how lucky we are with drawing the first bytes.
// With today's multicore processors, we can execute the process on multiple
// cores concurrently, accept the first valid result and cancel the rest of
// work. This way, with the same finding algorithm, we can get the result
// faster.
//
// Concurrency level should be set depending on what `bitLen` of prime is
// expected. For example, as of today, on a typical workstation, for 512-bit
// safe prime, `concurrencyLevel` should be set to `1` as generating the prime
// of this length is a matter of milliseconds for a single core.
// For 1024-bit safe prime, `concurrencyLevel` should be usually set to at least
// `2` and for 2048-bit safe prime, `concurrencyLevel` must be set to at least
// `4` to get the result in a reasonable time.
//
// This function generates safe primes of at least 6 `bitLen`. For every
// generated safe prime, the two most significant bits are always set to `1`
// - we don't want the generated number to be too small.
func GenerateSafePrime(
	bitLen int,
	concurrencyLevel int,
	timeout time.Duration,
	random io.Reader,
) (*big.Int, *big.Int, error) {
	if bitLen < 6 {
		return nil, nil, errors.New("safe prime size must be at least 6 bits")
	}

	primeChan := make(chan safePrime, concurrencyLevel)
	errChan := make(chan error, concurrencyLevel)

	waitGroup := &sync.WaitGroup{}

	defer close(primeChan)
	defer close(errChan)
	defer waitGroup.Wait()

	ctx, cancel := context.WithCancel(context.Background())

	for i := 0; i < concurrencyLevel; i++ {
		waitGroup.Add(1)
		runGenPrimeRoutine(
			ctx, primeChan, errChan, waitGroup, random, bitLen,
		)
	}

	// Cancel after the specified timeout.
	go func() {
		time.Sleep(timeout)
		cancel()
	}()

	select {
	case result := <-primeChan:
		cancel()
		return result.p, result.q, nil
	case err := <-errChan:
		cancel()
		return nil, nil, err
	case <-ctx.Done():
		return nil, nil, fmt.Errorf("generator timed out after %v", timeout)
	}
}

type safePrime struct {
	p *big.Int // p = 2q + 1
	q *big.Int
}

// Starts a Goroutine searching for a safe prime of the specified `pBitLen`.
// If succeeds, writes prime `p` and prime `q` such that `p = 2q+1` to the
// `primeChan`. Prime `p` has a bit length equal to `pBitLen` and prime `q` has
// a bit length equal to `pBitLen-1`.
//
// The algorithm is as follows:
// 1. Generate a random odd number `q` of length `pBitLen-1` with two the most
//    significant bits set to `1`.
// 2. Execute preliminary primality test on `q` checking whether it is coprime
//    to all the elements of `smallPrimes`. It allows to eliminate trivial
//    cases quickly, when `q` is obviously no prime, without running an
//    expensive final primality tests.
//    If `q` is coprime to all of the `smallPrimes`, then go to the point 3.
//    If not, add `2` and try again. Do it at most 10 times.
// 3. Check the potentially prime `q`, whether `q = 1 (mod 3)`. This will
//    happen for 50% of cases.
//    If it is, then `p = 2q+1` will be a multiple of 3, so it will be obviously
//    not a prime number. In this case, add `2` and try again. Do it at most 10
//    times. If `q != 1 (mod 3)`, go to the point 4.
// 4. Now we know `q` is potentially prime and `p = 2q+1` is not a multiple of
//    3. We execute a preliminary primality test on `p`, checking whether
//    it is coprime to all the elements of `smallPrimes` just like we did for
//    `q` in point 2. If `p` is not coprime to at least one element of the
//    `smallPrimes`, then go back to point 1.
//    If `p` is coprime to all the elements of `smallPrimes`, go to point 5.
// 5. At this point, we know `q` is potentially prime, and `p=q+1` is also
//    potentially prime. We need to execute a final primality test for `q`.
//    We apply Miller-Rabin and Baillie-PSW tests. If they succeed, it means
//    that `q` is prime with a very high probability. Knowing `q` is prime,
//    we use Pocklington's criterion to prove the primality of `p=2q+1`, that
//    is, we execute Fermat primality test to base 2 checking whether
//    `2^{p-1} = 1 (mod p)`. It's significantly faster than running full
//    Miller-Rabin and Baillie-PSW for `p`.
//    If `q` and `p` are found to be prime, return them as a result. If not, go
//    back to the point 1.
func runGenPrimeRoutine(
	ctx context.Context,
	primeChan chan safePrime,
	errChan chan error,
	waitGroup *sync.WaitGroup,
	rand io.Reader,
	pBitLen int,
) {
	qBitLen := pBitLen - 1
	b := uint(qBitLen % 8)
	if b == 0 {
		b = 8
	}

	bytes := make([]byte, (qBitLen+7)/8)
	p := new(big.Int)
	q := new(big.Int)

	bigMod := new(big.Int)

	go func() {
		defer waitGroup.Done()

		for {
			select {
			case <-ctx.Done():
				return
			default:
				_, err := io.ReadFull(rand, bytes)
				if err != nil {
					errChan <- err
					return
				}

				// Clear bits in the first byte to make sure the candidate has
				// a size <= bits.
				bytes[0] &= uint8(int(1<<b) - 1)
				// Don't let the value be too small, i.e, set the most
				// significant two bits.
				// Setting the top two bits, rather than just the top bit,
				// means that when two of these values are multiplied together,
				// the result isn't ever one bit short.
				if b >= 2 {
					bytes[0] |= 3 << (b - 2)
				} else {
					// Here b==1, because b cannot be zero.
					bytes[0] |= 1
					if len(bytes) > 1 {
						bytes[1] |= 0x80
					}
				}
				// Make the value odd since an even number this large certainly
				// isn't prime.
				bytes[len(bytes)-1] |= 1

				q.SetBytes(bytes)

				// Calculate the value mod the product of smallPrimes. If it's
				// a multiple of any of these primes we add two until it isn't.
				// The probability of overflowing is minimal and can be ignored
				// because we still perform Miller-Rabin tests on the result.
				bigMod.Mod(q, smallPrimesProduct)
				mod := bigMod.Uint64()

			NextDelta:
				for delta := uint64(0); delta < 1<<20; delta += 2 {
					m := mod + delta
					for _, prime := range smallPrimes {
						if m%uint64(prime) == 0 && (qBitLen > 6 || m != uint64(prime)) {
							continue NextDelta
						}
					}

					if delta > 0 {
						bigMod.SetUint64(delta)
						q.Add(q, bigMod)
					}

					// If `q = 1 (mod 3)`, then `p` is a multiple of `3` so it's
					// obviously no prime and such `q` should be rejected.
					// This will happen in 50% of cases and we should detect
					// and eliminate them early.
					//
					// Explanation:
					// If q = 1 (mod 3) then there exists a q' such that:
					// q = 3q' + 1
					//
					// Since p = 2q + 1:
					// p = 2q + 1 = 2(3q' + 1) + 1 = 6q' + 2 + 1 = 6q' + 3 =
					//   = 3(2q' + 1)
					// So `p` is a multiple of `3`.
					qMod3 := new(big.Int).Mod(q, big.NewInt(3))
					if qMod3.Cmp(big.NewInt(1)) == 0 {
						continue NextDelta
					}

					// p = 2q+1
					p.Mul(q, big.NewInt(2))
					p.Add(p, big.NewInt(1))
					if !isPrimeCandidate(p) {
						continue NextDelta
					}

					break
				}

				// There is a tiny possibility that, by adding delta, we caused
				// the number to be one bit too long. Thus we check BitLen
				// here.
				if q.ProbablyPrime(20) &&
					isPocklingtonCriterionSatisfied(p) &&
					q.BitLen() == qBitLen {

					primeChan <- safePrime{p, q}
					return
				}
			}
		}
	}()
}

func isPocklingtonCriterionSatisfied(p *big.Int) bool {
	return new(big.Int).Exp(
		big.NewInt(2),
		new(big.Int).Sub(p, big.NewInt(1)),
		p,
	).Cmp(big.NewInt(1)) == 0
}

func isPrimeCandidate(number *big.Int) bool {
	m := new(big.Int).Mod(number, smallPrimesProduct).Uint64()

	for _, prime := range smallPrimes {
		if m%uint64(prime) == 0 && m != uint64(prime) {
			return false
		}
	}

	return true
}
<commit_msg>Added documentation to isPoclingtonCriterionSatisfied<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The code is the original Go implementation of rand.Prime optimized for
// generating safe (Sophie Germain) primes.
// A safe prime is a prime number of the form 2p + 1, where p is also a prime.

package paillier

import (
	"context"
	"errors"
	"fmt"
	"io"
	"math/big"
	"sync"
	"time"
)

// smallPrimes is a list of small, prime numbers that allows us to rapidly
// exclude some fraction of composite candidates when searching for a random
// prime. This list is truncated at the point where smallPrimesProduct exceeds
// a uint64. It does not include two because we ensure that the candidates are
// odd by construction.
var smallPrimes = []uint8{
	3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53,
}

// smallPrimesProduct is the product of the values in smallPrimes and allows us
// to reduce a candidate prime by this number and then determine whether it's
// coprime to all the elements of smallPrimes without further big.Int
// operations.
var smallPrimesProduct = new(big.Int).SetUint64(16294579238595022365)

// GenerateSafePrime tries to find a safe prime concurrently.
// The returned result is a safe prime `p` and prime `q` such that `p=2q+1`.
// Concurrency level can be controlled with the `concurrencyLevel` parameter.
// If a safe prime could not be found in the specified `timeout`, the error
// is returned. Also, if at least one search process failed, error is returned
// as well.
//
// How fast we generate a prime number is mostly a matter of luck and it depends
// on how lucky we are with drawing the first bytes.
// With today's multicore processors, we can execute the process on multiple
// cores concurrently, accept the first valid result and cancel the rest of
// work. This way, with the same finding algorithm, we can get the result
// faster.
//
// Concurrency level should be set depending on what `bitLen` of prime is
// expected. For example, as of today, on a typical workstation, for 512-bit
// safe prime, `concurrencyLevel` should be set to `1` as generating the prime
// of this length is a matter of milliseconds for a single core.
// For 1024-bit safe prime, `concurrencyLevel` should be usually set to at least
// `2` and for 2048-bit safe prime, `concurrencyLevel` must be set to at least
// `4` to get the result in a reasonable time.
//
// This function generates safe primes of at least 6 `bitLen`. For every
// generated safe prime, the two most significant bits are always set to `1`
// - we don't want the generated number to be too small.
func GenerateSafePrime(
	bitLen int,
	concurrencyLevel int,
	timeout time.Duration,
	random io.Reader,
) (*big.Int, *big.Int, error) {
	if bitLen < 6 {
		return nil, nil, errors.New("safe prime size must be at least 6 bits")
	}

	primeChan := make(chan safePrime, concurrencyLevel)
	errChan := make(chan error, concurrencyLevel)

	waitGroup := &sync.WaitGroup{}

	defer close(primeChan)
	defer close(errChan)
	defer waitGroup.Wait()

	ctx, cancel := context.WithCancel(context.Background())

	for i := 0; i < concurrencyLevel; i++ {
		waitGroup.Add(1)
		runGenPrimeRoutine(
			ctx, primeChan, errChan, waitGroup, random, bitLen,
		)
	}

	// Cancel after the specified timeout.
	go func() {
		time.Sleep(timeout)
		cancel()
	}()

	select {
	case result := <-primeChan:
		cancel()
		return result.p, result.q, nil
	case err := <-errChan:
		cancel()
		return nil, nil, err
	case <-ctx.Done():
		return nil, nil, fmt.Errorf("generator timed out after %v", timeout)
	}
}

type safePrime struct {
	p *big.Int // p = 2q + 1
	q *big.Int
}

// Starts a Goroutine searching for a safe prime of the specified `pBitLen`.
// If succeeds, writes prime `p` and prime `q` such that `p = 2q+1` to the
// `primeChan`. Prime `p` has a bit length equal to `pBitLen` and prime `q` has
// a bit length equal to `pBitLen-1`.
//
// The algorithm is as follows:
// 1. Generate a random odd number `q` of length `pBitLen-1` with two the most
//    significant bits set to `1`.
// 2. Execute preliminary primality test on `q` checking whether it is coprime
//    to all the elements of `smallPrimes`. It allows to eliminate trivial
//    cases quickly, when `q` is obviously no prime, without running an
//    expensive final primality tests.
//    If `q` is coprime to all of the `smallPrimes`, then go to the point 3.
//    If not, add `2` and try again. Do it at most 10 times.
// 3. Check the potentially prime `q`, whether `q = 1 (mod 3)`. This will
//    happen for 50% of cases.
//    If it is, then `p = 2q+1` will be a multiple of 3, so it will be obviously
//    not a prime number. In this case, add `2` and try again. Do it at most 10
//    times. If `q != 1 (mod 3)`, go to the point 4.
// 4. Now we know `q` is potentially prime and `p = 2q+1` is not a multiple of
//    3. We execute a preliminary primality test on `p`, checking whether
//    it is coprime to all the elements of `smallPrimes` just like we did for
//    `q` in point 2. If `p` is not coprime to at least one element of the
//    `smallPrimes`, then go back to point 1.
//    If `p` is coprime to all the elements of `smallPrimes`, go to point 5.
// 5. At this point, we know `q` is potentially prime, and `p=q+1` is also
//    potentially prime. We need to execute a final primality test for `q`.
//    We apply Miller-Rabin and Baillie-PSW tests. If they succeed, it means
//    that `q` is prime with a very high probability. Knowing `q` is prime,
//    we use Pocklington's criterion to prove the primality of `p=2q+1`, that
//    is, we execute Fermat primality test to base 2 checking whether
//    `2^{p-1} = 1 (mod p)`. It's significantly faster than running full
//    Miller-Rabin and Baillie-PSW for `p`.
//    If `q` and `p` are found to be prime, return them as a result. If not, go
//    back to the point 1.
func runGenPrimeRoutine(
	ctx context.Context,
	primeChan chan safePrime,
	errChan chan error,
	waitGroup *sync.WaitGroup,
	rand io.Reader,
	pBitLen int,
) {
	qBitLen := pBitLen - 1
	b := uint(qBitLen % 8)
	if b == 0 {
		b = 8
	}

	bytes := make([]byte, (qBitLen+7)/8)
	p := new(big.Int)
	q := new(big.Int)

	bigMod := new(big.Int)

	go func() {
		defer waitGroup.Done()

		for {
			select {
			case <-ctx.Done():
				return
			default:
				_, err := io.ReadFull(rand, bytes)
				if err != nil {
					errChan <- err
					return
				}

				// Clear bits in the first byte to make sure the candidate has
				// a size <= bits.
				bytes[0] &= uint8(int(1<<b) - 1)
				// Don't let the value be too small, i.e, set the most
				// significant two bits.
				// Setting the top two bits, rather than just the top bit,
				// means that when two of these values are multiplied together,
				// the result isn't ever one bit short.
				if b >= 2 {
					bytes[0] |= 3 << (b - 2)
				} else {
					// Here b==1, because b cannot be zero.
					bytes[0] |= 1
					if len(bytes) > 1 {
						bytes[1] |= 0x80
					}
				}
				// Make the value odd since an even number this large certainly
				// isn't prime.
				bytes[len(bytes)-1] |= 1

				q.SetBytes(bytes)

				// Calculate the value mod the product of smallPrimes. If it's
				// a multiple of any of these primes we add two until it isn't.
				// The probability of overflowing is minimal and can be ignored
				// because we still perform Miller-Rabin tests on the result.
				bigMod.Mod(q, smallPrimesProduct)
				mod := bigMod.Uint64()

			NextDelta:
				for delta := uint64(0); delta < 1<<20; delta += 2 {
					m := mod + delta
					for _, prime := range smallPrimes {
						if m%uint64(prime) == 0 && (qBitLen > 6 || m != uint64(prime)) {
							continue NextDelta
						}
					}

					if delta > 0 {
						bigMod.SetUint64(delta)
						q.Add(q, bigMod)
					}

					// If `q = 1 (mod 3)`, then `p` is a multiple of `3` so it's
					// obviously no prime and such `q` should be rejected.
					// This will happen in 50% of cases and we should detect
					// and eliminate them early.
					//
					// Explanation:
					// If q = 1 (mod 3) then there exists a q' such that:
					// q = 3q' + 1
					//
					// Since p = 2q + 1:
					// p = 2q + 1 = 2(3q' + 1) + 1 = 6q' + 2 + 1 = 6q' + 3 =
					//   = 3(2q' + 1)
					// So `p` is a multiple of `3`.
					qMod3 := new(big.Int).Mod(q, big.NewInt(3))
					if qMod3.Cmp(big.NewInt(1)) == 0 {
						continue NextDelta
					}

					// p = 2q+1
					p.Mul(q, big.NewInt(2))
					p.Add(p, big.NewInt(1))
					if !isPrimeCandidate(p) {
						continue NextDelta
					}

					break
				}

				// There is a tiny possibility that, by adding delta, we caused
				// the number to be one bit too long. Thus we check BitLen
				// here.
				if q.ProbablyPrime(20) &&
					isPocklingtonCriterionSatisfied(p) &&
					q.BitLen() == qBitLen {

					primeChan <- safePrime{p, q}
					return
				}
			}
		}
	}()
}

// Pocklington's criterion can be used to prove the primality of `p = 2q + 1`
// once one has proven the primality of `q`.
// With `q` prime, `p = 2q + 1`, and `p` passing Fermat's primality test to base
// `2` that `2^{p-1} = 1 (mod p)` then `p` is prime as well.
func isPocklingtonCriterionSatisfied(p *big.Int) bool {
	return new(big.Int).Exp(
		big.NewInt(2),
		new(big.Int).Sub(p, big.NewInt(1)),
		p,
	).Cmp(big.NewInt(1)) == 0
}

func isPrimeCandidate(number *big.Int) bool {
	m := new(big.Int).Mod(number, smallPrimesProduct).Uint64()

	for _, prime := range smallPrimes {
		if m%uint64(prime) == 0 && m != uint64(prime) {
			return false
		}
	}

	return true
}
<|endoftext|>


<commit_before>package consensus

import (
	"github.com/NebulousLabs/Sia/modules"

	"github.com/NebulousLabs/bolt"
)

// computeConsensusChange computes the consensus change from the change entry
// at index 'i' in the change log. If i is out of bounds, an error is returned.
func (cs *ConsensusSet) computeConsensusChange(tx *bolt.Tx, ce changeEntry) (modules.ConsensusChange, error) {
	cc := modules.ConsensusChange{
		ID: ce.ID(),
	}
	for _, revertedBlockID := range ce.RevertedBlocks {
		revertedBlock, err := getBlockMap(tx, revertedBlockID)
		if err != nil {
			cs.log.Critical("getBlockMap failed in computeConsensusChange:", err)
			return modules.ConsensusChange{}, err
		}

		// Because the direction is 'revert', the order of the diffs needs to
		// be flipped and the direction of the diffs also needs to be flipped.
		cc.RevertedBlocks = append(cc.RevertedBlocks, revertedBlock.Block)
		for i := len(revertedBlock.SiacoinOutputDiffs) - 1; i >= 0; i-- {
			scod := revertedBlock.SiacoinOutputDiffs[i]
			scod.Direction = !scod.Direction
			cc.SiacoinOutputDiffs = append(cc.SiacoinOutputDiffs, scod)
		}
		for i := len(revertedBlock.FileContractDiffs) - 1; i >= 0; i-- {
			fcd := revertedBlock.FileContractDiffs[i]
			fcd.Direction = !fcd.Direction
			cc.FileContractDiffs = append(cc.FileContractDiffs, fcd)
		}
		for i := len(revertedBlock.SiafundOutputDiffs) - 1; i >= 0; i-- {
			sfod := revertedBlock.SiafundOutputDiffs[i]
			sfod.Direction = !sfod.Direction
			cc.SiafundOutputDiffs = append(cc.SiafundOutputDiffs, sfod)
		}
		for i := len(revertedBlock.DelayedSiacoinOutputDiffs) - 1; i >= 0; i-- {
			dscod := revertedBlock.DelayedSiacoinOutputDiffs[i]
			dscod.Direction = !dscod.Direction
			cc.DelayedSiacoinOutputDiffs = append(cc.DelayedSiacoinOutputDiffs, dscod)
		}
		for i := len(revertedBlock.SiafundPoolDiffs) - 1; i >= 0; i-- {
			sfpd := revertedBlock.SiafundPoolDiffs[i]
			sfpd.Direction = modules.DiffRevert
			cc.SiafundPoolDiffs = append(cc.SiafundPoolDiffs, sfpd)
		}
	}
	for _, appliedBlockID := range ce.AppliedBlocks {
		appliedBlock, err := getBlockMap(tx, appliedBlockID)
		if err != nil {
			cs.log.Critical("getBlockMap failed in computeConsensusChange:", err)
			return modules.ConsensusChange{}, err
		}

		cc.AppliedBlocks = append(cc.AppliedBlocks, appliedBlock.Block)
		for _, scod := range appliedBlock.SiacoinOutputDiffs {
			cc.SiacoinOutputDiffs = append(cc.SiacoinOutputDiffs, scod)
		}
		for _, fcd := range appliedBlock.FileContractDiffs {
			cc.FileContractDiffs = append(cc.FileContractDiffs, fcd)
		}
		for _, sfod := range appliedBlock.SiafundOutputDiffs {
			cc.SiafundOutputDiffs = append(cc.SiafundOutputDiffs, sfod)
		}
		for _, dscod := range appliedBlock.DelayedSiacoinOutputDiffs {
			cc.DelayedSiacoinOutputDiffs = append(cc.DelayedSiacoinOutputDiffs, dscod)
		}
		for _, sfpd := range appliedBlock.SiafundPoolDiffs {
			cc.SiafundPoolDiffs = append(cc.SiafundPoolDiffs, sfpd)
		}
	}
	return cc, nil
}

// readlockUpdateSubscribers will inform all subscribers of a new update to the
// consensus set. The call must be made with a demoted lock or a readlock.
// readlockUpdateSubscribers does not alter the changelog, the changelog must
// be updated beforehand.
func (cs *ConsensusSet) readlockUpdateSubscribers(ce changeEntry) {
	// Get the consensus change and send it to all subscribers.
	var cc modules.ConsensusChange
	err := cs.db.View(func(tx *bolt.Tx) error {
		// Compute the consensus change so it can be sent to subscribers.
		var err error
		cc, err = cs.computeConsensusChange(tx, ce)
		return err
	})
	if err != nil {
		cs.log.Critical("computeConsensusChange failed:", err)
		return
	}
	for _, subscriber := range cs.subscribers {
		subscriber.ProcessConsensusChange(cc)
	}
}

// initializeSubscribe will take a subscriber and feed them all of the
// consensus changes that have occurred since the change provided.
//
// As a special case, using an empty id as the start will have all the changes
// sent to the modules starting with the genesis block.
func (cs *ConsensusSet) initializeSubscribe(subscriber modules.ConsensusSetSubscriber, start modules.ConsensusChangeID) error {
	return cs.db.View(func(tx *bolt.Tx) error {
		// 'exists' and 'entry' are going to be pointed to the first entry that
		// has not yet been seen by subscriber.
		var exists bool
		var entry changeEntry

		if start == modules.ConsensusChangeBeginning {
			// Special case: for modules.ConsensusChangeBeginning, create an
			// initial node pointing to the genesis block. The subscriber will
			// receive the diffs for all blocks in the consensus set, including
			// the genesis block.
			entry = cs.genesisEntry()
			exists = true
		} else if start == modules.ConsensusChangeRecent {
			// Special case: for modules.ConsensusChangeRecent, set up the
			// subscriber to start receiving only new blocks, but the
			// subscriber does not need to do any catch-up. For this
			// implementation, a no-op will have this effect.
			return nil
		} else {
			// The subscriber has provided an existing consensus change.
			// Because the subscriber already has this consensus change,
			// 'entry' and 'exists' need to be pointed at the next consensus
			// change.
			entry, exists = getEntry(tx, start)
			if !exists {
				// modules.ErrInvalidConsensusChangeID is a named error that
				// signals a break in synchronization between the consensus set
				// persistence and the subscriber persistence. Typically,
				// receiving this error means that the subscriber needs to
				// perform a rescan of the consensus set.
				return modules.ErrInvalidConsensusChangeID
			}
			entry, exists = entry.NextEntry(tx)
		}

		// Send all remaining consensus changes to the subscriber.
		for exists {
			cc, err := cs.computeConsensusChange(tx, entry)
			if err != nil {
				return err
			}
			subscriber.ProcessConsensusChange(cc)
			entry, exists = entry.NextEntry(tx)
		}
		return nil
	})
}

// ConsensusSetSubscribe adds a subscriber to the list of subscribers, and
// gives them every consensus change that has occurred since the change with
// the provided id.
//
// As a special case, using an empty id as the start will have all the changes
// sent to the modules starting with the genesis block.
func (cs *ConsensusSet) ConsensusSetSubscribe(subscriber modules.ConsensusSetSubscriber, start modules.ConsensusChangeID) error {
	cs.mu.Lock()
	cs.subscribers = append(cs.subscribers, subscriber)
	cs.mu.Demote()
	defer cs.mu.DemotedUnlock()

	// Get the input module caught up to the currenct consnesus set.
	err := cs.initializeSubscribe(subscriber, start)
	if err != nil {
		return err
	}
	// Only add the module as a subscriber if there was no error.
	return nil
}

// Unsubscribe removes a subscriber from the list of subscribers, allowing for
// garbage collection and rescanning. If the subscriber is not found in the
// subscriber database, no action is taken.
func (cs *ConsensusSet) Unsubscribe(subscriber modules.ConsensusSetSubscriber) {
	cs.mu.Lock()
	defer cs.mu.Unlock()

	// Search for the subscriber in the list of subscribers and remove it if
	// found.
	for i := range cs.subscribers {
		if cs.subscribers[i] == subscriber {
			cs.subscribers = append(cs.subscribers[0:i], cs.subscribers[i+1:]...)
			break
		}
	}
}
<commit_msg>add rescan progress tracking to consensus set<commit_after>package consensus

import (
	"fmt"

	"github.com/NebulousLabs/Sia/modules"

	"github.com/NebulousLabs/bolt"
)

// computeConsensusChange computes the consensus change from the change entry
// at index 'i' in the change log. If i is out of bounds, an error is returned.
func (cs *ConsensusSet) computeConsensusChange(tx *bolt.Tx, ce changeEntry) (modules.ConsensusChange, error) {
	cc := modules.ConsensusChange{
		ID: ce.ID(),
	}
	for _, revertedBlockID := range ce.RevertedBlocks {
		revertedBlock, err := getBlockMap(tx, revertedBlockID)
		if err != nil {
			cs.log.Critical("getBlockMap failed in computeConsensusChange:", err)
			return modules.ConsensusChange{}, err
		}

		// Because the direction is 'revert', the order of the diffs needs to
		// be flipped and the direction of the diffs also needs to be flipped.
		cc.RevertedBlocks = append(cc.RevertedBlocks, revertedBlock.Block)
		for i := len(revertedBlock.SiacoinOutputDiffs) - 1; i >= 0; i-- {
			scod := revertedBlock.SiacoinOutputDiffs[i]
			scod.Direction = !scod.Direction
			cc.SiacoinOutputDiffs = append(cc.SiacoinOutputDiffs, scod)
		}
		for i := len(revertedBlock.FileContractDiffs) - 1; i >= 0; i-- {
			fcd := revertedBlock.FileContractDiffs[i]
			fcd.Direction = !fcd.Direction
			cc.FileContractDiffs = append(cc.FileContractDiffs, fcd)
		}
		for i := len(revertedBlock.SiafundOutputDiffs) - 1; i >= 0; i-- {
			sfod := revertedBlock.SiafundOutputDiffs[i]
			sfod.Direction = !sfod.Direction
			cc.SiafundOutputDiffs = append(cc.SiafundOutputDiffs, sfod)
		}
		for i := len(revertedBlock.DelayedSiacoinOutputDiffs) - 1; i >= 0; i-- {
			dscod := revertedBlock.DelayedSiacoinOutputDiffs[i]
			dscod.Direction = !dscod.Direction
			cc.DelayedSiacoinOutputDiffs = append(cc.DelayedSiacoinOutputDiffs, dscod)
		}
		for i := len(revertedBlock.SiafundPoolDiffs) - 1; i >= 0; i-- {
			sfpd := revertedBlock.SiafundPoolDiffs[i]
			sfpd.Direction = modules.DiffRevert
			cc.SiafundPoolDiffs = append(cc.SiafundPoolDiffs, sfpd)
		}
	}
	for _, appliedBlockID := range ce.AppliedBlocks {
		appliedBlock, err := getBlockMap(tx, appliedBlockID)
		if err != nil {
			cs.log.Critical("getBlockMap failed in computeConsensusChange:", err)
			return modules.ConsensusChange{}, err
		}

		cc.AppliedBlocks = append(cc.AppliedBlocks, appliedBlock.Block)
		for _, scod := range appliedBlock.SiacoinOutputDiffs {
			cc.SiacoinOutputDiffs = append(cc.SiacoinOutputDiffs, scod)
		}
		for _, fcd := range appliedBlock.FileContractDiffs {
			cc.FileContractDiffs = append(cc.FileContractDiffs, fcd)
		}
		for _, sfod := range appliedBlock.SiafundOutputDiffs {
			cc.SiafundOutputDiffs = append(cc.SiafundOutputDiffs, sfod)
		}
		for _, dscod := range appliedBlock.DelayedSiacoinOutputDiffs {
			cc.DelayedSiacoinOutputDiffs = append(cc.DelayedSiacoinOutputDiffs, dscod)
		}
		for _, sfpd := range appliedBlock.SiafundPoolDiffs {
			cc.SiafundPoolDiffs = append(cc.SiafundPoolDiffs, sfpd)
		}
	}
	return cc, nil
}

// readlockUpdateSubscribers will inform all subscribers of a new update to the
// consensus set. The call must be made with a demoted lock or a readlock.
// readlockUpdateSubscribers does not alter the changelog, the changelog must
// be updated beforehand.
func (cs *ConsensusSet) readlockUpdateSubscribers(ce changeEntry) {
	// Get the consensus change and send it to all subscribers.
	var cc modules.ConsensusChange
	err := cs.db.View(func(tx *bolt.Tx) error {
		// Compute the consensus change so it can be sent to subscribers.
		var err error
		cc, err = cs.computeConsensusChange(tx, ce)
		return err
	})
	if err != nil {
		cs.log.Critical("computeConsensusChange failed:", err)
		return
	}
	for _, subscriber := range cs.subscribers {
		subscriber.ProcessConsensusChange(cc)
	}
}

// initializeSubscribe will take a subscriber and feed them all of the
// consensus changes that have occurred since the change provided.
//
// As a special case, using an empty id as the start will have all the changes
// sent to the modules starting with the genesis block.
func (cs *ConsensusSet) initializeSubscribe(subscriber modules.ConsensusSetSubscriber, start modules.ConsensusChangeID) error {
	return cs.db.View(func(tx *bolt.Tx) error {
		// 'exists' and 'entry' are going to be pointed to the first entry that
		// has not yet been seen by subscriber.
		var exists bool
		var entry changeEntry

		if start == modules.ConsensusChangeBeginning {
			// Special case: for modules.ConsensusChangeBeginning, create an
			// initial node pointing to the genesis block. The subscriber will
			// receive the diffs for all blocks in the consensus set, including
			// the genesis block.
			entry = cs.genesisEntry()
			exists = true
		} else if start == modules.ConsensusChangeRecent {
			// Special case: for modules.ConsensusChangeRecent, set up the
			// subscriber to start receiving only new blocks, but the
			// subscriber does not need to do any catch-up. For this
			// implementation, a no-op will have this effect.
			return nil
		} else {
			// The subscriber has provided an existing consensus change.
			// Because the subscriber already has this consensus change,
			// 'entry' and 'exists' need to be pointed at the next consensus
			// change.
			entry, exists = getEntry(tx, start)
			if !exists {
				// modules.ErrInvalidConsensusChangeID is a named error that
				// signals a break in synchronization between the consensus set
				// persistence and the subscriber persistence. Typically,
				// receiving this error means that the subscriber needs to
				// perform a rescan of the consensus set.
				return modules.ErrInvalidConsensusChangeID
			}
			entry, exists = entry.NextEntry(tx)
		}

		// Send all remaining consensus changes to the subscriber.
		for i := 0; exists; i++ {
			cc, err := cs.computeConsensusChange(tx, entry)
			if err != nil {
				return err
			}

			// Get the height of the block to print out.
			if i%1000 == 0 {
				bid := cc.AppliedBlocks[0].ID()
				pb, err := getBlockMap(tx, bid)
				if err == nil {
					if i == 0 {
						fmt.Println("A consensus rescan has been requested. This can take a while.")
					}
					fmt.Println("Rescan has hit height", pb.Height, "out of", blockHeight(tx))
				} else {
					// Didn't get to print for this block, because it was not
					// recognized in the current path. Try printing for the
					// next block. Decrement of `i` is okay because it's only
					// used to decide when to print.
					i--
				}
			}

			subscriber.ProcessConsensusChange(cc)
			entry, exists = entry.NextEntry(tx)
		}
		return nil
	})
}

// ConsensusSetSubscribe adds a subscriber to the list of subscribers, and
// gives them every consensus change that has occurred since the change with
// the provided id.
//
// As a special case, using an empty id as the start will have all the changes
// sent to the modules starting with the genesis block.
func (cs *ConsensusSet) ConsensusSetSubscribe(subscriber modules.ConsensusSetSubscriber, start modules.ConsensusChangeID) error {
	cs.mu.Lock()
	cs.subscribers = append(cs.subscribers, subscriber)
	cs.mu.Demote()
	defer cs.mu.DemotedUnlock()

	// Get the input module caught up to the currenct consnesus set.
	err := cs.initializeSubscribe(subscriber, start)
	if err != nil {
		return err
	}
	// Only add the module as a subscriber if there was no error.
	return nil
}

// Unsubscribe removes a subscriber from the list of subscribers, allowing for
// garbage collection and rescanning. If the subscriber is not found in the
// subscriber database, no action is taken.
func (cs *ConsensusSet) Unsubscribe(subscriber modules.ConsensusSetSubscriber) {
	cs.mu.Lock()
	defer cs.mu.Unlock()

	// Search for the subscriber in the list of subscribers and remove it if
	// found.
	for i := range cs.subscribers {
		if cs.subscribers[i] == subscriber {
			cs.subscribers = append(cs.subscribers[0:i], cs.subscribers[i+1:]...)
			break
		}
	}
}
<|endoftext|>


<commit_before>package floatingips

import (
	"fmt"
	"net/http"
	"testing"

	"github.com/rackspace/gophercloud/pagination"
	th "github.com/rackspace/gophercloud/testhelper"
)

func TestList(t *testing.T) {
	th.SetupHTTP()
	defer th.TeardownHTTP()

	th.Mux.HandleFunc("/v2.0/floatingips", func(w http.ResponseWriter, r *http.Request) {
		th.TestMethod(t, r, "GET")
		th.TestHeader(t, r, "X-Auth-Token", th.TokenID)

		w.Header().Add("Content-Type", "application/json")
		w.WriteHeader(http.StatusOK)

		fmt.Fprintf(w, `
{
    "floatingips": [
        {
            "floating_network_id": "6d67c30a-ddb4-49a1-bec3-a65b286b4170",
            "router_id": null,
            "fixed_ip_address": null,
            "floating_ip_address": "192.0.0.4",
            "tenant_id": "017d8de156df4177889f31a9bd6edc00",
            "status": "DOWN",
            "port_id": null,
            "id": "2f95fd2b-9f6a-4e8e-9e9a-2cbe286cbf9e"
        },
        {
            "floating_network_id": "90f742b1-6d17-487b-ba95-71881dbc0b64",
            "router_id": "0a24cb83-faf5-4d7f-b723-3144ed8a2167",
            "fixed_ip_address": "192.0.0.2",
            "floating_ip_address": "10.0.0.3",
            "tenant_id": "017d8de156df4177889f31a9bd6edc00",
            "status": "DOWN",
            "port_id": "74a342ce-8e07-4e91-880c-9f834b68fa25",
            "id": "ada25a95-f321-4f59-b0e0-f3a970dd3d63"
        }
    ]
}
			`)
	})

	count := 0

	List(th.ServiceClient(), ListOpts{}).EachPage(func(page pagination.Page) (bool, error) {
		count++
		actual, err := ExtractFloatingIPs(page)
		if err != nil {
			t.Errorf("Failed to extract floating IPs: %v", err)
			return false, err
		}

		expected := []FloatingIP{
			FloatingIP{
				FloatingNetworkID: "6d67c30a-ddb4-49a1-bec3-a65b286b4170",
				FixedIP:           "",
				FloatingIP:        "192.0.0.4",
				TenantID:          "017d8de156df4177889f31a9bd6edc00",
				Status:            "DOWN",
				PortID:            "",
				ID:                "2f95fd2b-9f6a-4e8e-9e9a-2cbe286cbf9e",
			},
			FloatingIP{
				FloatingNetworkID: "90f742b1-6d17-487b-ba95-71881dbc0b64",
				FixedIP:           "192.0.0.2",
				FloatingIP:        "10.0.0.3",
				TenantID:          "017d8de156df4177889f31a9bd6edc00",
				Status:            "DOWN",
				PortID:            "74a342ce-8e07-4e91-880c-9f834b68fa25",
				ID:                "ada25a95-f321-4f59-b0e0-f3a970dd3d63",
			},
		}

		th.CheckDeepEquals(t, expected, actual)

		return true, nil
	})

	if count != 1 {
		t.Errorf("Expected 1 page, got %d", count)
	}
}

func TestCreate(t *testing.T) {
	th.SetupHTTP()
	defer th.TeardownHTTP()

	th.Mux.HandleFunc("/v2.0/floatingips", func(w http.ResponseWriter, r *http.Request) {
		th.TestMethod(t, r, "POST")
		th.TestHeader(t, r, "X-Auth-Token", th.TokenID)
		th.TestHeader(t, r, "Content-Type", "application/json")
		th.TestHeader(t, r, "Accept", "application/json")
		th.TestJSONRequest(t, r, `
{
    "floatingip": {
        "floating_network_id": "376da547-b977-4cfe-9cba-275c80debf57",
        "port_id": "ce705c24-c1ef-408a-bda3-7bbd946164ab"
    }
}
			`)

		w.Header().Add("Content-Type", "application/json")
		w.WriteHeader(http.StatusCreated)

		fmt.Fprintf(w, `
{
    "floatingip": {
        "router_id": "d23abc8d-2991-4a55-ba98-2aaea84cc72f",
        "tenant_id": "4969c491a3c74ee4af974e6d800c62de",
        "floating_network_id": "376da547-b977-4cfe-9cba-275c80debf57",
        "fixed_ip_address": "10.0.0.3",
        "floating_ip_address": "",
        "port_id": "ce705c24-c1ef-408a-bda3-7bbd946164ab",
        "id": "2f245a7b-796b-4f26-9cf9-9e82d248fda7"
    }
}
		`)
	})

	options := CreateOpts{
		FloatingNetworkID: "376da547-b977-4cfe-9cba-275c80debf57",
		PortID:            "ce705c24-c1ef-408a-bda3-7bbd946164ab",
	}

	ip, err := Create(th.ServiceClient(), options).Extract()
	th.AssertNoErr(t, err)

	th.AssertEquals(t, "2f245a7b-796b-4f26-9cf9-9e82d248fda7", ip.ID)
	th.AssertEquals(t, "4969c491a3c74ee4af974e6d800c62de", ip.TenantID)
	th.AssertEquals(t, "376da547-b977-4cfe-9cba-275c80debf57", ip.FloatingNetworkID)
	th.AssertEquals(t, "", ip.FloatingIP)
	th.AssertEquals(t, "ce705c24-c1ef-408a-bda3-7bbd946164ab", ip.PortID)
	th.AssertEquals(t, "10.0.0.3", ip.FixedIP)
}

func TestGet(t *testing.T) {
	th.SetupHTTP()
	defer th.TeardownHTTP()

	th.Mux.HandleFunc("/v2.0/floatingips/2f245a7b-796b-4f26-9cf9-9e82d248fda7", func(w http.ResponseWriter, r *http.Request) {
		th.TestMethod(t, r, "GET")
		th.TestHeader(t, r, "X-Auth-Token", th.TokenID)

		w.Header().Add("Content-Type", "application/json")
		w.WriteHeader(http.StatusOK)

		fmt.Fprintf(w, `
{
    "floatingip": {
        "floating_network_id": "90f742b1-6d17-487b-ba95-71881dbc0b64",
        "fixed_ip_address": "192.0.0.2",
        "floating_ip_address": "10.0.0.3",
        "tenant_id": "017d8de156df4177889f31a9bd6edc00",
        "status": "DOWN",
        "port_id": "74a342ce-8e07-4e91-880c-9f834b68fa25",
        "id": "2f245a7b-796b-4f26-9cf9-9e82d248fda7"
    }
}
      `)
	})

	ip, err := Get(th.ServiceClient(), "2f245a7b-796b-4f26-9cf9-9e82d248fda7").Extract()
	th.AssertNoErr(t, err)

	th.AssertEquals(t, "90f742b1-6d17-487b-ba95-71881dbc0b64", ip.FloatingNetworkID)
	th.AssertEquals(t, "10.0.0.3", ip.FloatingIP)
	th.AssertEquals(t, "74a342ce-8e07-4e91-880c-9f834b68fa25", ip.PortID)
	th.AssertEquals(t, "192.0.0.2", ip.FixedIP)
	th.AssertEquals(t, "017d8de156df4177889f31a9bd6edc00", ip.TenantID)
	th.AssertEquals(t, "DOWN", ip.Status)
	th.AssertEquals(t, "2f245a7b-796b-4f26-9cf9-9e82d248fda7", ip.ID)
}

func TestAssociate(t *testing.T) {

}

func TestDisassociate(t *testing.T) {
	th.SetupHTTP()
	defer th.TeardownHTTP()

	th.Mux.HandleFunc("/v2.0/floatingips/2f245a7b-796b-4f26-9cf9-9e82d248fda7", func(w http.ResponseWriter, r *http.Request) {
		th.TestMethod(t, r, "PUT")
		th.TestHeader(t, r, "X-Auth-Token", th.TokenID)
		th.TestHeader(t, r, "Content-Type", "application/json")
		th.TestHeader(t, r, "Accept", "application/json")
		th.TestJSONRequest(t, r, `
{
    "floatingip": {
      "port_id": null
    }
}
      `)

		w.Header().Add("Content-Type", "application/json")
		w.WriteHeader(http.StatusOK)

		fmt.Fprintf(w, `
{
    "floatingip": {
        "router_id": "d23abc8d-2991-4a55-ba98-2aaea84cc72f",
        "tenant_id": "4969c491a3c74ee4af974e6d800c62de",
        "floating_network_id": "376da547-b977-4cfe-9cba-275c80debf57",
        "fixed_ip_address": null,
        "floating_ip_address": "172.24.4.228",
        "port_id": null,
        "id": "2f245a7b-796b-4f26-9cf9-9e82d248fda7"
    }
}
    `)
	})

	ip, err := Update(th.ServiceClient(), "2f245a7b-796b-4f26-9cf9-9e82d248fda7", UpdateOpts{}).Extract()
	th.AssertNoErr(t, err)

	th.AssertDeepEquals(t, "", ip.FixedIP)
	th.AssertDeepEquals(t, "", ip.PortID)
}

func TestDelete(t *testing.T) {
	th.SetupHTTP()
	defer th.TeardownHTTP()

	th.Mux.HandleFunc("/v2.0/floatingips/2f245a7b-796b-4f26-9cf9-9e82d248fda7", func(w http.ResponseWriter, r *http.Request) {
		th.TestMethod(t, r, "DELETE")
		th.TestHeader(t, r, "X-Auth-Token", th.TokenID)
		w.WriteHeader(http.StatusNoContent)
	})

	res := Delete(th.ServiceClient(), "2f245a7b-796b-4f26-9cf9-9e82d248fda7")
	th.AssertNoErr(t, res.Err)
}
<commit_msg>Adding missing test for associate<commit_after>package floatingips

import (
	"fmt"
	"net/http"
	"testing"

	"github.com/rackspace/gophercloud/pagination"
	th "github.com/rackspace/gophercloud/testhelper"
)

func TestList(t *testing.T) {
	th.SetupHTTP()
	defer th.TeardownHTTP()

	th.Mux.HandleFunc("/v2.0/floatingips", func(w http.ResponseWriter, r *http.Request) {
		th.TestMethod(t, r, "GET")
		th.TestHeader(t, r, "X-Auth-Token", th.TokenID)

		w.Header().Add("Content-Type", "application/json")
		w.WriteHeader(http.StatusOK)

		fmt.Fprintf(w, `
{
    "floatingips": [
        {
            "floating_network_id": "6d67c30a-ddb4-49a1-bec3-a65b286b4170",
            "router_id": null,
            "fixed_ip_address": null,
            "floating_ip_address": "192.0.0.4",
            "tenant_id": "017d8de156df4177889f31a9bd6edc00",
            "status": "DOWN",
            "port_id": null,
            "id": "2f95fd2b-9f6a-4e8e-9e9a-2cbe286cbf9e"
        },
        {
            "floating_network_id": "90f742b1-6d17-487b-ba95-71881dbc0b64",
            "router_id": "0a24cb83-faf5-4d7f-b723-3144ed8a2167",
            "fixed_ip_address": "192.0.0.2",
            "floating_ip_address": "10.0.0.3",
            "tenant_id": "017d8de156df4177889f31a9bd6edc00",
            "status": "DOWN",
            "port_id": "74a342ce-8e07-4e91-880c-9f834b68fa25",
            "id": "ada25a95-f321-4f59-b0e0-f3a970dd3d63"
        }
    ]
}
			`)
	})

	count := 0

	List(th.ServiceClient(), ListOpts{}).EachPage(func(page pagination.Page) (bool, error) {
		count++
		actual, err := ExtractFloatingIPs(page)
		if err != nil {
			t.Errorf("Failed to extract floating IPs: %v", err)
			return false, err
		}

		expected := []FloatingIP{
			FloatingIP{
				FloatingNetworkID: "6d67c30a-ddb4-49a1-bec3-a65b286b4170",
				FixedIP:           "",
				FloatingIP:        "192.0.0.4",
				TenantID:          "017d8de156df4177889f31a9bd6edc00",
				Status:            "DOWN",
				PortID:            "",
				ID:                "2f95fd2b-9f6a-4e8e-9e9a-2cbe286cbf9e",
			},
			FloatingIP{
				FloatingNetworkID: "90f742b1-6d17-487b-ba95-71881dbc0b64",
				FixedIP:           "192.0.0.2",
				FloatingIP:        "10.0.0.3",
				TenantID:          "017d8de156df4177889f31a9bd6edc00",
				Status:            "DOWN",
				PortID:            "74a342ce-8e07-4e91-880c-9f834b68fa25",
				ID:                "ada25a95-f321-4f59-b0e0-f3a970dd3d63",
			},
		}

		th.CheckDeepEquals(t, expected, actual)

		return true, nil
	})

	if count != 1 {
		t.Errorf("Expected 1 page, got %d", count)
	}
}

func TestCreate(t *testing.T) {
	th.SetupHTTP()
	defer th.TeardownHTTP()

	th.Mux.HandleFunc("/v2.0/floatingips", func(w http.ResponseWriter, r *http.Request) {
		th.TestMethod(t, r, "POST")
		th.TestHeader(t, r, "X-Auth-Token", th.TokenID)
		th.TestHeader(t, r, "Content-Type", "application/json")
		th.TestHeader(t, r, "Accept", "application/json")
		th.TestJSONRequest(t, r, `
{
    "floatingip": {
        "floating_network_id": "376da547-b977-4cfe-9cba-275c80debf57",
        "port_id": "ce705c24-c1ef-408a-bda3-7bbd946164ab"
    }
}
			`)

		w.Header().Add("Content-Type", "application/json")
		w.WriteHeader(http.StatusCreated)

		fmt.Fprintf(w, `
{
    "floatingip": {
        "router_id": "d23abc8d-2991-4a55-ba98-2aaea84cc72f",
        "tenant_id": "4969c491a3c74ee4af974e6d800c62de",
        "floating_network_id": "376da547-b977-4cfe-9cba-275c80debf57",
        "fixed_ip_address": "10.0.0.3",
        "floating_ip_address": "",
        "port_id": "ce705c24-c1ef-408a-bda3-7bbd946164ab",
        "id": "2f245a7b-796b-4f26-9cf9-9e82d248fda7"
    }
}
		`)
	})

	options := CreateOpts{
		FloatingNetworkID: "376da547-b977-4cfe-9cba-275c80debf57",
		PortID:            "ce705c24-c1ef-408a-bda3-7bbd946164ab",
	}

	ip, err := Create(th.ServiceClient(), options).Extract()
	th.AssertNoErr(t, err)

	th.AssertEquals(t, "2f245a7b-796b-4f26-9cf9-9e82d248fda7", ip.ID)
	th.AssertEquals(t, "4969c491a3c74ee4af974e6d800c62de", ip.TenantID)
	th.AssertEquals(t, "376da547-b977-4cfe-9cba-275c80debf57", ip.FloatingNetworkID)
	th.AssertEquals(t, "", ip.FloatingIP)
	th.AssertEquals(t, "ce705c24-c1ef-408a-bda3-7bbd946164ab", ip.PortID)
	th.AssertEquals(t, "10.0.0.3", ip.FixedIP)
}

func TestGet(t *testing.T) {
	th.SetupHTTP()
	defer th.TeardownHTTP()

	th.Mux.HandleFunc("/v2.0/floatingips/2f245a7b-796b-4f26-9cf9-9e82d248fda7", func(w http.ResponseWriter, r *http.Request) {
		th.TestMethod(t, r, "GET")
		th.TestHeader(t, r, "X-Auth-Token", th.TokenID)

		w.Header().Add("Content-Type", "application/json")
		w.WriteHeader(http.StatusOK)

		fmt.Fprintf(w, `
{
    "floatingip": {
        "floating_network_id": "90f742b1-6d17-487b-ba95-71881dbc0b64",
        "fixed_ip_address": "192.0.0.2",
        "floating_ip_address": "10.0.0.3",
        "tenant_id": "017d8de156df4177889f31a9bd6edc00",
        "status": "DOWN",
        "port_id": "74a342ce-8e07-4e91-880c-9f834b68fa25",
        "id": "2f245a7b-796b-4f26-9cf9-9e82d248fda7"
    }
}
      `)
	})

	ip, err := Get(th.ServiceClient(), "2f245a7b-796b-4f26-9cf9-9e82d248fda7").Extract()
	th.AssertNoErr(t, err)

	th.AssertEquals(t, "90f742b1-6d17-487b-ba95-71881dbc0b64", ip.FloatingNetworkID)
	th.AssertEquals(t, "10.0.0.3", ip.FloatingIP)
	th.AssertEquals(t, "74a342ce-8e07-4e91-880c-9f834b68fa25", ip.PortID)
	th.AssertEquals(t, "192.0.0.2", ip.FixedIP)
	th.AssertEquals(t, "017d8de156df4177889f31a9bd6edc00", ip.TenantID)
	th.AssertEquals(t, "DOWN", ip.Status)
	th.AssertEquals(t, "2f245a7b-796b-4f26-9cf9-9e82d248fda7", ip.ID)
}

func TestAssociate(t *testing.T) {
	th.SetupHTTP()
	defer th.TeardownHTTP()

	th.Mux.HandleFunc("/v2.0/floatingips/2f245a7b-796b-4f26-9cf9-9e82d248fda7", func(w http.ResponseWriter, r *http.Request) {
		th.TestMethod(t, r, "PUT")
		th.TestHeader(t, r, "X-Auth-Token", th.TokenID)
		th.TestHeader(t, r, "Content-Type", "application/json")
		th.TestHeader(t, r, "Accept", "application/json")
		th.TestJSONRequest(t, r, `
{
	"floatingip": {
		"port_id": "423abc8d-2991-4a55-ba98-2aaea84cc72e"
	}
}
		`)

		w.Header().Add("Content-Type", "application/json")
		w.WriteHeader(http.StatusOK)

		fmt.Fprintf(w, `
{
	"floatingip": {
			"router_id": "d23abc8d-2991-4a55-ba98-2aaea84cc72f",
			"tenant_id": "4969c491a3c74ee4af974e6d800c62de",
			"floating_network_id": "376da547-b977-4cfe-9cba-275c80debf57",
			"fixed_ip_address": null,
			"floating_ip_address": "172.24.4.228",
			"port_id": "423abc8d-2991-4a55-ba98-2aaea84cc72e",
			"id": "2f245a7b-796b-4f26-9cf9-9e82d248fda7"
	}
}
	`)
	})

	ip, err := Update(th.ServiceClient(), "2f245a7b-796b-4f26-9cf9-9e82d248fda7", UpdateOpts{PortID: "423abc8d-2991-4a55-ba98-2aaea84cc72e"}).Extract()
	th.AssertNoErr(t, err)

	th.AssertDeepEquals(t, "423abc8d-2991-4a55-ba98-2aaea84cc72e", ip.PortID)
}

func TestDisassociate(t *testing.T) {
	th.SetupHTTP()
	defer th.TeardownHTTP()

	th.Mux.HandleFunc("/v2.0/floatingips/2f245a7b-796b-4f26-9cf9-9e82d248fda7", func(w http.ResponseWriter, r *http.Request) {
		th.TestMethod(t, r, "PUT")
		th.TestHeader(t, r, "X-Auth-Token", th.TokenID)
		th.TestHeader(t, r, "Content-Type", "application/json")
		th.TestHeader(t, r, "Accept", "application/json")
		th.TestJSONRequest(t, r, `
{
    "floatingip": {
      "port_id": null
    }
}
      `)

		w.Header().Add("Content-Type", "application/json")
		w.WriteHeader(http.StatusOK)

		fmt.Fprintf(w, `
{
    "floatingip": {
        "router_id": "d23abc8d-2991-4a55-ba98-2aaea84cc72f",
        "tenant_id": "4969c491a3c74ee4af974e6d800c62de",
        "floating_network_id": "376da547-b977-4cfe-9cba-275c80debf57",
        "fixed_ip_address": null,
        "floating_ip_address": "172.24.4.228",
        "port_id": null,
        "id": "2f245a7b-796b-4f26-9cf9-9e82d248fda7"
    }
}
    `)
	})

	ip, err := Update(th.ServiceClient(), "2f245a7b-796b-4f26-9cf9-9e82d248fda7", UpdateOpts{}).Extract()
	th.AssertNoErr(t, err)

	th.AssertDeepEquals(t, "", ip.FixedIP)
	th.AssertDeepEquals(t, "", ip.PortID)
}

func TestDelete(t *testing.T) {
	th.SetupHTTP()
	defer th.TeardownHTTP()

	th.Mux.HandleFunc("/v2.0/floatingips/2f245a7b-796b-4f26-9cf9-9e82d248fda7", func(w http.ResponseWriter, r *http.Request) {
		th.TestMethod(t, r, "DELETE")
		th.TestHeader(t, r, "X-Auth-Token", th.TokenID)
		w.WriteHeader(http.StatusNoContent)
	})

	res := Delete(th.ServiceClient(), "2f245a7b-796b-4f26-9cf9-9e82d248fda7")
	th.AssertNoErr(t, res.Err)
}
<|endoftext|>


<commit_before>package spotify

import (
	"fmt"
	"net/url"
	"strconv"
	"strings"
)

// SimpleArtist contains basic info about an artist.
type SimpleArtist struct {
	Name string `json:"name"`
	ID   ID     `json:"id"`
	// The Spotify URI for the artist.
	URI URI `json:"uri"`
	// A link to the Web API enpoint providing full details of the artist.
	Endpoint     string            `json:"href"`
	ExternalURLs map[string]string `json:"external_urls"`
}

// FullArtist provides extra artist data in addition to what is provided by SimpleArtist.
type FullArtist struct {
	SimpleArtist
	// The popularity of the artist, expressed as an integer between 0 and 100.
	// The artist's popularity is calculated from the popularity of the artist's tracks.
	Popularity int `json:"popularity"`
	// A list of genres the artist is associated with.  For example, "Prog Rock"
	// or "Post-Grunge".  If not yet classified, the slice is empty.
	Genres    []string  `json:"genres"`
	Followers Followers `json:"followers"`
	// Images of the artist in various sizes, widest first.
	Images []Image `json:"images"`
}

// GetArtist gets Spotify catalog information for a single artist, given its Spotify ID.
func (c *Client) GetArtist(id ID) (*FullArtist, error) {
	spotifyURL := fmt.Sprintf("%sartists/%s", c.baseURL, id)

	var a FullArtist
	err := c.get(spotifyURL, &a)
	if err != nil {
		return nil, err
	}

	return &a, nil
}

// GetArtists gets spotify catalog information for several artists based on their
// Spotify IDs.  It supports up to 50 artists in a single call.  Artists are
// returned in the order requested.  If an artist is not found, that position
// in the result will be nil.  Duplicate IDs will result in duplicate artists
// in the result.
func (c *Client) GetArtists(ids ...ID) ([]*FullArtist, error) {
	spotifyURL := fmt.Sprintf("%sartists?ids=%s", c.baseURL, strings.Join(toStringSlice(ids), ","))

	var a struct {
		Artists []*FullArtist
	}

	err := c.get(spotifyURL, &a)
	if err != nil {
		return nil, err
	}

	return a.Artists, nil
}

// GetArtistsTopTracks gets Spotify catalog information about an artist's top
// tracks in a particular country.  It returns a maximum of 10 tracks.  The
// country is specified as an ISO 3166-1 alpha-2 country code.
func (c *Client) GetArtistsTopTracks(artistID ID, country string) ([]FullTrack, error) {
	spotifyURL := fmt.Sprintf("%sartists/%s/top-tracks?country=%s", c.baseURL, artistID, country)

	var t struct {
		Tracks []FullTrack `json:"tracks"`
	}

	err := c.get(spotifyURL, &t)
	if err != nil {
		return nil, err
	}

	return t.Tracks, nil
}

// GetRelatedArtists gets Spotify catalog information about artists similar to a
// given artist.  Similarity is based on analysis of the Spotify community's
// listening history.  This function returns up to 20 artists that are considered
// related to the specified artist.
func (c *Client) GetRelatedArtists(id ID) ([]FullArtist, error) {
	spotifyURL := fmt.Sprintf("%sartists/%s/related-artists", c.baseURL, id)

	var a struct {
		Artists []FullArtist `json:"artists"`
	}

	err := c.get(spotifyURL, &a)
	if err != nil {
		return nil, err
	}

	return a.Artists, nil
}

// GetArtistAlbums gets Spotify catalog information about an artist's albums.
// It is equivalent to GetArtistAlbumsOpt(artistID, nil).
func (c *Client) GetArtistAlbums(artistID ID) (*SimpleAlbumPage, error) {
	return c.GetArtistAlbumsOpt(artistID, nil)
}

// GetArtistAlbumsOpt is just like GetArtistAlbums, but it accepts optional
// parameters used to filter and sort the result.
//
// The AlbumType argument can be used to find a particular types of album.
// If the market (Options) is not specified, Spotify will likely return a lot
// of duplicates (one for each market in which the album is available)
func (c *Client) GetArtistAlbumsOpt(artistID ID, options *Options, ts ...AlbumType) (*SimpleAlbumPage, error) {
	spotifyURL := fmt.Sprintf("%sartists/%s/albums", c.baseURL, artistID)
	// add optional query string if options were specified
	values := url.Values{}
	if ts != nil {
		types := make([]string, len(ts))
		for i := range ts {
			types[i] = ts[i].encode()
		}
		values.Set("include_groups", strings.Join(types, ","))
	}
	if options != nil {
		if options.Country != nil {
			values.Set("market", *options.Country)
		}
		if options.Limit != nil {
			values.Set("limit", strconv.Itoa(*options.Limit))
		}
		if options.Offset != nil {
			values.Set("offset", strconv.Itoa(*options.Offset))
		}
	}
	if query := values.Encode(); query != "" {
		spotifyURL += "?" + query
	}

	var p SimpleAlbumPage

	err := c.get(spotifyURL, &p)
	if err != nil {
		return nil, err
	}

	return &p, nil
}
<commit_msg>clarification<commit_after>package spotify

import (
	"fmt"
	"net/url"
	"strconv"
	"strings"
)

// SimpleArtist contains basic info about an artist.
type SimpleArtist struct {
	Name string `json:"name"`
	ID   ID     `json:"id"`
	// The Spotify URI for the artist.
	URI URI `json:"uri"`
	// A link to the Web API enpoint providing full details of the artist.
	Endpoint     string            `json:"href"`
	ExternalURLs map[string]string `json:"external_urls"`
}

// FullArtist provides extra artist data in addition to what is provided by SimpleArtist.
type FullArtist struct {
	SimpleArtist
	// The popularity of the artist, expressed as an integer between 0 and 100.
	// The artist's popularity is calculated from the popularity of the artist's tracks.
	Popularity int `json:"popularity"`
	// A list of genres the artist is associated with.  For example, "Prog Rock"
	// or "Post-Grunge".  If not yet classified, the slice is empty.
	Genres    []string  `json:"genres"`
	Followers Followers `json:"followers"`
	// Images of the artist in various sizes, widest first.
	Images []Image `json:"images"`
}

// GetArtist gets Spotify catalog information for a single artist, given its Spotify ID.
func (c *Client) GetArtist(id ID) (*FullArtist, error) {
	spotifyURL := fmt.Sprintf("%sartists/%s", c.baseURL, id)

	var a FullArtist
	err := c.get(spotifyURL, &a)
	if err != nil {
		return nil, err
	}

	return &a, nil
}

// GetArtists gets spotify catalog information for several artists based on their
// Spotify IDs.  It supports up to 50 artists in a single call.  Artists are
// returned in the order requested.  If an artist is not found, that position
// in the result will be nil.  Duplicate IDs will result in duplicate artists
// in the result.
func (c *Client) GetArtists(ids ...ID) ([]*FullArtist, error) {
	spotifyURL := fmt.Sprintf("%sartists?ids=%s", c.baseURL, strings.Join(toStringSlice(ids), ","))

	var a struct {
		Artists []*FullArtist
	}

	err := c.get(spotifyURL, &a)
	if err != nil {
		return nil, err
	}

	return a.Artists, nil
}

// GetArtistsTopTracks gets Spotify catalog information about an artist's top
// tracks in a particular country.  It returns a maximum of 10 tracks.  The
// country is specified as an ISO 3166-1 alpha-2 country code.
func (c *Client) GetArtistsTopTracks(artistID ID, country string) ([]FullTrack, error) {
	spotifyURL := fmt.Sprintf("%sartists/%s/top-tracks?country=%s", c.baseURL, artistID, country)

	var t struct {
		Tracks []FullTrack `json:"tracks"`
	}

	err := c.get(spotifyURL, &t)
	if err != nil {
		return nil, err
	}

	return t.Tracks, nil
}

// GetRelatedArtists gets Spotify catalog information about artists similar to a
// given artist.  Similarity is based on analysis of the Spotify community's
// listening history.  This function returns up to 20 artists that are considered
// related to the specified artist.
func (c *Client) GetRelatedArtists(id ID) ([]FullArtist, error) {
	spotifyURL := fmt.Sprintf("%sartists/%s/related-artists", c.baseURL, id)

	var a struct {
		Artists []FullArtist `json:"artists"`
	}

	err := c.get(spotifyURL, &a)
	if err != nil {
		return nil, err
	}

	return a.Artists, nil
}

// GetArtistAlbums gets Spotify catalog information about an artist's albums.
// It is equivalent to GetArtistAlbumsOpt(artistID, nil).
func (c *Client) GetArtistAlbums(artistID ID) (*SimpleAlbumPage, error) {
	return c.GetArtistAlbumsOpt(artistID, nil)
}

// GetArtistAlbumsOpt is just like GetArtistAlbums, but it accepts optional
// parameters used to filter and sort the result.
//
// The AlbumType argument can be used to find a particular types of album.
// If the market (Options.Country) is not specified, Spotify will likely return a lot
// of duplicates (one for each market in which the album is available)
func (c *Client) GetArtistAlbumsOpt(artistID ID, options *Options, ts ...AlbumType) (*SimpleAlbumPage, error) {
	spotifyURL := fmt.Sprintf("%sartists/%s/albums", c.baseURL, artistID)
	// add optional query string if options were specified
	values := url.Values{}
	if ts != nil {
		types := make([]string, len(ts))
		for i := range ts {
			types[i] = ts[i].encode()
		}
		values.Set("include_groups", strings.Join(types, ","))
	}
	if options != nil {
		if options.Country != nil {
			values.Set("market", *options.Country)
		}
		if options.Limit != nil {
			values.Set("limit", strconv.Itoa(*options.Limit))
		}
		if options.Offset != nil {
			values.Set("offset", strconv.Itoa(*options.Offset))
		}
	}
	if query := values.Encode(); query != "" {
		spotifyURL += "?" + query
	}

	var p SimpleAlbumPage

	err := c.get(spotifyURL, &p)
	if err != nil {
		return nil, err
	}

	return &p, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/csv"
	"fmt"
	"io"
	"os"
	"strconv"
)

type CSVTableSchemaRecord struct {
	recordType string
	index      int
}

type CSVTableSchema map[string]CSVTableSchemaRecord

func ReadSchema(r io.Reader) CSVTableSchema {
	csvReader := csv.NewReader(r)
	schemaData, _ := csvReader.ReadAll()
	schema := make(CSVTableSchema)

	for k, v := range schemaData {
		r := CSVTableSchemaRecord{
			recordType: v[1],
			index:      k}

		schema[v[0]] = r
	}

	return schema
}

type WhereCondition interface {
	Compute([]string, CSVTableSchema) bool
}

type WhereSimpleCondition struct {
	field, op, value string
}

func NewSimpleWhere(field, op, value string) WhereSimpleCondition {
	c := WhereSimpleCondition{
		field: field,
		op:    op,
		value: value,
	}

	return c
}

func (this WhereSimpleCondition) Compute(record []string, schema CSVTableSchema) bool {

	testInt := func(op string, value, test int) bool {
		var r bool
		switch op {
		case "=":
			r = value == test
		case "!=":
			r = value != test
		case "<":
			r = value < test
		case ">":
			r = value > test
		case "<=":
			r = value <= test
		case ">=":
			r = value >= test
		}

		return r
	}

	testStr := func(op, value, test string) bool {
		var r bool
		switch op {
		case "=":
			r = value == test
		case "!=":
			r = value != test
		case "<":
			r = value < test
		case ">":
			r = value > test
		case "<=":
			r = value <= test
		case ">=":
			r = value >= test
		}

		return r
	}

	fieldIndex := schema[this.field].index
	var r bool

	switch schema[this.field].recordType {
	case "int":
		fieldValue, _ := strconv.ParseInt(record[fieldIndex], 10, 32)
		testValue, _ := strconv.ParseInt(this.value, 10, 32)
		r = testInt(this.op, int(fieldValue), int(testValue))
	case "text":
		r = testStr(this.op, record[fieldIndex], this.value)
	}

	return r
}

type WhereComplexCondition struct {
	logicCond    string
	cond1, cond2 WhereCondition
}

func NewComplexWhereCondition(logicCond string, cond1, cond2 WhereCondition) WhereComplexCondition {
	c := WhereComplexCondition{
		logicCond: logicCond,
		cond1:     cond1,
		cond2:     cond2,
	}

	return c
}

func (this WhereComplexCondition) Compute(record []string, schema CSVTableSchema) bool {
	r1 := this.cond1.Compute(record, schema)
	r2 := this.cond2.Compute(record, schema)

	var r bool

	switch this.logicCond {
	case "AND":
		r = r1 && r2
	case "OR":
		r = r1 || r2
	}

	return r
}

type SelectQuery struct {
	fields []string
	table  string
	where  WhereCondition
	limit  int
	schema CSVTableSchema
}

type SelectQueryResultRecord map[int]string

type SelectQueryResult []SelectQueryResultRecord

func Select(fields ...string) *SelectQuery {
	q := new(SelectQuery)

	for _, v := range fields {
		q.fields = append(q.fields, v)
	}

	return q
}

func (this *SelectQuery) From(table string) *SelectQuery {
	r, _ := os.Open(table + ".schema")

	this.schema = ReadSchema(r)

	this.table = table

	return this
}

func (this *SelectQuery) Where(cond WhereCondition) *SelectQuery {
	this.where = cond

	return this
}

func (this *SelectQuery) Limit(limit int) *SelectQuery {
	this.limit = limit

	return this
}

func (this *SelectQuery) Do() [][]string {
	r, _ := os.Open(this.table + ".csv")

	csvReader := csv.NewReader(r)

	records, _ := csvReader.ReadAll()

	result := make([][]string, 0)

	for _, v := range records {
		if len(result) >= this.limit && this.limit != 0 {
			break
		}

		if this.where != nil {
			if this.where.Compute(v, this.schema) {
				result = append(result, v)
			}
		} else {
			result = append(result, v)
		}

	}

	return result
}

func main() {
	simpleWhere1 := NewSimpleWhere("id", ">", "1")
	simpleWhere2 := NewSimpleWhere("content", "=", "три")
	complexWhere := NewComplexWhereCondition("AND", simpleWhere1, simpleWhere2)
	q := Select("content", "id").From("index").Where(complexWhere)
	for _, r := range q.Do() {
		for _, f := range r {
			fmt.Print(f, " ")
		}
		fmt.Println()
	}
}
<commit_msg>csvstorage: результаты запроса теперь возращают тип SelectQueryResult<commit_after>package main

import (
	"encoding/csv"
	"fmt"
	"io"
	"os"
	"strconv"
)

type CSVTableSchemaRecord struct {
	recordType string
	index      int
}

type CSVTableSchema map[string]CSVTableSchemaRecord

func ReadSchema(r io.Reader) CSVTableSchema {
	csvReader := csv.NewReader(r)
	schemaData, _ := csvReader.ReadAll()
	schema := make(CSVTableSchema)

	for k, v := range schemaData {
		r := CSVTableSchemaRecord{
			recordType: v[1],
			index:      k}

		schema[v[0]] = r
	}

	return schema
}

type WhereCondition interface {
	Compute([]string, CSVTableSchema) bool
}

type WhereSimpleCondition struct {
	field, op, value string
}

func NewSimpleWhere(field, op, value string) WhereSimpleCondition {
	c := WhereSimpleCondition{
		field: field,
		op:    op,
		value: value,
	}

	return c
}

func (this WhereSimpleCondition) Compute(record []string, schema CSVTableSchema) bool {

	testInt := func(op string, value, test int) bool {
		var r bool
		switch op {
		case "=":
			r = value == test
		case "!=":
			r = value != test
		case "<":
			r = value < test
		case ">":
			r = value > test
		case "<=":
			r = value <= test
		case ">=":
			r = value >= test
		}

		return r
	}

	testStr := func(op, value, test string) bool {
		var r bool
		switch op {
		case "=":
			r = value == test
		case "!=":
			r = value != test
		case "<":
			r = value < test
		case ">":
			r = value > test
		case "<=":
			r = value <= test
		case ">=":
			r = value >= test
		}

		return r
	}

	fieldIndex := schema[this.field].index
	var r bool

	switch schema[this.field].recordType {
	case "int":
		fieldValue, _ := strconv.ParseInt(record[fieldIndex], 10, 32)
		testValue, _ := strconv.ParseInt(this.value, 10, 32)
		r = testInt(this.op, int(fieldValue), int(testValue))
	case "text":
		r = testStr(this.op, record[fieldIndex], this.value)
	}

	return r
}

type WhereComplexCondition struct {
	logicCond    string
	cond1, cond2 WhereCondition
}

func NewComplexWhereCondition(logicCond string, cond1, cond2 WhereCondition) WhereComplexCondition {
	c := WhereComplexCondition{
		logicCond: logicCond,
		cond1:     cond1,
		cond2:     cond2,
	}

	return c
}

func (this WhereComplexCondition) Compute(record []string, schema CSVTableSchema) bool {
	r1 := this.cond1.Compute(record, schema)
	r2 := this.cond2.Compute(record, schema)

	var r bool

	switch this.logicCond {
	case "AND":
		r = r1 && r2
	case "OR":
		r = r1 || r2
	}

	return r
}

type SelectQuery struct {
	fields []string
	table  string
	where  WhereCondition
	limit  int
	schema CSVTableSchema
}

type SelectQueryResultRecord map[string]string

type SelectQueryResult []SelectQueryResultRecord

func Select(fields ...string) *SelectQuery {
	q := new(SelectQuery)

	for _, v := range fields {
		q.fields = append(q.fields, v)
	}

	return q
}

func (this *SelectQuery) From(table string) *SelectQuery {
	r, _ := os.Open(table + ".schema")

	this.schema = ReadSchema(r)

	this.table = table

	return this
}

func (this *SelectQuery) Where(cond WhereCondition) *SelectQuery {
	this.where = cond

	return this
}

func (this *SelectQuery) Limit(limit int) *SelectQuery {
	this.limit = limit

	return this
}

func (this *SelectQuery) Do() SelectQueryResult {
	r, _ := os.Open(this.table + ".csv")

	csvReader := csv.NewReader(r)

	records, _ := csvReader.ReadAll()

	result := make(SelectQueryResult, 0)

	outputFields := make(map[string]int)
	for _, f := range this.fields {
		outputFields[f] = this.schema[f].index
	}

	for _, v := range records {
		if len(result) >= this.limit && this.limit != 0 {
			break
		}

		selectQueryResultRecord := make(SelectQueryResultRecord)

		for fName, fIndex := range outputFields {
			selectQueryResultRecord[fName] = v[fIndex]
		}

		if this.where != nil {
			if this.where.Compute(v, this.schema) {
				result = append(result, selectQueryResultRecord)
			}
		} else {
			result = append(result, selectQueryResultRecord)
		}

	}

	return result
}

func main() {
	simpleWhere1 := NewSimpleWhere("id", "<", "3")
	simpleWhere2 := NewSimpleWhere("content", "=", "четыре")
	complexWhere := NewComplexWhereCondition("OR", simpleWhere1, simpleWhere2)
	q := Select("content", "id").From("index").Where(complexWhere)
	for _, r := range q.Do() {
		fmt.Println(r)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"io"
	"fmt"
	"log"
	"strings"
	"sync"
	"io/ioutil"
	"encoding/json"
	"encoding/csv"
	"github.com/PuerkitoBio/goquery"
	"github.com/spf13/cobra"
	"net/http"
	"errors"
)

const BASE_URL string = "https://m.avito.ru"
var IPBanned error = errors.New("You are banned by Avito!")

type Item struct {
	header   string
	location string
	url      string
	phone    string
}
var (
	Debug   *log.Logger
	Info    *log.Logger
	Error   *log.Logger
)

func perror(err error) {
	if err != nil {
		Error.Println(err)
		//		panic(err)
	}
}

func InitLoggers(verbose bool) {
	var infoHandle, errorHandle, debugHandle io.Writer

	if verbose {
		debugHandle = os.Stdout
	}else {
		debugHandle = ioutil.Discard
	}

	infoHandle = os.Stdout
	errorHandle = os.Stderr

	Debug = log.New(debugHandle, "DEBUG: ", 0)
	Info = log.New(infoHandle, "INFO: ", 0)
	Error = log.New(errorHandle, "ERROR: ", 0)
}

// Достаёт телефонный номер из JSON по заданному URL
func getPhone(phone_url, referer string) (string, error) {
	Debug.Println("Persing phone url:", phone_url)

	client := &http.Client{}
	req, err := http.NewRequest("GET", phone_url, nil)
	req.Header.Add("referer", referer)

	res, err := client.Do(req)
	perror(err)

	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	perror(err)
	if res.StatusCode == 403 {
		return "", IPBanned
	}

	phone_data := make(map[string]string)
	err = json.Unmarshal(body, &phone_data)
	perror(err)

	Debug.Println("Phone number:", phone_data["phone"])
	return phone_data["phone"], nil
}

func parseItem(item *Item, wg *sync.WaitGroup, items chan *Item) {
	doc, err := goquery.NewDocument(item.url)
	perror(err)

	doc.Find(".action-show-number").Each(func(i int, s *goquery.Selection) {
		phone_url, exists := s.Attr("href")
		if exists {
			Debug.Println("Found phone url:", phone_url)

			phone_url := strings.Join([]string{BASE_URL, phone_url, "?async"}, "")
			item.phone, err = getPhone(phone_url, item.url)
			if err != nil {
				Error.Println(err)
			}else {
				items <- item
			}

		}
	})
	wg.Done()
}

func saveToCSV(items chan *Item, wg *sync.WaitGroup) {
	w := csv.NewWriter(os.Stdout)

	for item := range items {
		record := []string{item.header, item.location, item.phone, item.url}
		if err := w.Write(record); err != nil {
			Error.Println("error writing record to csv:", err)
		}
	}

	// Write any buffered data to the underlying writer (standard output).
	w.Flush()

	if err := w.Error(); err != nil {
		log.Fatal(err)
	}

	wg.Done()
}

func main() {
	var query string
	var location string
	var category string
	var verbose bool
	var max_items int8

	var SelaAvitoCmd = &cobra.Command{
		Use: "selavito",
		Short: "Утилита для получения телефонных номеров с Avito (avito.ru)",
		Example: "selavito -l moskva -q macbook\nselavito -l sankt-peterburg -с rabota -q golang",

		Run: func(cmd *cobra.Command, args []string) {
			if query == "" {
				cmd.Help()
				return
			}

			InitLoggers(verbose)

			items := make(chan *Item)

			save_wg := new(sync.WaitGroup)
			parse_wg := new(sync.WaitGroup)

			save_wg.Add(1)
			go saveToCSV(items, save_wg)

			var search_url string

			// TODO: добавить паджинацию
			if category == "" {
				search_url = fmt.Sprintf("%s/%s?q=%s", BASE_URL, location, query)
			} else {
				search_url = fmt.Sprintf("%s/%s/%s?q=%s", BASE_URL, location, category, query)
			}

			Info.Println(search_url)

			doc, err := goquery.NewDocument(search_url)
			perror(err)

			counter := max_items

			doc.Find(".b-item").Each(func(i int, s *goquery.Selection) {
				if counter > 0 {
					item_url, exists := s.Find(".item-link").Attr("href")
					if exists {
						var item Item
						item.header = s.Find(".header-text").First().Text()
						item.location = s.Find(".info-location").First().Text()
						item.url = fmt.Sprintf("%s%s", BASE_URL, item_url)
						parse_wg.Add(1)
						go parseItem(&item, parse_wg, items)
						counter--
						Debug.Printf("%+v\n", item)
					}else {
						Error.Println(".item-link not found")
					}
				}
			})

			// Дожидаемся завершения работы всех парсеров...
			parse_wg.Wait()

			// ...и только после закрываем канал
			close(items)

			// Ждём пока данные окончательно сохранятся
			save_wg.Wait()
		},
	}

	SelaAvitoCmd.Flags().StringVarP(&query, "query", "q", "", "Строка для поиска")
	SelaAvitoCmd.Flags().StringVarP(&location, "location", "l", "rossiya",
		"Фильтр по региону (примеры: moskva, moskovskaya_oblast, sankt-peterburg)")
	SelaAvitoCmd.Flags().StringVarP(&category, "category", "c", "",
		"Фильтр по категории (примеры: nedvizhimost, transport, rabota, rezume, vakansii)")

	SelaAvitoCmd.Flags().BoolVarP(&verbose, "verbose", "v", false,
		"Выводит больше информации в консоль")

	SelaAvitoCmd.Flags().Int8VarP(&max_items, "max", "m", 1,
		"Максимальное количество элементов для поиска")

	SelaAvitoCmd.Execute()

}
<commit_msg>add pagenation<commit_after>package main

import (
	"os"
	"io"
	"fmt"
	"log"
	"strings"
	"sync"
	"io/ioutil"
	"encoding/json"
	"encoding/csv"
	"github.com/PuerkitoBio/goquery"
	"github.com/spf13/cobra"
	"net/http"
	"errors"
)

const BASE_URL string = "https://m.avito.ru"
var IPBanned error = errors.New("You are banned by Avito!")

type Item struct {
	header   string
	location string
	url      string
	phone    string
}
var (
	Debug   *log.Logger
	Info    *log.Logger
	Error   *log.Logger
)

func perror(err error) {
	if err != nil {
		Error.Println(err)
		//		panic(err)
	}
}

func InitLoggers(verbose bool) {
	var infoHandle, errorHandle, debugHandle io.Writer

	if verbose {
		debugHandle = os.Stdout
	}else {
		debugHandle = ioutil.Discard
	}

	infoHandle = os.Stdout
	errorHandle = os.Stderr

	Debug = log.New(debugHandle, "DEBUG: ", 0)
	Info = log.New(infoHandle, "INFO: ", 0)
	Error = log.New(errorHandle, "ERROR: ", 0)
}

// Достаёт телефонный номер из JSON по заданному URL
func getPhone(phone_url, referer string) (string, error) {
	Debug.Println("Persing phone url:", phone_url)

	client := &http.Client{}
	req, err := http.NewRequest("GET", phone_url, nil)
	req.Header.Add("referer", referer)

	res, err := client.Do(req)
	perror(err)

	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	perror(err)
	if res.StatusCode == 403 {
		return "", IPBanned
	}

	phone_data := make(map[string]string)
	err = json.Unmarshal(body, &phone_data)
	perror(err)

	Debug.Println("Phone number:", phone_data["phone"])
	return phone_data["phone"], nil
}

func parseItem(item *Item, wg *sync.WaitGroup, items chan *Item) {
	doc, err := goquery.NewDocument(item.url)
	perror(err)

	doc.Find(".action-show-number").Each(func(i int, s *goquery.Selection) {
		phone_url, exists := s.Attr("href")
		if exists {
			Debug.Println("Found phone url:", phone_url)

			phone_url := strings.Join([]string{BASE_URL, phone_url, "?async"}, "")
			item.phone, err = getPhone(phone_url, item.url)
			if err != nil {
				Error.Println(err)
			}else {
				items <- item
			}

		}
	})
	wg.Done()
}

func saveToCSV(items chan *Item, wg *sync.WaitGroup) {
	w := csv.NewWriter(os.Stdout)

	for item := range items {
		record := []string{item.header, item.location, item.phone, item.url}
		if err := w.Write(record); err != nil {
			Error.Println("error writing record to csv:", err)
		}
	}

	// Write any buffered data to the underlying writer (standard output).
	w.Flush()

	if err := w.Error(); err != nil {
		log.Fatal(err)
	}

	wg.Done()
}

func main() {
	var query string
	var location string
	var category string
	var verbose bool
	var max_items int8

	var SelaAvitoCmd = &cobra.Command{
		Use: "selavito",
		Short: "Утилита для получения телефонных номеров с Avito (avito.ru)",
		Example: "selavito -l moskva -q macbook\nselavito -l sankt-peterburg -с rabota -q golang",

		Run: func(cmd *cobra.Command, args []string) {
			if query == "" {
				cmd.Help()
				return
			}

			var page_url string
			counter := max_items
			items := make(chan *Item)
			save_wg := new(sync.WaitGroup)
			parse_wg := new(sync.WaitGroup)

			InitLoggers(verbose)

			save_wg.Add(1)
			go saveToCSV(items, save_wg)

			if category == "" {
				page_url = fmt.Sprintf("%s/%s?q=%s", BASE_URL, location, query)
			} else {
				page_url = fmt.Sprintf("%s/%s/%s?q=%s", BASE_URL, location, category, query)
			}

			for page_url != "" && counter > 0 {
				Info.Println(page_url)

				doc, err := goquery.NewDocument(page_url)
				perror(err)

				next_page_url, _ := doc.Find(".page-next").Find("a").First().Attr("href")
				next_page_url = fmt.Sprintf("%s%s", BASE_URL, next_page_url)
				Debug.Println("Next page:", next_page_url)

				doc.Find(".b-item").Each(func(i int, s *goquery.Selection) {
					if counter > 0 {
						item_url, exists := s.Find(".item-link").Attr("href")
						if exists {
							var item Item
							item.header = s.Find(".header-text").First().Text()
							item.location = s.Find(".info-location").First().Text()
							item.url = fmt.Sprintf("%s%s", BASE_URL, item_url)
							parse_wg.Add(1)
							go parseItem(&item, parse_wg, items)
							counter--
							Debug.Printf("%+v\n", item)
						}else {
							Error.Println(".item-link not found")
						}
					}
				})
				page_url = next_page_url
			}

			// Дожидаемся завершения работы всех парсеров...
			parse_wg.Wait()

			// ...и только после закрываем канал
			close(items)

			// Ждём пока данные окончательно сохранятся
			save_wg.Wait()
		},
	}

	SelaAvitoCmd.Flags().StringVarP(&query, "query", "q", "", "Строка для поиска")
	SelaAvitoCmd.Flags().StringVarP(&location, "location", "l", "rossiya",
		"Фильтр по региону (примеры: moskva, moskovskaya_oblast, sankt-peterburg)")
	SelaAvitoCmd.Flags().StringVarP(&category, "category", "c", "",
		"Фильтр по категории (примеры: nedvizhimost, transport, rabota, rezume, vakansii)")

	SelaAvitoCmd.Flags().BoolVarP(&verbose, "verbose", "v", false,
		"Более подробный вывод в консоль")

	SelaAvitoCmd.Flags().Int8VarP(&max_items, "max", "m", 1,
		"Максимальное количество элементов для поиска")

	SelaAvitoCmd.Execute()

}
<|endoftext|>


<commit_before>// Copyright 2014 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ebiten

import (
	"math"

	"github.com/hajimehoshi/ebiten/internal/clock"
	"github.com/hajimehoshi/ebiten/internal/hooks"
	"github.com/hajimehoshi/ebiten/internal/restorable"
	"github.com/hajimehoshi/ebiten/internal/ui"
	"github.com/hajimehoshi/ebiten/internal/web"
)

func newGraphicsContext(f func(*Image) error) *graphicsContext {
	return &graphicsContext{
		f: f,
	}
}

type graphicsContext struct {
	f           func(*Image) error
	offscreen   *Image
	offscreen2  *Image // TODO: better name
	screen      *Image
	initialized bool
	invalidated bool // browser only
}

func (c *graphicsContext) Invalidate() {
	// Note that this is called only on browsers so far.
	// TODO: On mobiles, this function is not called and instead IsTexture is called
	// to detect if the context is lost. This is simple but might not work on some platforms.
	// Should Invalidate be called explicitly?
	c.invalidated = true
}

func (c *graphicsContext) SetSize(screenWidth, screenHeight int, screenScale float64) {
	if c.screen != nil {
		_ = c.screen.Dispose()
	}
	if c.offscreen != nil {
		_ = c.offscreen.Dispose()
	}
	if c.offscreen2 != nil {
		_ = c.offscreen2.Dispose()
	}
	offscreen := newVolatileImage(screenWidth, screenHeight, FilterNearest)

	intScreenScale := int(math.Ceil(screenScale))
	w := screenWidth * intScreenScale
	h := screenHeight * intScreenScale
	offscreen2 := newVolatileImage(w, h, FilterLinear)

	w = int(float64(screenWidth) * screenScale)
	h = int(float64(screenHeight) * screenScale)
	ox, oy := ui.ScreenOffset()
	c.screen = newImageWithScreenFramebuffer(w, h, ox, oy)
	_ = c.screen.Clear()

	c.offscreen = offscreen
	c.offscreen2 = offscreen2
}

func (c *graphicsContext) initializeIfNeeded() error {
	if !c.initialized {
		if err := restorable.InitializeGLState(); err != nil {
			return err
		}
		c.initialized = true
	}
	if err := c.restoreIfNeeded(); err != nil {
		return err
	}
	return nil
}

func drawWithFittingScale(dst *Image, src *Image) {
	wd, hd := dst.Size()
	ws, hs := src.Size()
	sw := float64(wd) / float64(ws)
	sh := float64(hd) / float64(hs)
	op := &DrawImageOptions{}
	op.GeoM.Scale(sw, sh)
	_ = dst.DrawImage(src, op)
}

func (c *graphicsContext) Update(afterFrameUpdate func()) error {
	updateCount := clock.Update()

	if err := c.initializeIfNeeded(); err != nil {
		return err
	}
	for i := 0; i < updateCount; i++ {
		restorable.ClearVolatileImages()
		setRunningSlowly(i < updateCount-1)
		if err := hooks.Run(); err != nil {
			return err
		}
		if err := c.f(c.offscreen); err != nil {
			return err
		}
		afterFrameUpdate()
	}
	if 0 < updateCount {
		drawWithFittingScale(c.offscreen2, c.offscreen)
		_ = c.screen.Clear()
		drawWithFittingScale(c.screen, c.offscreen2)
	}

	if err := restorable.ResolveStaleImages(); err != nil {
		return err
	}
	return nil
}

func (c *graphicsContext) needsRestoring() (bool, error) {
	if web.IsBrowser() {
		return c.invalidated, nil
	}
	return c.offscreen.restorable.IsInvalidated()
}

func (c *graphicsContext) restoreIfNeeded() error {
	if !restorable.IsRestoringEnabled() {
		return nil
	}
	r, err := c.needsRestoring()
	if err != nil {
		return err
	}
	if !r {
		return nil
	}
	if err := restorable.Restore(); err != nil {
		return err
	}
	c.invalidated = false
	return nil
}
<commit_msg>graphics: Speed optimization: Use CompositeModeCopy instead of Clear<commit_after>// Copyright 2014 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ebiten

import (
	"math"

	"github.com/hajimehoshi/ebiten/internal/clock"
	"github.com/hajimehoshi/ebiten/internal/hooks"
	"github.com/hajimehoshi/ebiten/internal/restorable"
	"github.com/hajimehoshi/ebiten/internal/ui"
	"github.com/hajimehoshi/ebiten/internal/web"
)

func newGraphicsContext(f func(*Image) error) *graphicsContext {
	return &graphicsContext{
		f: f,
	}
}

type graphicsContext struct {
	f           func(*Image) error
	offscreen   *Image
	offscreen2  *Image // TODO: better name
	screen      *Image
	initialized bool
	invalidated bool // browser only
}

func (c *graphicsContext) Invalidate() {
	// Note that this is called only on browsers so far.
	// TODO: On mobiles, this function is not called and instead IsTexture is called
	// to detect if the context is lost. This is simple but might not work on some platforms.
	// Should Invalidate be called explicitly?
	c.invalidated = true
}

func (c *graphicsContext) SetSize(screenWidth, screenHeight int, screenScale float64) {
	if c.screen != nil {
		_ = c.screen.Dispose()
	}
	if c.offscreen != nil {
		_ = c.offscreen.Dispose()
	}
	if c.offscreen2 != nil {
		_ = c.offscreen2.Dispose()
	}
	offscreen := newVolatileImage(screenWidth, screenHeight, FilterNearest)

	intScreenScale := int(math.Ceil(screenScale))
	w := screenWidth * intScreenScale
	h := screenHeight * intScreenScale
	offscreen2 := newVolatileImage(w, h, FilterLinear)

	w = int(float64(screenWidth) * screenScale)
	h = int(float64(screenHeight) * screenScale)
	ox, oy := ui.ScreenOffset()
	c.screen = newImageWithScreenFramebuffer(w, h, ox, oy)
	_ = c.screen.Clear()

	c.offscreen = offscreen
	c.offscreen2 = offscreen2
}

func (c *graphicsContext) initializeIfNeeded() error {
	if !c.initialized {
		if err := restorable.InitializeGLState(); err != nil {
			return err
		}
		c.initialized = true
	}
	if err := c.restoreIfNeeded(); err != nil {
		return err
	}
	return nil
}

func drawWithFittingScale(dst *Image, src *Image) {
	wd, hd := dst.Size()
	ws, hs := src.Size()
	sw := float64(wd) / float64(ws)
	sh := float64(hd) / float64(hs)
	op := &DrawImageOptions{}
	op.GeoM.Scale(sw, sh)
	op.CompositeMode = CompositeModeCopy
	_ = dst.DrawImage(src, op)
}

func (c *graphicsContext) Update(afterFrameUpdate func()) error {
	updateCount := clock.Update()

	if err := c.initializeIfNeeded(); err != nil {
		return err
	}
	for i := 0; i < updateCount; i++ {
		restorable.ClearVolatileImages()
		setRunningSlowly(i < updateCount-1)
		if err := hooks.Run(); err != nil {
			return err
		}
		if err := c.f(c.offscreen); err != nil {
			return err
		}
		afterFrameUpdate()
	}
	if 0 < updateCount {
		drawWithFittingScale(c.offscreen2, c.offscreen)
		drawWithFittingScale(c.screen, c.offscreen2)
	}

	if err := restorable.ResolveStaleImages(); err != nil {
		return err
	}
	return nil
}

func (c *graphicsContext) needsRestoring() (bool, error) {
	if web.IsBrowser() {
		return c.invalidated, nil
	}
	return c.offscreen.restorable.IsInvalidated()
}

func (c *graphicsContext) restoreIfNeeded() error {
	if !restorable.IsRestoringEnabled() {
		return nil
	}
	r, err := c.needsRestoring()
	if err != nil {
		return err
	}
	if !r {
		return nil
	}
	if err := restorable.Restore(); err != nil {
		return err
	}
	c.invalidated = false
	return nil
}
<|endoftext|>