<commit_before>package qln

import (
	"fmt"

	"github.com/mit-dci/lit/lnutil"

	"github.com/adiabat/btcd/wire"
	"github.com/adiabat/btcutil/txsort"
)

// GetStateIdxFromTx returns the state index from a commitment transaction.
// No errors; returns 0 if there is no retrievable index.
// Takes the xor input X which is derived from the 0th elkrems.
func GetStateIdxFromTx(tx *wire.MsgTx, x uint64) uint64 {
	// no tx, so no index
	if tx == nil {
		return 0
	}
	// more than 1 input, so not a close tx
	if len(tx.TxIn) != 1 {
		return 0
	}
	// mask need two high bytes of 0s
	if x > 1<<48 {
		return 0
	}
	// check that indicating high bytes are correct.  If not, return 0
	if tx.TxIn[0].Sequence>>24 != 0xff || tx.LockTime>>24 != 0x21 {
		//		fmt.Printf("sequence byte %x, locktime byte %x\n",
		//			tx.TxIn[0].Sequence>>24, tx.LockTime>>24 != 0x21)
		return 0
	}
	// high 24 bits sequence, low 24 bits locktime
	seqBits := uint64(tx.TxIn[0].Sequence & 0x00ffffff)
	timeBits := uint64(tx.LockTime & 0x00ffffff)

	return (seqBits<<24 | timeBits) ^ x
}

// SetStateIdxBits modifies the tx in place, setting the sequence and locktime
// fields to indicate the given state index.
func SetStateIdxBits(tx *wire.MsgTx, idx, x uint64) error {
	if tx == nil {
		return fmt.Errorf("SetStateIdxBits: nil tx")
	}
	if len(tx.TxIn) != 1 {
		return fmt.Errorf("SetStateIdxBits: tx has %d inputs", len(tx.TxIn))
	}
	if idx >= 1<<48 {
		return fmt.Errorf(
			"SetStateIdxBits: index %d greater than max %d", idx, uint64(1<<48)-1)
	}

	idx = idx ^ x
	// high 24 bits sequence, low 24 bits locktime
	seqBits := uint32(idx >> 24)
	timeBits := uint32(idx & 0x00ffffff)

	tx.TxIn[0].Sequence = seqBits | seqMask
	tx.LockTime = timeBits | timeMask

	return nil
}

// SimpleCloseTx produces a close tx based on the current state.
// The PKH addresses are my refund base with their r-elkrem point, and
// their refund base with my r-elkrem point.  "Their" point means they have
// the point but not the scalar.
func (q *Qchan) SimpleCloseTx() (*wire.MsgTx, error) {
	// sanity checks
	if q == nil || q.State == nil {
		return nil, fmt.Errorf("SimpleCloseTx: nil chan / state")
	}
	fee := q.State.Fee // symmetric fee

	// make my output
	myScript := lnutil.DirectWPKHScript(q.MyRefundPub)
	myOutput := wire.NewTxOut(q.State.MyAmt-fee, myScript)
	// make their output
	theirScript := lnutil.DirectWPKHScript(q.TheirRefundPub)
	theirOutput := wire.NewTxOut((q.Value-q.State.MyAmt)-fee, theirScript)

	// make tx with these outputs
	tx := wire.NewMsgTx()
	tx.AddTxOut(myOutput)
	tx.AddTxOut(theirOutput)
	// add channel outpoint as txin
	tx.AddTxIn(wire.NewTxIn(&q.Op, nil, nil))
	// sort and return
	txsort.InPlaceSort(tx)
	return tx, nil
}

// BuildStateTx constructs and returns a state tx.  As simple as I can make it.
// This func just makes the tx with data from State in ram, and HAKD key arg
func (q *Qchan) BuildStateTx(mine bool) (*wire.MsgTx, error) {
	if q == nil {
		return nil, fmt.Errorf("BuildStateTx: nil chan")
	}
	// sanity checks
	s := q.State // use it a lot, make shorthand variable
	if s == nil {
		return nil, fmt.Errorf("channel (%d,%d) has no state", q.KeyGen.Step[3], q.KeyGen.Step[4])
	}
	// if delta is non-zero, something is wrong.
	// this is old? remove...
	/*	if s.Delta != 0 {
		return nil, fmt.Errorf(
			"BuildStateTx: delta is %d (expect 0)", s.Delta)
	}*/
	var fancyAmt, pkhAmt int64   // output amounts
	var revPub, timePub [33]byte // pubkeys
	var pkhPub [33]byte          // the simple output's pub key hash

	fee := s.Fee // fixed fee for now

	// the PKH clear refund also has elkrem points added to mask the PKH.
	// this changes the txouts at each state to blind sorceror better.
	if mine { // build MY tx (to verify) (unless breaking)
		// My tx that I store.  They get funds unencumbered. SH is mine eventually
		// SH pubkeys are base points combined with the elk point we give them
		// Create latest elkrem point (the one I create)
		curElk, err := q.ElkPoint(false, q.State.StateIdx)
		if err != nil {
			return nil, err
		}
		revPub = lnutil.CombinePubs(q.TheirHAKDBase, curElk)
		timePub = lnutil.AddPubsEZ(q.MyHAKDBase, curElk)

		pkhPub = q.TheirRefundPub
		pkhAmt = (q.Value - s.MyAmt) - fee
		fancyAmt = s.MyAmt - fee

	} else { // build THEIR tx (to sign)
		// Their tx that they store.  I get funds PKH.  SH is theirs eventually.
		fmt.Printf("using elkpoint %x\n", s.ElkPoint)
		// SH pubkeys are our base points plus the received elk point
		revPub = lnutil.CombinePubs(q.MyHAKDBase, s.ElkPoint)
		timePub = lnutil.AddPubsEZ(q.TheirHAKDBase, s.ElkPoint)

		fancyAmt = (q.Value - s.MyAmt) - fee

		// PKH output
		pkhPub = q.MyRefundPub
		pkhAmt = s.MyAmt - fee
	}

	// now that everything is chosen, build fancy script and pkh script
	fancyScript := lnutil.CommitScript(revPub, timePub, q.Delay)
	pkhScript := lnutil.DirectWPKHScript(pkhPub) // p2wpkh-ify

	fmt.Printf("> made SH script, state %d\n", s.StateIdx)
	fmt.Printf("\t revPub %x timeout pub %x \n", revPub, timePub)
	fmt.Printf("\t script %x ", fancyScript)

	fancyScript = lnutil.P2WSHify(fancyScript) // p2wsh-ify

	fmt.Printf("\t scripthash %x\n", fancyScript)

	// create txouts by assigning amounts
	outFancy := wire.NewTxOut(fancyAmt, fancyScript)
	outPKH := wire.NewTxOut(pkhAmt, pkhScript)

	fmt.Printf("\tcombined refund %x, pkh %x\n", pkhPub, outPKH.PkScript)

	// make a new tx
	tx := wire.NewMsgTx()
	// add txouts
	tx.AddTxOut(outFancy)
	tx.AddTxOut(outPKH)
	// add unsigned txin
	tx.AddTxIn(wire.NewTxIn(&q.Op, nil, nil))
	// set index hints

	// state 0 and 1 can't use mask?  Think they can now.
	SetStateIdxBits(tx, s.StateIdx, q.GetChanHint(mine))

	// sort outputs
	txsort.InPlaceSort(tx)
	return tx, nil
}

// the scriptsig to put on a P2SH input.  Sigs need to be in order!
func SpendMultiSigWitStack(pre, sigA, sigB []byte) [][]byte {

	witStack := make([][]byte, 4)

	witStack[0] = nil // it's not an OP_0 !!!! argh!
	witStack[1] = sigA
	witStack[2] = sigB
	witStack[3] = pre

	return witStack
}
<commit_msg>add dust cutoff to buildtx<commit_after>package qln

import (
	"fmt"

	"github.com/mit-dci/lit/lnutil"

	"github.com/adiabat/btcd/wire"
	"github.com/adiabat/btcutil/txsort"
)

// GetStateIdxFromTx returns the state index from a commitment transaction.
// No errors; returns 0 if there is no retrievable index.
// Takes the xor input X which is derived from the 0th elkrems.
func GetStateIdxFromTx(tx *wire.MsgTx, x uint64) uint64 {
	// no tx, so no index
	if tx == nil {
		return 0
	}
	// more than 1 input, so not a close tx
	if len(tx.TxIn) != 1 {
		return 0
	}
	// mask need two high bytes of 0s
	if x > 1<<48 {
		return 0
	}
	// check that indicating high bytes are correct.  If not, return 0
	if tx.TxIn[0].Sequence>>24 != 0xff || tx.LockTime>>24 != 0x21 {
		//		fmt.Printf("sequence byte %x, locktime byte %x\n",
		//			tx.TxIn[0].Sequence>>24, tx.LockTime>>24 != 0x21)
		return 0
	}
	// high 24 bits sequence, low 24 bits locktime
	seqBits := uint64(tx.TxIn[0].Sequence & 0x00ffffff)
	timeBits := uint64(tx.LockTime & 0x00ffffff)

	return (seqBits<<24 | timeBits) ^ x
}

// SetStateIdxBits modifies the tx in place, setting the sequence and locktime
// fields to indicate the given state index.
func SetStateIdxBits(tx *wire.MsgTx, idx, x uint64) error {
	if tx == nil {
		return fmt.Errorf("SetStateIdxBits: nil tx")
	}
	if len(tx.TxIn) != 1 {
		return fmt.Errorf("SetStateIdxBits: tx has %d inputs", len(tx.TxIn))
	}
	if idx >= 1<<48 {
		return fmt.Errorf(
			"SetStateIdxBits: index %d greater than max %d", idx, uint64(1<<48)-1)
	}

	idx = idx ^ x
	// high 24 bits sequence, low 24 bits locktime
	seqBits := uint32(idx >> 24)
	timeBits := uint32(idx & 0x00ffffff)

	tx.TxIn[0].Sequence = seqBits | seqMask
	tx.LockTime = timeBits | timeMask

	return nil
}

// SimpleCloseTx produces a close tx based on the current state.
// The PKH addresses are my refund base with their r-elkrem point, and
// their refund base with my r-elkrem point.  "Their" point means they have
// the point but not the scalar.
func (q *Qchan) SimpleCloseTx() (*wire.MsgTx, error) {
	// sanity checks
	if q == nil || q.State == nil {
		return nil, fmt.Errorf("SimpleCloseTx: nil chan / state")
	}
	fee := q.State.Fee // symmetric fee

	// make my output
	myScript := lnutil.DirectWPKHScript(q.MyRefundPub)
	myOutput := wire.NewTxOut(q.State.MyAmt-fee, myScript)
	// make their output
	theirScript := lnutil.DirectWPKHScript(q.TheirRefundPub)
	theirOutput := wire.NewTxOut((q.Value-q.State.MyAmt)-fee, theirScript)

	// make tx with these outputs
	tx := wire.NewMsgTx()
	tx.AddTxOut(myOutput)
	tx.AddTxOut(theirOutput)
	// add channel outpoint as txin
	tx.AddTxIn(wire.NewTxIn(&q.Op, nil, nil))
	// sort and return
	txsort.InPlaceSort(tx)
	return tx, nil
}

// BuildStateTx constructs and returns a state tx.  As simple as I can make it.
// This func just makes the tx with data from State in ram, and HAKD key arg
func (q *Qchan) BuildStateTx(mine bool) (*wire.MsgTx, error) {
	dustLimit := int64(10000) // hardcoded dust limit, change later

	if q == nil {
		return nil, fmt.Errorf("BuildStateTx: nil chan")
	}
	// sanity checks
	s := q.State // use it a lot, make shorthand variable
	if s == nil {
		return nil, fmt.Errorf("channel (%d,%d) has no state", q.KeyGen.Step[3], q.KeyGen.Step[4])
	}
	// if delta is non-zero, something is wrong.
	// this is old? remove...
	/*	if s.Delta != 0 {
		return nil, fmt.Errorf(
			"BuildStateTx: delta is %d (expect 0)", s.Delta)
	}*/
	var fancyAmt, pkhAmt int64   // output amounts
	var revPub, timePub [33]byte // pubkeys
	var pkhPub [33]byte          // the simple output's pub key hash

	fee := s.Fee // fixed fee for now

	// the PKH clear refund also has elkrem points added to mask the PKH.
	// this changes the txouts at each state to blind sorceror better.
	if mine { // build MY tx (to verify) (unless breaking)
		// My tx that I store.  They get funds unencumbered. SH is mine eventually
		// SH pubkeys are base points combined with the elk point we give them
		// Create latest elkrem point (the one I create)
		curElk, err := q.ElkPoint(false, q.State.StateIdx)
		if err != nil {
			return nil, err
		}
		revPub = lnutil.CombinePubs(q.TheirHAKDBase, curElk)
		timePub = lnutil.AddPubsEZ(q.MyHAKDBase, curElk)

		pkhPub = q.TheirRefundPub
		pkhAmt = (q.Value - s.MyAmt) - fee
		fancyAmt = s.MyAmt - fee
	} else { // build THEIR tx (to sign)
		// Their tx that they store.  I get funds PKH.  SH is theirs eventually.
		fmt.Printf("using elkpoint %x\n", s.ElkPoint)
		// SH pubkeys are our base points plus the received elk point
		revPub = lnutil.CombinePubs(q.MyHAKDBase, s.ElkPoint)
		timePub = lnutil.AddPubsEZ(q.TheirHAKDBase, s.ElkPoint)

		fancyAmt = (q.Value - s.MyAmt) - fee

		// PKH output
		pkhPub = q.MyRefundPub
		pkhAmt = s.MyAmt - fee
	}

	// now that everything is chosen, build fancy script and pkh script
	fancyScript := lnutil.CommitScript(revPub, timePub, q.Delay)
	pkhScript := lnutil.DirectWPKHScript(pkhPub) // p2wpkh-ify

	fmt.Printf("> made SH script, state %d\n", s.StateIdx)
	fmt.Printf("\t revPub %x timeout pub %x \n", revPub, timePub)
	fmt.Printf("\t script %x ", fancyScript)

	fancyScript = lnutil.P2WSHify(fancyScript) // p2wsh-ify

	fmt.Printf("\t scripthash %x\n", fancyScript)

	// create txouts by assigning amounts
	outFancy := wire.NewTxOut(fancyAmt, fancyScript)
	outPKH := wire.NewTxOut(pkhAmt, pkhScript)

	fmt.Printf("\tcombined refund %x, pkh %x\n", pkhPub, outPKH.PkScript)

	// make a new tx
	tx := wire.NewMsgTx()
	// add txouts
	if fancyAmt > dustLimit {
		tx.AddTxOut(outFancy)
	}

	if pkhAmt > dustLimit {
		tx.AddTxOut(outPKH)
	}

	if len(tx.TxOut) < 1 {
		return nil, fmt.Errorf("No outputs, all below dust limit")
	}

	// add unsigned txin
	tx.AddTxIn(wire.NewTxIn(&q.Op, nil, nil))
	// set index hints

	// state 0 and 1 can't use mask?  Think they can now.
	SetStateIdxBits(tx, s.StateIdx, q.GetChanHint(mine))

	// sort outputs
	txsort.InPlaceSort(tx)
	return tx, nil
}

// the scriptsig to put on a P2SH input.  Sigs need to be in order!
func SpendMultiSigWitStack(pre, sigA, sigB []byte) [][]byte {

	witStack := make([][]byte, 4)

	witStack[0] = nil // it's not an OP_0 !!!! argh!
	witStack[1] = sigA
	witStack[2] = sigB
	witStack[3] = pre

	return witStack
}
<|endoftext|>


<commit_before>package ali_mns

import (
	"sync/atomic"
	"time"
)

type QPSMonitor struct {
	latestIndex  int32
	delaySecond  int32
	totalQueries []int32
}

func (p *QPSMonitor) Pulse() {
	index := int32(time.Now().Second()) % p.delaySecond

	if p.latestIndex != index {
		atomic.StoreInt32(&p.latestIndex, index)
		atomic.StoreInt32(&p.totalQueries[p.latestIndex], 0)
	}

	atomic.AddInt32(&p.totalQueries[index], 1)
}

func (p *QPSMonitor) QPS() int32 {
	var totalCount int32 = 0
	for _, queryCount := range p.totalQueries {
		totalCount += queryCount
	}
	return totalCount / p.delaySecond
}

func NewQPSMonitor(delaySecond int32) *QPSMonitor {
	if delaySecond < 5 {
		delaySecond = 5
	}
	monitor := QPSMonitor{
		delaySecond:  delaySecond,
		totalQueries: make([]int32, delaySecond),
	}
	return &monitor
}
<commit_msg>improve qps alog<commit_after>package ali_mns

import (
	"sync/atomic"
	"time"
)

type QPSMonitor struct {
	latestIndex  int32
	delaySecond  int32
	totalQueries []int32
}

func (p *QPSMonitor) Pulse() {
	index := int32(time.Now().Second()) % p.delaySecond

	if p.latestIndex != index {
		atomic.StoreInt32(&p.latestIndex, index)
		atomic.StoreInt32(&p.totalQueries[p.latestIndex], 0)
	}

	atomic.AddInt32(&p.totalQueries[index], 1)
}

func (p *QPSMonitor) QPS() int32 {
	var totalCount int32 = 0
	for i, queryCount := range p.totalQueries {
		if int32(i) != p.latestIndex {
			totalCount += queryCount
		}
	}
	return totalCount / (p.delaySecond - 1)
}

func NewQPSMonitor(delaySecond int32) *QPSMonitor {
	if delaySecond < 5 {
		delaySecond = 5
	}
	monitor := QPSMonitor{
		delaySecond:  delaySecond,
		totalQueries: make([]int32, delaySecond),
	}
	return &monitor
}
<|endoftext|>


<commit_before>package queue

import (
	"crypto/rand"
	"errors"
	"sync"
	"time"
)

const MaxReserTime = 30

var ERROR_HASH_NOT_FOUND = errors.New("Hash not found")
var ERROR_ALREADY_RESERVED = errors.New("Item already reserved")
var ERROR_NOT_RESERVED = errors.New("Item not reserved")
var ERROR_NO_ITEMS_AVALIABLE = errors.New("No items available")

// Item struct is the basic queue item
type Item struct {
	ReservedAt time.Time `json:"reserved_at"`
	Tag        string    `json:"tag"`
	Value      []byte    `json:"value"`
}

// Data contains all data in the current queue
type Data struct {
	Tag      string          `json:"tag"`
	ItemList map[string]Item `json:"item"`
	sync.RWMutex
}

// New queue
func New() (q *Data, err error) {
	q = &Data{
		ItemList: make(map[string]Item),
	}
	return
}

// Count items in the queue
func (q *Data) Count() int {
	return len(q.ItemList)
}

// Put data in the queue
func (q *Data) Put(b []byte) {
	q.Lock()
	defer q.Unlock()
	q.ItemList[randStr()] = Item{Value: b}
}

// Reserve an item to be processed.
// If the item is not removed or the reservation time is not
// renewed, the item will returns to the queue automatically
func (q *Data) Reserve(hash string) (item Item, err error) {
	q.Lock()
	defer q.Unlock()
	v, ok := q.ItemList[hash]
	if !ok {
		err = ERROR_HASH_NOT_FOUND
		return
	}
	now := time.Now()
	diff := now.Sub(v.ReservedAt)
	if diff.Seconds() < MaxReserTime {
		err = ERROR_ALREADY_RESERVED
		return
	}
	v.ReservedAt = now
	item = v
	q.ItemList[hash] = v
	return
}

// Remove item from the queue, the item must be reserved
func (q *Data) Remove(hash string) (err error) {
	q.Lock()
	defer q.Unlock()
	v, ok := q.ItemList[hash]
	if !ok {
		err = ERROR_HASH_NOT_FOUND
		return
	}
	diff := time.Since(v.ReservedAt)
	if diff.Seconds() >= MaxReserTime {
		err = ERROR_NOT_RESERVED
		return
	}
	delete(q.ItemList, hash)
	return
}

// Renew the reservation of an item in the queue
func (q *Data) Renew(hash string) (err error) {
	q.Lock()
	defer q.Unlock()
	v, ok := q.ItemList[hash]
	if !ok {
		err = ERROR_HASH_NOT_FOUND
		return
	}
	now := time.Now()
	diff := now.Sub(v.ReservedAt)
	if diff.Seconds() >= MaxReserTime {
		err = ERROR_NOT_RESERVED
		return
	}
	v.ReservedAt = now
	q.ItemList[hash] = v
	return
}

// ReserveNext searches for the next available item in the queue
func (q *Data) ReserveNext() (hash string, value []byte, err error) {
	q.Lock()
	defer q.Unlock()
	for k, v := range q.ItemList {
		now := time.Now()
		diff := now.Sub(v.ReservedAt)
		if diff.Seconds() > MaxReserTime {
			v.ReservedAt = now
			q.ItemList[k] = v
			value = v.Value
			hash = k
			return
		}
	}
	err = ERROR_NO_ITEMS_AVALIABLE
	return
}

// List all items in the queue
func (q *Data) List() (hashList []string) {
	q.RLock()
	defer q.RUnlock()
	for k := range q.ItemList {
		hashList = append(hashList, k)
	}

	return
}

func randStr() (ret string) {
	const charList = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
	const clLen byte = 62
	const size = 36
	var bytes = make([]byte, size)
	if _, err := rand.Read(bytes); err != nil {
		panic(err.Error())
	}
	for k, v := range bytes {
		bytes[k] = charList[v%clLen]
	}
	ret = string(bytes)
	return
}
<commit_msg>remove tag field<commit_after>package queue

import (
	"crypto/rand"
	"errors"
	"sync"
	"time"
)

const MaxReserTime = 30

var ERROR_HASH_NOT_FOUND = errors.New("Hash not found")
var ERROR_ALREADY_RESERVED = errors.New("Item already reserved")
var ERROR_NOT_RESERVED = errors.New("Item not reserved")
var ERROR_NO_ITEMS_AVALIABLE = errors.New("No items available")

// Item struct is the basic queue item
type Item struct {
	ReservedAt time.Time `json:"reserved_at"`
	Value      []byte    `json:"value"`
}

// Data contains all data in the current queue
type Data struct {
	ItemList map[string]Item `json:"item"`
	sync.RWMutex
}

// New queue
func New() (q *Data, err error) {
	q = &Data{
		ItemList: make(map[string]Item),
	}
	return
}

// Count items in the queue
func (q *Data) Count() int {
	return len(q.ItemList)
}

// Put data in the queue
func (q *Data) Put(b []byte) {
	q.Lock()
	defer q.Unlock()
	q.ItemList[randStr()] = Item{Value: b}
}

// Reserve an item to be processed.
// If the item is not removed or the reservation time is not
// renewed, the item will returns to the queue automatically
func (q *Data) Reserve(hash string) (item Item, err error) {
	q.Lock()
	defer q.Unlock()
	v, ok := q.ItemList[hash]
	if !ok {
		err = ERROR_HASH_NOT_FOUND
		return
	}
	now := time.Now()
	diff := now.Sub(v.ReservedAt)
	if diff.Seconds() < MaxReserTime {
		err = ERROR_ALREADY_RESERVED
		return
	}
	v.ReservedAt = now
	item = v
	q.ItemList[hash] = v
	return
}

// Remove item from the queue, the item must be reserved
func (q *Data) Remove(hash string) (err error) {
	q.Lock()
	defer q.Unlock()
	v, ok := q.ItemList[hash]
	if !ok {
		err = ERROR_HASH_NOT_FOUND
		return
	}
	diff := time.Since(v.ReservedAt)
	if diff.Seconds() >= MaxReserTime {
		err = ERROR_NOT_RESERVED
		return
	}
	delete(q.ItemList, hash)
	return
}

// Renew the reservation of an item in the queue
func (q *Data) Renew(hash string) (err error) {
	q.Lock()
	defer q.Unlock()
	v, ok := q.ItemList[hash]
	if !ok {
		err = ERROR_HASH_NOT_FOUND
		return
	}
	now := time.Now()
	diff := now.Sub(v.ReservedAt)
	if diff.Seconds() >= MaxReserTime {
		err = ERROR_NOT_RESERVED
		return
	}
	v.ReservedAt = now
	q.ItemList[hash] = v
	return
}

// ReserveNext searches for the next available item in the queue
func (q *Data) ReserveNext() (hash string, value []byte, err error) {
	q.Lock()
	defer q.Unlock()
	for k, v := range q.ItemList {
		now := time.Now()
		diff := now.Sub(v.ReservedAt)
		if diff.Seconds() > MaxReserTime {
			v.ReservedAt = now
			q.ItemList[k] = v
			value = v.Value
			hash = k
			return
		}
	}
	err = ERROR_NO_ITEMS_AVALIABLE
	return
}

// List all items in the queue
func (q *Data) List() (hashList []string) {
	q.RLock()
	defer q.RUnlock()
	for k := range q.ItemList {
		hashList = append(hashList, k)
	}

	return
}

func randStr() (ret string) {
	const charList = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
	const clLen byte = 62
	const size = 36
	var bytes = make([]byte, size)
	if _, err := rand.Read(bytes); err != nil {
		panic(err.Error())
	}
	for k, v := range bytes {
		bytes[k] = charList[v%clLen]
	}
	ret = string(bytes)
	return
}
<|endoftext|>


<commit_before>package rados

// #cgo LDFLAGS: -lrados
// #include <errno.h>
// #include <stdlib.h>
// #include <rados/librados.h>
import "C"

import (
	"fmt"
	"runtime"
	"unsafe"

	"github.com/ceph/go-ceph/errutil"
)

type RadosError int

func (e RadosError) Error() string {
	errno, s := errutil.FormatErrno(int(e))
	if s == "" {
		return fmt.Sprintf("rados: ret=%d", errno)
	}
	return fmt.Sprintf("rados: ret=%d, %s", errno, s)
}

var RadosAllNamespaces = C.LIBRADOS_ALL_NSPACES

var RadosErrorNotFound = RadosError(-C.ENOENT)
var RadosErrorPermissionDenied = RadosError(-C.EPERM)

func getRadosError(err int) error {
	if err == 0 {
		return nil
	}
	return RadosError(err)
}

// Version returns the major, minor, and patch components of the version of
// the RADOS library linked against.
func Version() (int, int, int) {
	var c_major, c_minor, c_patch C.int
	C.rados_version(&c_major, &c_minor, &c_patch)
	return int(c_major), int(c_minor), int(c_patch)
}

func makeConn() *Conn {
	return &Conn{connected: false}
}

func newConn(user *C.char) (*Conn, error) {
	conn := makeConn()
	ret := C.rados_create(&conn.cluster, user)

	if ret != 0 {
		return nil, RadosError(int(ret))
	}

	runtime.SetFinalizer(conn, freeConn)
	return conn, nil
}

// NewConn creates a new connection object. It returns the connection and an
// error, if any.
func NewConn() (*Conn, error) {
	return newConn(nil)
}

// NewConnWithUser creates a new connection object with a custom username.
// It returns the connection and an error, if any.
func NewConnWithUser(user string) (*Conn, error) {
	c_user := C.CString(user)
	defer C.free(unsafe.Pointer(c_user))
	return newConn(c_user)
}

// NewConnWithClusterAndUser creates a new connection object for a specific cluster and username.
// It returns the connection and an error, if any.
func NewConnWithClusterAndUser(clusterName string, userName string) (*Conn, error) {
	c_cluster_name := C.CString(clusterName)
	defer C.free(unsafe.Pointer(c_cluster_name))

	c_name := C.CString(userName)
	defer C.free(unsafe.Pointer(c_name))

	conn := makeConn()
	ret := C.rados_create2(&conn.cluster, c_cluster_name, c_name, 0)
	if ret != 0 {
		return nil, RadosError(int(ret))
	}

	runtime.SetFinalizer(conn, freeConn)
	return conn, nil
}

// freeConn releases resources that are allocated while configuring the
// connection to the cluster. rados_shutdown() should only be needed after a
// successful call to rados_connect(), however if the connection has been
// configured with non-default parameters, some of the parameters may be
// allocated before connecting. rados_shutdown() will free the allocated
// resources, even if there has not been a connection yet.
//
// This function is setup as a destructor/finalizer when rados_create() is
// called.
func freeConn(conn *Conn) {
	if conn.cluster != nil {
		C.rados_shutdown(conn.cluster)
		// prevent calling rados_shutdown() more than once
		conn.cluster = nil
	}
}
<commit_msg>rados: add doc comments for RadosError type<commit_after>package rados

// #cgo LDFLAGS: -lrados
// #include <errno.h>
// #include <stdlib.h>
// #include <rados/librados.h>
import "C"

import (
	"fmt"
	"runtime"
	"unsafe"

	"github.com/ceph/go-ceph/errutil"
)

// RadosError represents an error condition returned from the Ceph RADOS APIs.
type RadosError int

// Error returns the error string for the RadosError type.
func (e RadosError) Error() string {
	errno, s := errutil.FormatErrno(int(e))
	if s == "" {
		return fmt.Sprintf("rados: ret=%d", errno)
	}
	return fmt.Sprintf("rados: ret=%d, %s", errno, s)
}

var RadosAllNamespaces = C.LIBRADOS_ALL_NSPACES

var RadosErrorNotFound = RadosError(-C.ENOENT)
var RadosErrorPermissionDenied = RadosError(-C.EPERM)

func getRadosError(err int) error {
	if err == 0 {
		return nil
	}
	return RadosError(err)
}

// Version returns the major, minor, and patch components of the version of
// the RADOS library linked against.
func Version() (int, int, int) {
	var c_major, c_minor, c_patch C.int
	C.rados_version(&c_major, &c_minor, &c_patch)
	return int(c_major), int(c_minor), int(c_patch)
}

func makeConn() *Conn {
	return &Conn{connected: false}
}

func newConn(user *C.char) (*Conn, error) {
	conn := makeConn()
	ret := C.rados_create(&conn.cluster, user)

	if ret != 0 {
		return nil, RadosError(int(ret))
	}

	runtime.SetFinalizer(conn, freeConn)
	return conn, nil
}

// NewConn creates a new connection object. It returns the connection and an
// error, if any.
func NewConn() (*Conn, error) {
	return newConn(nil)
}

// NewConnWithUser creates a new connection object with a custom username.
// It returns the connection and an error, if any.
func NewConnWithUser(user string) (*Conn, error) {
	c_user := C.CString(user)
	defer C.free(unsafe.Pointer(c_user))
	return newConn(c_user)
}

// NewConnWithClusterAndUser creates a new connection object for a specific cluster and username.
// It returns the connection and an error, if any.
func NewConnWithClusterAndUser(clusterName string, userName string) (*Conn, error) {
	c_cluster_name := C.CString(clusterName)
	defer C.free(unsafe.Pointer(c_cluster_name))

	c_name := C.CString(userName)
	defer C.free(unsafe.Pointer(c_name))

	conn := makeConn()
	ret := C.rados_create2(&conn.cluster, c_cluster_name, c_name, 0)
	if ret != 0 {
		return nil, RadosError(int(ret))
	}

	runtime.SetFinalizer(conn, freeConn)
	return conn, nil
}

// freeConn releases resources that are allocated while configuring the
// connection to the cluster. rados_shutdown() should only be needed after a
// successful call to rados_connect(), however if the connection has been
// configured with non-default parameters, some of the parameters may be
// allocated before connecting. rados_shutdown() will free the allocated
// resources, even if there has not been a connection yet.
//
// This function is setup as a destructor/finalizer when rados_create() is
// called.
func freeConn(conn *Conn) {
	if conn.cluster != nil {
		C.rados_shutdown(conn.cluster)
		// prevent calling rados_shutdown() more than once
		conn.cluster = nil
	}
}
<|endoftext|>


<commit_before>// geomi, 거미, is a basic spider for crawling specific webs. It was designed to crawl
// either a site, or a subset of a site, indexing the pages and where they link to.
// Any links that go outside of the provided base URL are noted but not followed.
//
//
// Even though geomi was designed with the intended use being crawling ones own site,
// or a client's site, it does come with some basic behavior configuration to make it
// a friendly bot:
//   * respects ROBOTS.txt TODO
//   * configurable concurrent walkers TODO
//   * configurable wait interval range TODO
//   * configurable max requests per: TODO
//     * min
//     * hour
//     * day
//     * month
//
// To start, go get the geomi package:
//    go get github.com/mohae/geomi
//
// Import it into your code:
//    import github.com/mohae/geomi
//
// Set up the site for the spider:
//   s := geomi.NewSite("http://example.com")
//
// The url that is passed to the NewSite() function is the base case, after which
// a BFS traversal is done.
//
package geomi

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"math/rand"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/mohae/utilitybelt/queue"
	"github.com/temoto/robotstxt.go"
	"golang.org/x/net/html"
)

// Fetcher is an interface that makes it easier to test. In the future, it may be
// useful for other purposes, but that would require exporting the method that uses
// it first.
type Fetcher interface {
	// Fetch returns the body of URL and
	// a slice of URLs found on that page
	Fetch(url string) (body string, urls []string, err error)
}

//}
// fetched tracks URLs that have been (or are being) fetched.
// The lock must be held while reading from or writing to the map.
var fetched = struct {
	m map[string]error
	sync.Mutex
}{m: make(map[string]error)}

var loading = errors.New("url load in progress") // sentinel value

// a page is a url. This is usually some content wiht a number of elements, but it
// could be something as simple as the url for cat.jpg on your site and represent that
// image.
type Page struct {
	*url.URL
	distance int
	body     string
	links    []string // immediate children
}

// Site is a type that implements fetcher
type Site struct {
	*url.URL
}

// Implements fetcher.
// TODO: make the design cleaner
func (s Site) Fetch(url string) (body string, urls []string, err error) {
	// see if the passed url is outside of the baseURL
	resp, err := http.Get(url)
	if err != nil {
		return "", nil, err
	}
	defer resp.Body.Close()
	buff := &bytes.Buffer{}
	tee := io.TeeReader(resp.Body, buff)
	tokens := getTokens(tee)
	if len(tokens) == 0 {
		return "", nil, fmt.Errorf("%s: nothing in body", url)
	}
	urls, err = s.linksFromTokens(tokens)
	if err != nil {
		return "", nil, err
	}
	return buff.String(), urls, nil
}

// linksFromTokens returns a list of links (href a) found in the token slice
// TODO should internal links be tracked separatly? i.e. record them in a
// separate var (so they don't get fetched)
func (s *Site) linksFromTokens(tokens []html.Token) ([]string, error) {
	var links []string
	for _, token := range tokens {
		if token.Type == html.StartTagToken && token.DataAtom.String() == "a" {
			for _, attr := range token.Attr {
				// We only care about links that aren't #
				if attr.Key == "href" && !strings.HasPrefix(attr.Val, "#") {
					// parse the url
					url, err := url.Parse(attr.Val)
					if err != nil {
						return nil, err
					}
					link := s.URL.ResolveReference(url)
					links = append(links, link.String())
				}
			}
		}
	}
	return links, nil
}

// Spider crawls the target. It contains all information needed to manage the crawl
// including keeping track of work to do, work done, and the results.
type Spider struct {
	*queue.Queue
	sync.Mutex
	wg               sync.WaitGroup
	*url.URL                // the start url
	bot              string // the name of the bot for robots
	concurrency      int    // concurrency level of crawling
	fetchInterval    int64  // if > 0, interval, in milliseconds to wait between gets
	intervalJitter   int64  // the max jitter to be added, per fetch. Jitter is a rand with this as max.
	RestrictToScheme bool   // if true, only crawls urls with the same scheme as baseURL
	RespectRobots    bool   // whether or not to respect the robots.txt
	robots           *robotstxt.Group
	maxDepth         int
	Pages            map[string]Page
	foundURLs        map[string]struct{} // keeps track of urls found to prevent recrawling
	fetchedURLs      map[string]error    // urls that have been fetched with their status
	skippedURLs      map[string]struct{} // urls within the same domain that are not retrieved
	extHosts         map[string]struct{} // list of external hosts TODO: elide?
	extLinks         map[string]struct{} // list of external links; not fetched
}

// returns a Spider with the its site's baseUrl set. The baseUrl is the start point for
// the crawl. It is also the restriction on the crawl:
func NewSpider(start string) (*Spider, error) {
	if start == "" {
		return nil, errors.New("newSpider: the start url cannot be empty")
	}
	var err error
	spider := &Spider{Queue: queue.New(128, 0),
		bot:           "geomi",
		RespectRobots: true,
		Pages:         make(map[string]Page),
		foundURLs:     make(map[string]struct{}),
		fetchedURLs:   make(map[string]error),
		skippedURLs:   make(map[string]struct{}),
		extHosts:      make(map[string]struct{}),
		extLinks:      make(map[string]struct{}),
	}
	spider.URL, err = url.Parse(start)
	if err != nil {
		return nil, err
	}
	return spider, nil
}

// SetFetchInterval set's the spiders wait interval, and calculates the intervalJitter
// value for the random jitter added to the wait. This value is in milliseconds, 1000
// means 1 second.
//
// When fetchInterval > 0:
// time between fetches = fetchInterval + rand(intervalJitter)
// max time between fetches = fetchInterval + intervalJitter
func (s *Spider) SetFetchInterval(i int64) {
	s.fetchInterval = i
	jitter := i / 5 // jitter is, at most 20% of the fetchInterval, Min interval is 10ms
	// 10ms is used as the floor because arbitrary. A larger floor might be reasonable
	if jitter >= 10 {
		s.intervalJitter = jitter
	}
}

// Crawl is the exposed method for starting a crawl at baseURL. The crawl private method
// does the actual work. The depth is the maximum depth, or distance, to crawl from the
// baseURL. If depth == -1, no limits are set and it is expected that the entire site
// will be crawled.
func (s *Spider) Crawl(depth int) (message string, err error) {
	s.maxDepth = depth
	S := Site{URL: s.URL}
	// if we are to respect the robots.txt, set up the info
	if s.RespectRobots {
		s.getRobotsTxt()
	}
	s.Queue.Enqueue(Page{URL: s.URL})
	err = s.crawl(S)
	return fmt.Sprintf("%d nodes were processed; %d external links linking to %d external hosts were not processed", len(s.Pages), len(s.extLinks), len(s.extHosts)), err
}

// This crawl does all the work.
func (s *Spider) crawl(fetcher Fetcher) error {
	var err error
	for !s.Queue.IsEmpty() {
		// get next item from queue
		page := s.Queue.Dequeue().(Page)
		// if a depth value was passed and the distance is > depth, we are done
		// depth of 0 means no limit
		if s.maxDepth != -1 && page.distance > s.maxDepth {
			return nil
		}
		// see if this is an external url
		if s.externalURL(page.URL) {
			continue
		}
		// check to see if this url should be skipped for other reasons
		if s.skip(page.URL) {
			// see if the skipped is external and process accordingly
			continue
		}
		s.foundURLs[page.URL.String()] = struct{}{}
		// get the url
		page.body, page.links, err = fetcher.Fetch(page.URL.String())
		// add the page and status to the map. map isn't checked for membership becuase we don't
		// fetch found urls.
		s.Lock()
		s.Pages[page.URL.String()] = page
		s.fetchedURLs[page.URL.String()] = err
		s.Unlock()
		// add the urls that the node contains to the queue
		for _, l := range page.links {
			u, _ := url.Parse(l)
			s.Queue.Enqueue(Page{URL: u, distance: page.distance + 1})
		}
		// if their is a wait between fetches, sleep for that + random jitter
		if s.fetchInterval > 0 {
			wait := s.fetchInterval
			// if there is a value for jitter, add a random jitter
			if s.intervalJitter > 0 {
				wait += rand.Int63n(s.intervalJitter)
			}
			time.Sleep(time.Duration(wait) * time.Millisecond)
		}
	}
	return nil
}

// skip determines whether the url should be skipped.
//   * skip urls that have already been fetched
//   * skip urls that are outside of the basePath
//   * conditionally skip urls that are outside of the current scheme, even if
//     they are within the current pasePath
//   * skip if not allowed by robots
func (s *Spider) skip(u *url.URL) bool {
	s.Lock()
	_, ok := s.foundURLs[u.String()]
	s.Unlock()
	if ok { // if it was found, skip it
		s.addSkippedURL(u)
		return true
	}

	// skip if we are restricted to current scheme
	if s.RestrictToScheme {
		if u.Scheme != s.URL.Scheme {
			s.addSkippedURL(u)
			return true
		}
	}
	if s.RespectRobots {
		ok := s.robotsAllowed(u)
		if !ok {
			return false
		}
	}
	// skip if the url is outside of base
	// remove the scheme + schemePrefix so just the rest of the url is being compared
	if !strings.HasPrefix(u.Path, s.URL.Path) {
		s.addSkippedURL(u)
		return true
	}
	return false
}

// addSkippedURL add's the url info too the skipped info
func (s *Spider) addSkippedURL(u *url.URL) {
	s.Lock()
	s.skippedURLs[u.Path] = struct{}{}
	s.Unlock()
}

// externalURL check's to see if the url is external to the site, host isn't the same.
// and add's that info to the ext structs.
func (s *Spider) externalURL(u *url.URL) bool {
	if u.Host != s.URL.Host {
		s.Lock()
		// see if the host is already in the map
		_, ok := s.extHosts[u.Host]
		if !ok {
			s.extHosts[u.Host] = struct{}{}
		}
		// same with url
		_, ok = s.extLinks[u.Path]
		if !ok {
			s.extLinks[u.Path] = struct{}{}
		}
		s.Unlock()
		return true
	}
	return false
}

// getRobotsTxt retrieves and processes the site's robot.txt. If the robots.txt doesn't
// exist, it is assumed that everything is allowed.
func (s *Spider) getRobotsTxt() error {
	resp, err := http.Get("http://" + s.URL.Host)
	if err != nil {
		return err
	}
	robots, err := robotstxt.FromResponse(resp)
	if err != nil {
		return err
	}
	s.robots = robots.FindGroup(s.bot)
	return nil
}

// robotsAllowed checks to see if the passed path is allowed by Robots.txt. If the
// robots isn't set, it's always true
func (s *Spider) robotsAllowed(u *url.URL) bool {
	if s.robots != nil {
		return s.robots.Test(u.Path)
	}
	return true
}

// getTokens returns all tokens in the body
func getTokens(body io.Reader) []html.Token {
	tokens := make([]html.Token, 0)
	page := html.NewTokenizer(body)
	for {
		typ := page.Next()
		if typ == html.ErrorToken {
			return tokens
		}
		tokens = append(tokens, page.Token())
	}
}

func init() {
	// We just use math/rand because it's good enough for our purpose.
	rand.Seed(time.Now().UTC().UnixNano())
}
<commit_msg>refactor ext stuff to be named external, correct u.Path() to u.String() and add bool to determine whether ext urls should be checked<commit_after>// geomi, 거미, is a basic spider for crawling specific webs. It was designed to crawl
// either a site, or a subset of a site, indexing the pages and where they link to.
// Any links that go outside of the provided base URL are noted but not followed.
//
//
// Even though geomi was designed with the intended use being crawling ones own site,
// or a client's site, it does come with some basic behavior configuration to make it
// a friendly bot:
//   * respects ROBOTS.txt TODO
//   * configurable concurrent walkers TODO
//   * configurable wait interval range TODO
//   * configurable max requests per: TODO
//     * min
//     * hour
//     * day
//     * month
//
// To start, go get the geomi package:
//    go get github.com/mohae/geomi
//
// Import it into your code:
//    import github.com/mohae/geomi
//
// Set up the site for the spider:
//   s := geomi.NewSite("http://example.com")
//
// The url that is passed to the NewSite() function is the base case, after which
// a BFS traversal is done.
//
package geomi

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"math/rand"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/mohae/utilitybelt/queue"
	"github.com/temoto/robotstxt.go"
	"golang.org/x/net/html"
)

// Fetcher is an interface that makes it easier to test. In the future, it may be
// useful for other purposes, but that would require exporting the method that uses
// it first.
type Fetcher interface {
	// Fetch returns the body of URL and
	// a slice of URLs found on that page
	Fetch(url string) (body string, urls []string, err error)
}

//}
// fetched tracks URLs that have been (or are being) fetched.
// The lock must be held while reading from or writing to the map.
var fetched = struct {
	m map[string]error
	sync.Mutex
}{m: make(map[string]error)}

var loading = errors.New("url load in progress") // sentinel value

// a page is a url. This is usually some content wiht a number of elements, but it
// could be something as simple as the url for cat.jpg on your site and represent that
// image.
type Page struct {
	*url.URL
	distance int
	body     string
	links    []string // immediate children
}

// Site is a type that implements fetcher
type Site struct {
	*url.URL
}

// Implements fetcher.
// TODO: make the design cleaner
func (s Site) Fetch(url string) (body string, urls []string, err error) {
	// see if the passed url is outside of the baseURL
	resp, err := http.Get(url)
	if err != nil {
		return "", nil, err
	}
	defer resp.Body.Close()
	buff := &bytes.Buffer{}
	tee := io.TeeReader(resp.Body, buff)
	tokens := getTokens(tee)
	if len(tokens) == 0 {
		return "", nil, fmt.Errorf("%s: nothing in body", url)
	}
	urls, err = s.linksFromTokens(tokens)
	if err != nil {
		return "", nil, err
	}
	return buff.String(), urls, nil
}

// linksFromTokens returns a list of links (href a) found in the token slice
// TODO should internal links be tracked separatly? i.e. record them in a
// separate var (so they don't get fetched)
func (s *Site) linksFromTokens(tokens []html.Token) ([]string, error) {
	var links []string
	for _, token := range tokens {
		if token.Type == html.StartTagToken && token.DataAtom.String() == "a" {
			for _, attr := range token.Attr {
				// We only care about links that aren't #
				if attr.Key == "href" && !strings.HasPrefix(attr.Val, "#") {
					// parse the url
					url, err := url.Parse(attr.Val)
					if err != nil {
						return nil, err
					}
					link := s.URL.ResolveReference(url)
					links = append(links, link.String())
				}
			}
		}
	}
	return links, nil
}

// Spider crawls the target. It contains all information needed to manage the crawl
// including keeping track of work to do, work done, and the results.
type Spider struct {
	*queue.Queue
	sync.Mutex
	wg                 sync.WaitGroup
	*url.URL                  // the start url
	bot                string // the name of the bot for robots
	concurrency        int    // concurrency level of crawling
	fetchInterval      int64  // if > 0, interval, in milliseconds to wait between gets
	intervalJitter     int64  // the max jitter to be added, per fetch. Jitter is a rand with this as max.
	RestrictToScheme   bool   // if true, only crawls urls with the same scheme as baseURL
	RespectRobots      bool   // whether or not to respect the robots.txt
	CheckExternalLinks bool   // Check the status of any links that go to other domains
	robots             *robotstxt.Group
	maxDepth           int
	Pages              map[string]Page
	foundURLs          map[string]struct{} // keeps track of urls found to prevent recrawling
	fetchedURLs        map[string]error    // urls that have been fetched with their status
	skippedURLs        map[string]struct{} // urls within the same domain that are not retrieved
	externalHosts      map[string]struct{} // list of external hosts TODO: elide?
	externalLinks      map[string]struct{} // list of external links; not fetched
}

// returns a Spider with the its site's baseUrl set. The baseUrl is the start point for
// the crawl. It is also the restriction on the crawl:
func NewSpider(start string) (*Spider, error) {
	if start == "" {
		return nil, errors.New("newSpider: the start url cannot be empty")
	}
	var err error
	spider := &Spider{Queue: queue.New(128, 0),
		bot:           "geomi",
		RespectRobots: true,
		Pages:         make(map[string]Page),
		foundURLs:     make(map[string]struct{}),
		fetchedURLs:   make(map[string]error),
		skippedURLs:   make(map[string]struct{}),
		externalHosts: make(map[string]struct{}),
		externalLinks: make(map[string]struct{}),
	}
	spider.URL, err = url.Parse(start)
	if err != nil {
		return nil, err
	}
	return spider, nil
}

// SetFetchInterval set's the spiders wait interval, and calculates the intervalJitter
// value for the random jitter added to the wait. This value is in milliseconds, 1000
// means 1 second.
//
// When fetchInterval > 0:
// time between fetches = fetchInterval + rand(intervalJitter)
// max time between fetches = fetchInterval + intervalJitter
func (s *Spider) SetFetchInterval(i int64) {
	s.fetchInterval = i
	jitter := i / 5 // jitter is, at most 20% of the fetchInterval, Min interval is 10ms
	// 10ms is used as the floor because arbitrary. A larger floor might be reasonable
	if jitter >= 10 {
		s.intervalJitter = jitter
	}
}

// Crawl is the exposed method for starting a crawl at baseURL. The crawl private method
// does the actual work. The depth is the maximum depth, or distance, to crawl from the
// baseURL. If depth == -1, no limits are set and it is expected that the entire site
// will be crawled.
func (s *Spider) Crawl(depth int) (message string, err error) {
	s.maxDepth = depth
	S := Site{URL: s.URL}
	// if we are to respect the robots.txt, set up the info
	if s.RespectRobots {
		s.getRobotsTxt()
	}
	s.Queue.Enqueue(Page{URL: s.URL})
	err = s.crawl(S)
	return fmt.Sprintf("%d nodes were processed; %d external links linking to %d external hosts were not processed", len(s.Pages), len(s.externalLinks), len(s.externalHosts)), err
}

// This crawl does all the work.
func (s *Spider) crawl(fetcher Fetcher) error {
	var err error
	for !s.Queue.IsEmpty() {
		// get next item from queue
		page := s.Queue.Dequeue().(Page)
		// if a depth value was passed and the distance is > depth, we are done
		// depth of 0 means no limit
		if s.maxDepth != -1 && page.distance > s.maxDepth {
			return nil
		}
		// see if this is an external url
		if s.externalURL(page.URL) {
			if s.CheckExternalLinks {
				s.fetchExternalLink(page.URL)
			}
			continue
		}
		// check to see if this url should be skipped for other reasons
		if s.skip(page.URL) {
			// see if the skipped is external and process accordingly
			continue
		}
		s.foundURLs[page.URL.String()] = struct{}{}
		// get the url
		page.body, page.links, err = fetcher.Fetch(page.URL.String())
		// add the page and status to the map. map isn't checked for membership becuase we don't
		// fetch found urls.
		s.Lock()
		s.Pages[page.URL.String()] = page
		s.fetchedURLs[page.URL.String()] = err
		s.Unlock()
		// add the urls that the node contains to the queue
		for _, l := range page.links {
			u, _ := url.Parse(l)
			s.Queue.Enqueue(Page{URL: u, distance: page.distance + 1})
		}
		// if their is a wait between fetches, sleep for that + random jitter
		if s.fetchInterval > 0 {
			wait := s.fetchInterval
			// if there is a value for jitter, add a random jitter
			if s.intervalJitter > 0 {
				wait += rand.Int63n(s.intervalJitter)
			}
			time.Sleep(time.Duration(wait) * time.Millisecond)
		}
	}
	return nil
}

// skip determines whether the url should be skipped.
//   * skip urls that have already been fetched
//   * skip urls that are outside of the basePath
//   * conditionally skip urls that are outside of the current scheme, even if
//     they are within the current pasePath
//   * skip if not allowed by robots
func (s *Spider) skip(u *url.URL) bool {
	s.Lock()
	_, ok := s.foundURLs[u.String()]
	s.Unlock()
	if ok { // if it was found, skip it
		s.addSkippedURL(u)
		return true
	}

	// skip if we are restricted to current scheme
	if s.RestrictToScheme {
		if u.Scheme != s.URL.Scheme {
			s.addSkippedURL(u)
			return true
		}
	}
	if s.RespectRobots {
		ok := s.robotsAllowed(u)
		if !ok {
			return false
		}
	}
	// skip if the url is outside of base
	// remove the scheme + schemePrefix so just the rest of the url is being compared
	if !strings.HasPrefix(u.Path, s.URL.Path) {
		s.addSkippedURL(u)
		return true
	}
	return false
}

// addSkippedURL add's the url info too the skipped info
func (s *Spider) addSkippedURL(u *url.URL) {
	s.Lock()
	s.skippedURLs[u.Path] = struct{}{}
	s.Unlock()
}

// externalURL check's to see if the url is external to the site, host isn't the same.
// and add's that info to the ext structs.
func (s *Spider) externalURL(u *url.URL) bool {
	if u.Host != s.URL.Host {
		s.Lock()
		// see if the host is already in the map
		_, ok := s.externalHosts[u.Host]
		if !ok {
			s.externalHosts[u.Host] = struct{}{}
		}
		// same with url
		_, ok = s.externalLinks[u.String()]
		if !ok {
			s.externalLinks[u.String()] = struct{}{}
		}
		s.Unlock()
		return true
	}
	return false
}

// getRobotsTxt retrieves and processes the site's robot.txt. If the robots.txt doesn't
// exist, it is assumed that everything is allowed.
func (s *Spider) getRobotsTxt() error {
	resp, err := http.Get("http://" + s.URL.Host)
	if err != nil {
		return err
	}
	robots, err := robotstxt.FromResponse(resp)
	if err != nil {
		return err
	}
	s.robots = robots.FindGroup(s.bot)
	return nil
}

// robotsAllowed checks to see if the passed path is allowed by Robots.txt. If the
// robots isn't set, it's always true
func (s *Spider) robotsAllowed(u *url.URL) bool {
	if s.robots != nil {
		return s.robots.Test(u.Path)
	}
	return true
}

// getTokens returns all tokens in the body
func getTokens(body io.Reader) []html.Token {
	tokens := make([]html.Token, 0)
	page := html.NewTokenizer(body)
	for {
		typ := page.Next()
		if typ == html.ErrorToken {
			return tokens
		}
		tokens = append(tokens, page.Token())
	}
}

func init() {
	// We just use math/rand because it's good enough for our purpose.
	rand.Seed(time.Now().UTC().UnixNano())
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"code.google.com/p/rog-go/exp/go/ast"
	"code.google.com/p/rog-go/exp/go/parser"
	"code.google.com/p/rog-go/exp/go/printer"
	"code.google.com/p/rog-go/exp/go/token"
	"code.google.com/p/rog-go/exp/go/types"
	"strconv"
)

type symInfo struct {
	pos      token.Pos   // position of symbol.
	expr     ast.Expr    // expression for symbol (*ast.Ident or *ast.SelectorExpr)
	ident    *ast.Ident  // identifier in parse tree (changing ident.Name changes the parse tree)
	exprType types.Type  // type of expression.
	referPos token.Pos   // position of referred-to symbol.
	referObj *ast.Object // object referred to.
	local    bool        // whether referred-to object is function-local.
	universe bool        // whether referred-to object is in universe.
}

type vcontext struct {
	importer types.Importer
	logf     func(pos token.Pos, f string, a ...interface{})
}

func (ctxt *vcontext) visitSyms(pkg *ast.File, visitf func(*symInfo) bool) {
	var visit astVisitor
	ok := true
	local := false // TODO set to true inside function body
	visit = func(n ast.Node) bool {
		if !ok {
			return false
		}
		switch n := n.(type) {
		case *ast.ImportSpec:
			// If the file imports a package to ".", abort
			// because we don't support that (yet).
			if n.Name != nil && n.Name.Name == "." {
				ctxt.logf(n.Pos(), "import to . not supported")
				ok = false
				return false
			}
			return true

		case *ast.FuncDecl:
			// add object for init functions
			if n.Recv == nil && n.Name.Name == "init" {
				n.Name.Obj = ast.NewObj(ast.Fun, "init")
			}
			return true

		case *ast.Ident:
			ok = ctxt.visitExpr(n, local, visitf)
			return false

		case *ast.KeyValueExpr:
			// don't try to resolve the key part of a key-value
			// because it might be a map key which doesn't
			// need resolving, and we can't tell without being
			// complicated with types.
			ast.Walk(visit, n.Value)
			return false

		case *ast.SelectorExpr:
			ast.Walk(visit, n.X)
			ok = ctxt.visitExpr(n, local, visitf)
			return false

		case *ast.File:
			for _, d := range n.Decls {
				ast.Walk(visit, d)
			}
			return false
		}

		return true
	}
	ast.Walk(visit, pkg)
}

func (ctxt *vcontext) visitExpr(e ast.Expr, local bool, visitf func(*symInfo) bool) bool {
	var info symInfo
	info.expr = e
	switch e := e.(type) {
	case *ast.Ident:
		info.pos = e.Pos()
		info.ident = e
	case *ast.SelectorExpr:
		info.pos = e.Sel.Pos()
		info.ident = e.Sel
	}
	obj, t := types.ExprType(e, ctxt.importer)
	if obj == nil {
		ctxt.logf(e.Pos(), "no object for %s", pretty(e))
		return true
	}
	info.exprType = t
	info.referObj = obj
	if parser.Universe.Lookup(obj.Name) != obj {
		info.referPos = types.DeclPos(obj)
		if info.referPos == token.NoPos {
			ctxt.logf(e.Pos(), "no declaration for %s", pretty(e))
			return true
		}
	} else {
		info.universe = true
	}
	info.local = local
	return visitf(&info)
}

// litToString converts from a string literal to a regular string.
func litToString(lit *ast.BasicLit) (v string) {
	if lit.Kind != token.STRING {
		panic("expected string")
	}
	v, err := strconv.Unquote(string(lit.Value))
	if err != nil {
		panic("cannot unquote")
	}
	return v
}

type astVisitor func(n ast.Node) bool

func (f astVisitor) Visit(n ast.Node) ast.Visitor {
	if f(n) {
		return f
	}
	return nil
}

var emptyFileSet = token.NewFileSet()

func pretty(n ast.Node) string {
	var b bytes.Buffer
	printer.Fprint(&b, emptyFileSet, n)
	return b.String()
}
<commit_msg>add comment<commit_after>package main

import (
	"bytes"
	"code.google.com/p/rog-go/exp/go/ast"
	"code.google.com/p/rog-go/exp/go/parser"
	"code.google.com/p/rog-go/exp/go/printer"
	"code.google.com/p/rog-go/exp/go/token"
	"code.google.com/p/rog-go/exp/go/types"
	"strconv"
)

type symInfo struct {
	pos      token.Pos   // position of symbol.
	expr     ast.Expr    // expression for symbol (*ast.Ident or *ast.SelectorExpr)
	ident    *ast.Ident  // identifier in parse tree (changing ident.Name changes the parse tree)
	exprType types.Type  // type of expression.
	referPos token.Pos   // position of referred-to symbol.
	referObj *ast.Object // object referred to.
	local    bool        // whether referred-to object is function-local.
	universe bool        // whether referred-to object is in universe.
}

type vcontext struct {
	importer types.Importer
	logf     func(pos token.Pos, f string, a ...interface{})
}

// visitSyms calls visitf for each identifier in the given file.
func (ctxt *vcontext) visitSyms(pkg *ast.File, visitf func(*symInfo) bool) {
	var visit astVisitor
	ok := true
	local := false // TODO set to true inside function body
	visit = func(n ast.Node) bool {
		if !ok {
			return false
		}
		switch n := n.(type) {
		case *ast.ImportSpec:
			// If the file imports a package to ".", abort
			// because we don't support that (yet).
			if n.Name != nil && n.Name.Name == "." {
				ctxt.logf(n.Pos(), "import to . not supported")
				ok = false
				return false
			}
			return true

		case *ast.FuncDecl:
			// add object for init functions
			if n.Recv == nil && n.Name.Name == "init" {
				n.Name.Obj = ast.NewObj(ast.Fun, "init")
			}
			return true

		case *ast.Ident:
			ok = ctxt.visitExpr(n, local, visitf)
			return false

		case *ast.KeyValueExpr:
			// don't try to resolve the key part of a key-value
			// because it might be a map key which doesn't
			// need resolving, and we can't tell without being
			// complicated with types.
			ast.Walk(visit, n.Value)
			return false

		case *ast.SelectorExpr:
			ast.Walk(visit, n.X)
			ok = ctxt.visitExpr(n, local, visitf)
			return false

		case *ast.File:
			for _, d := range n.Decls {
				ast.Walk(visit, d)
			}
			return false
		}

		return true
	}
	ast.Walk(visit, pkg)
}

func (ctxt *vcontext) visitExpr(e ast.Expr, local bool, visitf func(*symInfo) bool) bool {
	var info symInfo
	info.expr = e
	switch e := e.(type) {
	case *ast.Ident:
		info.pos = e.Pos()
		info.ident = e
	case *ast.SelectorExpr:
		info.pos = e.Sel.Pos()
		info.ident = e.Sel
	}
	obj, t := types.ExprType(e, ctxt.importer)
	if obj == nil {
		ctxt.logf(e.Pos(), "no object for %s", pretty(e))
		return true
	}
	info.exprType = t
	info.referObj = obj
	if parser.Universe.Lookup(obj.Name) != obj {
		info.referPos = types.DeclPos(obj)
		if info.referPos == token.NoPos {
			ctxt.logf(e.Pos(), "no declaration for %s", pretty(e))
			return true
		}
	} else {
		info.universe = true
	}
	info.local = local
	return visitf(&info)
}

// litToString converts from a string literal to a regular string.
func litToString(lit *ast.BasicLit) (v string) {
	if lit.Kind != token.STRING {
		panic("expected string")
	}
	v, err := strconv.Unquote(string(lit.Value))
	if err != nil {
		panic("cannot unquote")
	}
	return v
}

type astVisitor func(n ast.Node) bool

func (f astVisitor) Visit(n ast.Node) ast.Visitor {
	if f(n) {
		return f
	}
	return nil
}

var emptyFileSet = token.NewFileSet()

func pretty(n ast.Node) string {
	var b bytes.Buffer
	printer.Fprint(&b, emptyFileSet, n)
	return b.String()
}
<|endoftext|>


<commit_before>package httplib

import (
    "bytes"
    "crypto/tls"
    "http"
    "io"
    "io/ioutil"
    "net"
    "os"
    "strings"
)

var defaultUserAgent = "httplib.go"

var debugprint = false

type Client struct {
    conn    *http.ClientConn
    lastURL *http.URL
}

type nopCloser struct {
    io.Reader
}

func (nopCloser) Close() os.Error { return nil }

func getNopCloser(buf *bytes.Buffer) nopCloser {
    return nopCloser{buf}
}

func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") }

func newConn(url *http.URL) (*http.ClientConn, os.Error) {
    addr := url.Host
    if !hasPort(addr) {
        addr += ":" + url.Scheme
    }
    var conn net.Conn
    var err os.Error
    if url.Scheme == "http" {
        conn, err = net.Dial("tcp", "", addr)
        if err != nil {
            return nil, err
        }
    } else { // https
        conn, err = tls.Dial("tcp", "", addr, nil)
        if err != nil {
            return nil, err
        }
        h := url.Host
        if hasPort(h) {
            h = h[0:strings.LastIndex(h, ":")]
        }
        if err := conn.(*tls.Conn).VerifyHostname(h); err != nil {
            return nil, err
        }
    }

    return http.NewClientConn(conn, nil), nil
}

func getResponse(rawUrl string, req *http.Request) (*http.ClientConn, *http.Response, os.Error) {
    url, err := http.ParseURL(rawUrl)
    if err != nil {
        return nil, nil, err
    }
    req.URL = url
    if debugprint {
        dump, _ := http.DumpRequest(req, true)
        print(string(dump))
    }

    conn, err := newConn(url)
    if err != nil {
        return nil, nil, err
    }

    err = conn.Write(req)
    if err != nil {
        return nil, nil, err
    }

    resp, err := conn.Read()
    if err != nil {
        if err != http.ErrPersistEOF {
            return nil, nil, err
        }
    }
    return conn, resp, nil
}

func (client *Client) Request(rawurl string, method string, headers map[string]string, body string) (*http.Response, os.Error) {
    var url *http.URL
    var err os.Error
    if url, err = http.ParseURL(rawurl); err != nil {
        return nil, err
    }

    if client.conn == nil || client.lastURL.Host != url.Host {
        client.conn, err = newConn(url)
    }

    if headers == nil {
        headers = map[string]string{}
    }

    client.lastURL = url
    var req http.Request
    req.URL = url
    req.Method = method
    req.Header = headers
    req.UserAgent = headers["User-Agent"]
    if req.UserAgent == "" {
        req.UserAgent = "httplib.go"
    }
    req.Body = nopCloser{bytes.NewBufferString(body)}

    if debugprint {
        dump, _ := http.DumpRequest(&req, true)
        print(string(dump))
    }

    err = client.conn.Write(&req)
    if err != nil {
        return nil, err
    }

    resp, err := client.conn.Read()
    if err != nil {
        return nil, err
    }

    return resp, nil
}

func Get(url string) *HttpRequestBuilder {
    var req http.Request
    req.Method = "GET"
    req.Header = map[string]string{}
    req.UserAgent = defaultUserAgent
    return &HttpRequestBuilder{url, &req, nil, map[string]string{}}
}

func Post(url string) *HttpRequestBuilder {
    var req http.Request
    req.Method = "POST"
    req.Header = map[string]string{}
    req.UserAgent = defaultUserAgent
    return &HttpRequestBuilder{url, &req, nil, map[string]string{}}
}

func Put(url string) *HttpRequestBuilder {
    var req http.Request
    req.Method = "PUT"
    req.Header = map[string]string{}
    req.UserAgent = defaultUserAgent
    return &HttpRequestBuilder{url, &req, nil, map[string]string{}}
}

func Delete(url string) *HttpRequestBuilder {
    var req http.Request
    req.Method = "DELETE"
    req.Header = map[string]string{}
    req.UserAgent = defaultUserAgent
    return &HttpRequestBuilder{url, &req, nil, map[string]string{}}
}

type HttpRequestBuilder struct {
    url    string
    req    *http.Request
    clientConn *http.ClientConn
    params map[string]string
}

func (b *HttpRequestBuilder) getResponse() (*http.Response, os.Error) {
    var paramBody string
    if b.params != nil && len(b.params) > 0 {
        var buf bytes.Buffer
        for k, v := range b.params {
            buf.WriteString(http.URLEscape(k))
            buf.WriteByte('=')
            buf.WriteString(http.URLEscape(v))
            buf.WriteByte('&')
        }
        paramBody = buf.String()
        paramBody = paramBody[0 : len(paramBody)-1]
    }
    if b.req.Method == "GET" && len(paramBody) > 0 {
        if strings.Index(b.url, "?") != -1 {
            b.url += "&" + paramBody
        } else {
            b.url = b.url + "?" + paramBody
        }
    } else if b.req.Method == "POST" && b.req.Body == nil && len(paramBody) > 0 {
        b.Header("Content-Type", "application/x-www-form-urlencoded")
        b.req.Body = nopCloser{bytes.NewBufferString(paramBody)}
        b.req.ContentLength = int64(len(paramBody))
    }

    conn, resp, err :=  getResponse(b.url, b.req)
    b.clientConn = conn
    return resp,err
}

func (b *HttpRequestBuilder) Header(key, value string) *HttpRequestBuilder {
    b.req.Header[key] = value
    return b
}

func (b *HttpRequestBuilder) Param(key, value string) *HttpRequestBuilder {
    b.params[key] = value
    return b
}

func (b *HttpRequestBuilder) Body(data interface{}) *HttpRequestBuilder {
    switch t := data.(type) {
    case string:
        b.req.Body = getNopCloser(bytes.NewBufferString(t))
        b.req.ContentLength = int64(len(t))
    case []byte:
        b.req.Body = getNopCloser(bytes.NewBuffer(t))
        b.req.ContentLength = int64(len(t))
    }
    return b
}

func (b *HttpRequestBuilder) AsString() (string, os.Error) {
    resp, err := b.getResponse()
    if err != nil {
        return "", err
    }
    if resp.Body == nil {
        return "", nil
    }
    data, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        return "", err
    }

    return string(data), nil
}

func (b *HttpRequestBuilder) AsBytes() ([]byte, os.Error) {
    resp, err := b.getResponse()
    if err != nil {
        return nil, err
    }
    if resp.Body == nil {
        return nil, nil
    }
    data, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        return nil, err
    }

    return data, nil
}

func (b *HttpRequestBuilder) AsFile(filename string) os.Error {
    f, err := os.Open(filename, os.O_RDWR|os.O_CREATE, 0644)
    if err != nil {
        return err
    }
    defer f.Close()

    resp, err := b.getResponse()
    if err != nil {
        return err
    }
    if resp.Body == nil {
        return nil
    }
    _, err = io.Copy(f, resp.Body)
    if err != nil {
        return err
    }
    return nil
}

func (b *HttpRequestBuilder) AsResponse() (*http.Response, os.Error) {
    return b.getResponse()
}

func (b *HttpRequestBuilder) Close() {
    if b.clientConn != nil {
        tcpConn, _ := b.clientConn.Close()
        if tcpConn != nil {
            tcpConn.Close()
        }
    }
}
<commit_msg>gofmt<commit_after>package httplib

import (
    "bytes"
    "crypto/tls"
    "http"
    "io"
    "io/ioutil"
    "net"
    "os"
    "strings"
)

var defaultUserAgent = "httplib.go"

var debugprint = false

type Client struct {
    conn    *http.ClientConn
    lastURL *http.URL
}

type nopCloser struct {
    io.Reader
}

func (nopCloser) Close() os.Error { return nil }

func getNopCloser(buf *bytes.Buffer) nopCloser {
    return nopCloser{buf}
}

func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") }

func newConn(url *http.URL) (*http.ClientConn, os.Error) {
    addr := url.Host
    if !hasPort(addr) {
        addr += ":" + url.Scheme
    }
    var conn net.Conn
    var err os.Error
    if url.Scheme == "http" {
        conn, err = net.Dial("tcp", "", addr)
        if err != nil {
            return nil, err
        }
    } else { // https
        conn, err = tls.Dial("tcp", "", addr, nil)
        if err != nil {
            return nil, err
        }
        h := url.Host
        if hasPort(h) {
            h = h[0:strings.LastIndex(h, ":")]
        }
        if err := conn.(*tls.Conn).VerifyHostname(h); err != nil {
            return nil, err
        }
    }

    return http.NewClientConn(conn, nil), nil
}

func getResponse(rawUrl string, req *http.Request) (*http.ClientConn, *http.Response, os.Error) {
    url, err := http.ParseURL(rawUrl)
    if err != nil {
        return nil, nil, err
    }
    req.URL = url
    if debugprint {
        dump, _ := http.DumpRequest(req, true)
        print(string(dump))
    }

    conn, err := newConn(url)
    if err != nil {
        return nil, nil, err
    }

    err = conn.Write(req)
    if err != nil {
        return nil, nil, err
    }

    resp, err := conn.Read()
    if err != nil {
        if err != http.ErrPersistEOF {
            return nil, nil, err
        }
    }
    return conn, resp, nil
}

func (client *Client) Request(rawurl string, method string, headers map[string]string, body string) (*http.Response, os.Error) {
    var url *http.URL
    var err os.Error
    if url, err = http.ParseURL(rawurl); err != nil {
        return nil, err
    }

    if client.conn == nil || client.lastURL.Host != url.Host {
        client.conn, err = newConn(url)
    }

    if headers == nil {
        headers = map[string]string{}
    }

    client.lastURL = url
    var req http.Request
    req.URL = url
    req.Method = method
    req.Header = headers
    req.UserAgent = headers["User-Agent"]
    if req.UserAgent == "" {
        req.UserAgent = "httplib.go"
    }
    req.Body = nopCloser{bytes.NewBufferString(body)}

    if debugprint {
        dump, _ := http.DumpRequest(&req, true)
        print(string(dump))
    }

    err = client.conn.Write(&req)
    if err != nil {
        return nil, err
    }

    resp, err := client.conn.Read()
    if err != nil {
        return nil, err
    }

    return resp, nil
}

func Get(url string) *HttpRequestBuilder {
    var req http.Request
    req.Method = "GET"
    req.Header = map[string]string{}
    req.UserAgent = defaultUserAgent
    return &HttpRequestBuilder{url, &req, nil, map[string]string{}}
}

func Post(url string) *HttpRequestBuilder {
    var req http.Request
    req.Method = "POST"
    req.Header = map[string]string{}
    req.UserAgent = defaultUserAgent
    return &HttpRequestBuilder{url, &req, nil, map[string]string{}}
}

func Put(url string) *HttpRequestBuilder {
    var req http.Request
    req.Method = "PUT"
    req.Header = map[string]string{}
    req.UserAgent = defaultUserAgent
    return &HttpRequestBuilder{url, &req, nil, map[string]string{}}
}

func Delete(url string) *HttpRequestBuilder {
    var req http.Request
    req.Method = "DELETE"
    req.Header = map[string]string{}
    req.UserAgent = defaultUserAgent
    return &HttpRequestBuilder{url, &req, nil, map[string]string{}}
}

type HttpRequestBuilder struct {
    url        string
    req        *http.Request
    clientConn *http.ClientConn
    params     map[string]string
}

func (b *HttpRequestBuilder) getResponse() (*http.Response, os.Error) {
    var paramBody string
    if b.params != nil && len(b.params) > 0 {
        var buf bytes.Buffer
        for k, v := range b.params {
            buf.WriteString(http.URLEscape(k))
            buf.WriteByte('=')
            buf.WriteString(http.URLEscape(v))
            buf.WriteByte('&')
        }
        paramBody = buf.String()
        paramBody = paramBody[0 : len(paramBody)-1]
    }
    if b.req.Method == "GET" && len(paramBody) > 0 {
        if strings.Index(b.url, "?") != -1 {
            b.url += "&" + paramBody
        } else {
            b.url = b.url + "?" + paramBody
        }
    } else if b.req.Method == "POST" && b.req.Body == nil && len(paramBody) > 0 {
        b.Header("Content-Type", "application/x-www-form-urlencoded")
        b.req.Body = nopCloser{bytes.NewBufferString(paramBody)}
        b.req.ContentLength = int64(len(paramBody))
    }

    conn, resp, err := getResponse(b.url, b.req)
    b.clientConn = conn
    return resp, err
}

func (b *HttpRequestBuilder) Header(key, value string) *HttpRequestBuilder {
    b.req.Header[key] = value
    return b
}

func (b *HttpRequestBuilder) Param(key, value string) *HttpRequestBuilder {
    b.params[key] = value
    return b
}

func (b *HttpRequestBuilder) Body(data interface{}) *HttpRequestBuilder {
    switch t := data.(type) {
    case string:
        b.req.Body = getNopCloser(bytes.NewBufferString(t))
        b.req.ContentLength = int64(len(t))
    case []byte:
        b.req.Body = getNopCloser(bytes.NewBuffer(t))
        b.req.ContentLength = int64(len(t))
    }
    return b
}

func (b *HttpRequestBuilder) AsString() (string, os.Error) {
    resp, err := b.getResponse()
    if err != nil {
        return "", err
    }
    if resp.Body == nil {
        return "", nil
    }
    data, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        return "", err
    }

    return string(data), nil
}

func (b *HttpRequestBuilder) AsBytes() ([]byte, os.Error) {
    resp, err := b.getResponse()
    if err != nil {
        return nil, err
    }
    if resp.Body == nil {
        return nil, nil
    }
    data, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        return nil, err
    }

    return data, nil
}

func (b *HttpRequestBuilder) AsFile(filename string) os.Error {
    f, err := os.Open(filename, os.O_RDWR|os.O_CREATE, 0644)
    if err != nil {
        return err
    }
    defer f.Close()

    resp, err := b.getResponse()
    if err != nil {
        return err
    }
    if resp.Body == nil {
        return nil
    }
    _, err = io.Copy(f, resp.Body)
    if err != nil {
        return err
    }
    return nil
}

func (b *HttpRequestBuilder) AsResponse() (*http.Response, os.Error) {
    return b.getResponse()
}

func (b *HttpRequestBuilder) Close() {
    if b.clientConn != nil {
        tcpConn, _ := b.clientConn.Close()
        if tcpConn != nil {
            tcpConn.Close()
        }
    }
}
<|endoftext|>


<commit_before>package raft

import (
  "net/http"
  "fmt"
  "io/ioutil"
  "strconv"
  "encoding/json"
)

// Used to store any field to interact with the RPC
// functions
type RPC struct {
  Server *http.Server
}

// Start the RPC server
func (rpc *RPC) Start() error {
  // TODO: test the server
  go func() {
  }()

  err := rpc.Server.ListenAndServe()

  return err
}

// Create RPC connection
// TODO: create error object for http responses
func NewRPC(sm *StateMachine) (*RPC, error) {
  mux := http.NewServeMux()

  addr := fmt.Sprintf("%s:%d", sm.Configuration.RPCHost, sm.Configuration.RPCPort)

  server := &http.Server{
    Addr: addr,
    Handler: mux,
  }

  mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
    fmt.Fprintf(w, "Hello World!")
  })
  mux.HandleFunc("/appendEntry", func(w http.ResponseWriter, r *http.Request) {
    // Declare a variable that will be used to be the statusCode of the response
    // because WriteHeader actually writes header, it is not possible to play
    // with the header in a defer function's call
    statusCode := 200

    // Acquire the lock since we'll write
    sm.State.C <- struct{}{}
    defer func() {
      // Send back the Current-Term in the response
      w.Header().Add("X-Current-Term", strconv.FormatUint(sm.State.CurrentTerm, 10))
      w.WriteHeader(statusCode)

      // Unlock
      <- sm.State.C
    }()

    err := r.ParseForm()
    if err != nil {
      statusCode = 422
      return
    }

    // start parsing term
    term, ok := r.Form["term"]
    if !ok || term[0] == "" {
      statusCode = 422
      return
    }

    termId, err := strconv.ParseUint(term[0], 10, 0)
    if err != nil {
      statusCode = 422
      return
    }

    if termId < sm.State.CurrentTerm {
      statusCode = 422
      return
    }
    // end parsing term

    //// start parsing leaderCommit
    //leaderCommitValue, ok := r.Form["leaderCommit"]
    //if !ok || leaderCommitValue[0] == "" {
      //statusCode = 422
      //return
    //}

    //leaderCommit, err := strconv.ParseInt(leaderCommitValue[0], 10, 0)
    //if err != nil {
      //statusCode = 422
      //return
    //}
    //// end parsing leaderCommit

    //// start parsing prevLogTerm
    //prevLogTerm, ok := r.Form["prevLogTerm"]
    //if !ok || prevLogTerm[0] == "" {
      //statusCode = 422
      //return
    //}

    //prevTerm, err := strconv.ParseUint(prevLogTerm[0], 10, 0)
    //if err != nil {
      //statusCode = 422
      //return
    //}
    //// end parsing prevLogTerm

    //// start parsing prevLogIndex
    //prevLogIndex, ok := r.Form["prevLogIndex"]
    //if !ok || prevLogIndex[0] == "" {
      //statusCode = 422
      //return
    //}

    //prevIndex, err := strconv.ParseInt(prevLogIndex[0], 10, 0)
    //if err != nil {
      //statusCode = 422
      //return
    //}

    //// Acquire the lock
    //sm.Storage.C <- struct{}{}
    //defer func() {
      //<- sm.Storage.C
    //}()

    //// end parsing term

    // start parsing body
    body, err := ioutil.ReadAll(r.Body)
    if err != nil {
      statusCode = 400
      return
    }
    switch contentType := r.Header.Get("Content-Type"); contentType {
      case "application/json":
        var logs Logs

        err := json.Unmarshal(body, &logs)
        if err != nil {
          statusCode = 400
          return
        }

        if len(logs) == 0 && termId != sm.State.CurrentTerm {
          sm.State.CurrentTerm = termId
          sm.Exec("term::changed", termId)
        }
      default:
        statusCode = 415
        return
    }
    // end parsing body

  })

  rpc := &RPC{
    Server: server,
  }

  return rpc, nil
}
<commit_msg>Modified doc<commit_after>package raft

import (
  "net/http"
  "fmt"
  "io/ioutil"
  "strconv"
  "encoding/json"
)

// Used to store any field to interact with the RPC
// functions
type RPC struct {
  Server *http.Server
}

// Start the RPC server
func (rpc *RPC) Start() error {
  // TODO: test the server
  go func() {
  }()

  err := rpc.Server.ListenAndServe()

  return err
}

// Create RPC connection
// TODO: create error object for http responses
func NewRPC(sm *StateMachine) (*RPC, error) {
  mux := http.NewServeMux()

  addr := fmt.Sprintf("%s:%d", sm.Configuration.RPCHost, sm.Configuration.RPCPort)

  server := &http.Server{
    Addr: addr,
    Handler: mux,
  }

  mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
    fmt.Fprintf(w, "Hello World!")
  })
  mux.HandleFunc("/appendEntry", func(w http.ResponseWriter, r *http.Request) {
    // Declare a variable that will be used to be the statusCode of the response
    // because WriteHeader actually writes header, it is not possible to play
    // with the header in a defer function's call
    statusCode := 200

    // Acquire the lock since we'll write and we don't want dirty reads
    sm.State.C <- struct{}{}
    defer func() {
      // Send back the Current-Term in the response
      w.Header().Add("X-Current-Term", strconv.FormatUint(sm.State.CurrentTerm, 10))
      w.WriteHeader(statusCode)

      // Unlock
      <- sm.State.C
    }()

    err := r.ParseForm()
    if err != nil {
      statusCode = 422
      return
    }

    // start parsing term
    term, ok := r.Form["term"]
    if !ok || term[0] == "" {
      statusCode = 422
      return
    }

    termId, err := strconv.ParseUint(term[0], 10, 0)
    if err != nil {
      statusCode = 422
      return
    }

    if termId < sm.State.CurrentTerm {
      statusCode = 422
      return
    }
    // end parsing term

    //// start parsing leaderCommit
    //leaderCommitValue, ok := r.Form["leaderCommit"]
    //if !ok || leaderCommitValue[0] == "" {
      //statusCode = 422
      //return
    //}

    //leaderCommit, err := strconv.ParseInt(leaderCommitValue[0], 10, 0)
    //if err != nil {
      //statusCode = 422
      //return
    //}
    //// end parsing leaderCommit

    //// start parsing prevLogTerm
    //prevLogTerm, ok := r.Form["prevLogTerm"]
    //if !ok || prevLogTerm[0] == "" {
      //statusCode = 422
      //return
    //}

    //prevTerm, err := strconv.ParseUint(prevLogTerm[0], 10, 0)
    //if err != nil {
      //statusCode = 422
      //return
    //}
    //// end parsing prevLogTerm

    //// start parsing prevLogIndex
    //prevLogIndex, ok := r.Form["prevLogIndex"]
    //if !ok || prevLogIndex[0] == "" {
      //statusCode = 422
      //return
    //}

    //prevIndex, err := strconv.ParseInt(prevLogIndex[0], 10, 0)
    //if err != nil {
      //statusCode = 422
      //return
    //}

    //// Acquire the lock
    //sm.Storage.C <- struct{}{}
    //defer func() {
      //<- sm.Storage.C
    //}()

    //// end parsing term

    // start parsing body
    body, err := ioutil.ReadAll(r.Body)
    if err != nil {
      statusCode = 400
      return
    }
    switch contentType := r.Header.Get("Content-Type"); contentType {
      case "application/json":
        var logs Logs

        err := json.Unmarshal(body, &logs)
        if err != nil {
          statusCode = 400
          return
        }

        if len(logs) == 0 && termId != sm.State.CurrentTerm {
          sm.State.CurrentTerm = termId
          sm.Exec("term::changed", termId)
        }
      default:
        statusCode = 415
        return
    }
    // end parsing body

  })

  rpc := &RPC{
    Server: server,
  }

  return rpc, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2013-2014 by Michael Dvorkin. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package donna

import(
	`fmt`
	`io/ioutil`
	`regexp`
	`strconv`
	`strings`
	`time`
)

const (
	escRed   = "\033[0;31m"
	escGreen = "\033[0;32m"
	escTeal  = "\033[0;36m"
	escNone  = "\033[0m"
)

func (e *Engine) replBestMove(move Move) *Engine {
	fmt.Printf(escTeal + "Donna's move: %s", move)
	if game.nodes == 0 {
		fmt.Printf(" (book)")
	}
	fmt.Println(escNone + "\n")

	return e
}

func (e *Engine) replPrincipal(depth, score, status int, duration int64) {
	fmt.Printf(`%2d %s %10d %10d %9d   `, depth, ms(duration), game.nodes, game.qnodes, nps(duration))
	switch status {
	case WhiteWon:
		fmt.Println(`1-0 White Checkmates`)
	case BlackWon:
		fmt.Println(`0-1 Black Checkmates`)
	case Stalemate:
		fmt.Println(`1/2 Stalemate`)
	case Repetition:
		fmt.Println(`1/2 Repetition`)
	case FiftyMoves:
		fmt.Println(`1/2 Fifty Moves`)
	case WhiteWinning, BlackWinning: // Show moves till checkmate.
		fmt.Printf("%4dX   %v Checkmate\n", (Checkmate - abs(score)) / 2, game.rootpv)
	default:
		fmt.Printf("%5.2f   %v\n", float32(score) / float32(onePawn), game.rootpv)
	}
}

func (e *Engine) Repl() *Engine {
	var game *Game
	var position *Position

	setup := func() {
		if game == nil || position == nil {
			game = NewGame()
			position = game.start()
			fmt.Printf("%s\n", position)
		}
	}

	think := func() {
		if move := game.Think(); move != 0 {
			position = position.makeMove(move)
			fmt.Printf("%s\n", position)
		}
	}

	benchmark := func(fileName string) {
		maxDepth, moveTime := e.options.maxDepth, e.options.moveTime
		e.options.maxDepth, e.options.moveTime = 0, 10000
		defer func() {
			e.options.maxDepth, e.options.moveTime = maxDepth, moveTime
		}()

		content, err := ioutil.ReadFile(fileName)
		if err == nil {
			total, solved := 0, 0
			re := regexp.MustCompile(`[\+\?!]`)

			NextLine:
			for _, line := range strings.Split(string(content), "\n") {
				if len(line) > 0 && line[0] != '#' {
					total++
					game := NewGame(line)
					position := game.start()

					best := strings.Split(line, ` # `)[1] // TODO: add support for "am" (avoid move).
					fmt.Printf(escTeal + "%d) %s for %s" + escNone + "\n%s\n", total, best, C(position.color), position)
					move := game.Think()

					for _, nextBest := range strings.Split(best, ` `) {
						if move.str() == re.ReplaceAllLiteralString(nextBest, ``) {
							solved++
							fmt.Printf(escGreen + "%d) Solved (%d/%d %2.1f%%)\n\n\n" + escNone, total, solved, total - solved, float32(solved) * 100.0 / float32(total))
							continue NextLine
						}
					}
					fmt.Printf(escRed + "%d) Not solved (%d/%d %2.1f%%)\n\n\n" + escNone, total, solved, total - solved, float32(solved) * 100.0 / float32(total))
				}
			}
		} else {
			fmt.Printf("Could not open benchmark file '%s'\n", fileName)
		}
	}

	perft := func(parameter string) {
		if parameter == `` {
			parameter = `5`
		}
		if depth, err := strconv.Atoi(parameter); err == nil {
			position := NewGame().start()
			start := time.Now()
			total := position.Perft(depth)
			finish := since(start)
			fmt.Printf("  Depth: %d\n", depth)
			fmt.Printf("  Nodes: %d\n", total)
			fmt.Printf("Elapsed: %s\n", ms(finish))
			fmt.Printf("Nodes/s: %dK\n", total / finish)
		}
	}

	fmt.Printf("Donna v%s Copyright (c) 2014 by Michael Dvorkin. All Rights Reserved.\nType ? for help.\n\n", Version)
	for command, parameter := ``, ``; ; command, parameter = ``, `` {
		fmt.Print(`donna> `)
		fmt.Scanln(&command, &parameter)

		switch command {
		case ``:
		case `bench`:
			benchmark(parameter)
		case `exit`, `quit`:
			return e
		case `go`:
			setup()
			think()
		case `help`, `?`:
			fmt.Println("The commands are:\n\n" +
				"  bench <file>   Run benchmarks\n" +
				"  exit           Exit the program\n" +
				"  go             Take side and make a move\n" +
				"  help           Display this help\n" +
				"  new            Start new game\n" +
				"  perft [depth]  Run perft test\n" +
				"  score          Show evaluation summary\n" +
				"  undo           Undo last move\n\n" +
				"To make a move use algebraic notation, for example e2e4, Ng1f3, or e7e8Q\n")
		case `new`:
			game, position = nil, nil
			setup()
		case `perft`:
			perft(parameter)
		case `score`:
			setup()
			_, metrics := position.EvaluateWithTrace()
			Summary(metrics)
		case `undo`:
			if position != nil {
				position = position.undoLastMove()
				fmt.Printf("%s\n", position)
			}
		default:
			setup()
			if move, validMoves := NewMoveFromString(position, command); move != 0 {
				position = position.makeMove(move)
				think()
			} else { // Invalid move or non-evasion on check.
				fmt.Printf("%s appears to be an invalid move. Valid moves are %v\n", command, validMoves)
			}
		}
	}
	return e
}
<commit_msg>Suppress fancy notation when printing a list of valid moves<commit_after>// Copyright (c) 2013-2014 by Michael Dvorkin. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package donna

import(
	`fmt`
	`io/ioutil`
	`regexp`
	`strconv`
	`strings`
	`time`
)

const (
	escRed   = "\033[0;31m"
	escGreen = "\033[0;32m"
	escTeal  = "\033[0;36m"
	escNone  = "\033[0m"
)

func (e *Engine) replBestMove(move Move) *Engine {
	fmt.Printf(escTeal + "Donna's move: %s", move)
	if game.nodes == 0 {
		fmt.Printf(" (book)")
	}
	fmt.Println(escNone + "\n")

	return e
}

func (e *Engine) replPrincipal(depth, score, status int, duration int64) {
	fmt.Printf(`%2d %s %10d %10d %9d   `, depth, ms(duration), game.nodes, game.qnodes, nps(duration))
	switch status {
	case WhiteWon:
		fmt.Println(`1-0 White Checkmates`)
	case BlackWon:
		fmt.Println(`0-1 Black Checkmates`)
	case Stalemate:
		fmt.Println(`1/2 Stalemate`)
	case Repetition:
		fmt.Println(`1/2 Repetition`)
	case FiftyMoves:
		fmt.Println(`1/2 Fifty Moves`)
	case WhiteWinning, BlackWinning: // Show moves till checkmate.
		fmt.Printf("%4dX   %v Checkmate\n", (Checkmate - abs(score)) / 2, game.rootpv)
	default:
		fmt.Printf("%5.2f   %v\n", float32(score) / float32(onePawn), game.rootpv)
	}
}

func (e *Engine) Repl() *Engine {
	var game *Game
	var position *Position

	setup := func() {
		if game == nil || position == nil {
			game = NewGame()
			position = game.start()
			fmt.Printf("%s\n", position)
		}
	}

	think := func() {
		if move := game.Think(); move != 0 {
			position = position.makeMove(move)
			fmt.Printf("%s\n", position)
		}
	}

	benchmark := func(fileName string) {
		maxDepth, moveTime := e.options.maxDepth, e.options.moveTime
		e.options.maxDepth, e.options.moveTime = 0, 10000
		defer func() {
			e.options.maxDepth, e.options.moveTime = maxDepth, moveTime
		}()

		content, err := ioutil.ReadFile(fileName)
		if err == nil {
			total, solved := 0, 0
			re := regexp.MustCompile(`[\+\?!]`)

			NextLine:
			for _, line := range strings.Split(string(content), "\n") {
				if len(line) > 0 && line[0] != '#' {
					total++
					game := NewGame(line)
					position := game.start()

					best := strings.Split(line, ` # `)[1] // TODO: add support for "am" (avoid move).
					fmt.Printf(escTeal + "%d) %s for %s" + escNone + "\n%s\n", total, best, C(position.color), position)
					move := game.Think()

					for _, nextBest := range strings.Split(best, ` `) {
						if move.str() == re.ReplaceAllLiteralString(nextBest, ``) {
							solved++
							fmt.Printf(escGreen + "%d) Solved (%d/%d %2.1f%%)\n\n\n" + escNone, total, solved, total - solved, float32(solved) * 100.0 / float32(total))
							continue NextLine
						}
					}
					fmt.Printf(escRed + "%d) Not solved (%d/%d %2.1f%%)\n\n\n" + escNone, total, solved, total - solved, float32(solved) * 100.0 / float32(total))
				}
			}
		} else {
			fmt.Printf("Could not open benchmark file '%s'\n", fileName)
		}
	}

	perft := func(parameter string) {
		if parameter == `` {
			parameter = `5`
		}
		if depth, err := strconv.Atoi(parameter); err == nil {
			position := NewGame().start()
			start := time.Now()
			total := position.Perft(depth)
			finish := since(start)
			fmt.Printf("  Depth: %d\n", depth)
			fmt.Printf("  Nodes: %d\n", total)
			fmt.Printf("Elapsed: %s\n", ms(finish))
			fmt.Printf("Nodes/s: %dK\n", total / finish)
		}
	}

	fmt.Printf("Donna v%s Copyright (c) 2014 by Michael Dvorkin. All Rights Reserved.\nType ? for help.\n\n", Version)
	for command, parameter := ``, ``; ; command, parameter = ``, `` {
		fmt.Print(`donna> `)
		fmt.Scanln(&command, &parameter)

		switch command {
		case ``:
		case `bench`:
			benchmark(parameter)
		case `exit`, `quit`:
			return e
		case `go`:
			setup()
			think()
		case `help`, `?`:
			fmt.Println("The commands are:\n\n" +
				"  bench <file>   Run benchmarks\n" +
				"  exit           Exit the program\n" +
				"  go             Take side and make a move\n" +
				"  help           Display this help\n" +
				"  new            Start new game\n" +
				"  perft [depth]  Run perft test\n" +
				"  score          Show evaluation summary\n" +
				"  undo           Undo last move\n\n" +
				"To make a move use algebraic notation, for example e2e4, Ng1f3, or e7e8Q\n")
		case `new`:
			game, position = nil, nil
			setup()
		case `perft`:
			perft(parameter)
		case `score`:
			setup()
			_, metrics := position.EvaluateWithTrace()
			Summary(metrics)
		case `undo`:
			if position != nil {
				position = position.undoLastMove()
				fmt.Printf("%s\n", position)
			}
		default:
			setup()
			if move, validMoves := NewMoveFromString(position, command); move != 0 {
				position = position.makeMove(move)
				think()
			} else { // Invalid move or non-evasion on check.
				fancy := e.fancy; e.fancy = false
				fmt.Printf("%s appears to be an invalid move; valid moves are %v\n", command, validMoves)
				e.fancy = fancy
			}
		}
	}
	return e
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/sclevine/agouti"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/suite"
)

const fixturesPath = "test_fixtures"

var baseURL string

type acceptanceTestSuite struct {
	suite.Suite
	driver *agouti.WebDriver
	page   *agouti.Page
}

func TestAcceptanceTests(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping acceptance tests in short mode.")
	}
	suite.Run(t, new(acceptanceTestSuite))
}

func (s *acceptanceTestSuite) SetupSuite() {
	var err error

	baseURL = fmt.Sprintf("https://%s/", config.addr)

	config.gitHubClientID = "abc"
	config.gitHubClientSecret = "xyz"
	config.tlsCert = filepath.Join(fixturesPath, "cert.pem")
	config.tlsKey = filepath.Join(fixturesPath, "key.pem")

	go main()

	s.driver = agouti.NewWebDriver("http://{{.Address}}", []string{"phantomjs", "--webdriver={{.Address}}", "--ignore-ssl-errors=true"})
	s.driver.Start()

	s.page, err = s.driver.NewPage(agouti.Desired(agouti.NewCapabilities().Browser("chrome")))
	if err != nil {
		s.T().Error(err)
	}

	// don't verify our development TLS certificates
	http.DefaultTransport = &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}

	// Make sure server is actually running (see go main() call above) before
	// running tests to avoid race conditions
	for {
		_, err := http.Get(baseURL)

		if err == nil {
			break
		} else {
			switch err.(type) {
			case *url.Error:
				if strings.HasSuffix(err.Error(), "connection refused") {
					time.Sleep(500 * time.Millisecond)
					continue
				}
			default:
				s.T().Error(err)
				break
			}
		}
	}
}

func (s *acceptanceTestSuite) TearDownSuite() {
	s.driver.Stop()
}

func (s *acceptanceTestSuite) TestDebugVarsExposed() {
	testURL := baseURL + "debug/vars"
	resp, err := http.Get(testURL)
	if err != nil {
		s.T().Error(err)
	}

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		s.T().Error(err)
	}

	assert.Contains(s.T(), string(body), "cmdline")
	assert.Contains(s.T(), string(body), "memstats")
}

func (s *acceptanceTestSuite) TestHomePageForJavascriptErrors() {
	testURL := baseURL
	err := s.page.Navigate(testURL)
	if err != nil {
		s.T().Error(err)
	}

	logs, err := s.page.ReadAllLogs("browser")
	if err != nil {
		s.T().Error(err)
	}

	u, _ := s.page.URL()
	assert.Equal(s.T(), testURL, u)

	for _, log := range logs {
		assert.NotEqual(s.T(), "WARNING", log.Level, log.Message)
		assert.NotEqual(s.T(), "SEVERE", log.Level, log.Message)
	}
}

func (s *acceptanceTestSuite) TestPageNotFound() {
	resp, err := http.Get(baseURL + "non-existentent-page")
	if err != nil {
		s.T().Error(err)
	}

	assert.Equal(s.T(), http.StatusNotFound, resp.StatusCode)
}
<commit_msg>Move logic that waits for connection to own func<commit_after>package main

import (
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/sclevine/agouti"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/suite"
)

const fixturesPath = "test_fixtures"

var baseURL string

type acceptanceTestSuite struct {
	suite.Suite
	driver *agouti.WebDriver
	page   *agouti.Page
}

func TestAcceptanceTests(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping acceptance tests in short mode.")
	}
	suite.Run(t, new(acceptanceTestSuite))
}

func (s *acceptanceTestSuite) SetupSuite() {
	var err error

	baseURL = fmt.Sprintf("https://%s/", config.addr)

	config.gitHubClientID = "abc"
	config.gitHubClientSecret = "xyz"
	config.tlsCert = filepath.Join(fixturesPath, "cert.pem")
	config.tlsKey = filepath.Join(fixturesPath, "key.pem")
	// FIXME need to configure and clean up database

	go main()

	s.driver = agouti.NewWebDriver("http://{{.Address}}", []string{"phantomjs", "--webdriver={{.Address}}", "--ignore-ssl-errors=true"})
	s.driver.Start()

	s.page, err = s.driver.NewPage(agouti.Desired(agouti.NewCapabilities().Browser("chrome")))
	if err != nil {
		s.T().Error(err)
	}

	// don't verify our development TLS certificates
	http.DefaultTransport = &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}

	// Make sure server is actually running (see go main() call above) before
	// running tests to avoid race conditions
	err = waitForServer(baseURL)
	if err != nil {
		s.T().Error(err)
	}
}

func (s *acceptanceTestSuite) TearDownSuite() {
	s.driver.Stop()
}

func (s *acceptanceTestSuite) TestDebugVarsExposed() {
	testURL := baseURL + "debug/vars"
	resp, err := http.Get(testURL)
	if err != nil {
		s.T().Error(err)
	}

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		s.T().Error(err)
	}

	assert.Contains(s.T(), string(body), "cmdline")
	assert.Contains(s.T(), string(body), "memstats")
}

func (s *acceptanceTestSuite) TestHomePageForJavascriptErrors() {
	testURL := baseURL
	err := s.page.Navigate(testURL)
	if err != nil {
		s.T().Error(err)
	}

	logs, err := s.page.ReadAllLogs("browser")
	if err != nil {
		s.T().Error(err)
	}

	u, _ := s.page.URL()
	assert.Equal(s.T(), testURL, u)

	for _, log := range logs {
		assert.NotEqual(s.T(), "WARNING", log.Level, log.Message)
		assert.NotEqual(s.T(), "SEVERE", log.Level, log.Message)
	}
}

func (s *acceptanceTestSuite) TestPageNotFound() {
	resp, err := http.Get(baseURL + "non-existentent-page")
	if err != nil {
		s.T().Error(err)
	}

	assert.Equal(s.T(), http.StatusNotFound, resp.StatusCode)
}

func waitForServer(u string) error {
	for {
		_, err := http.Get(u)

		if err == nil {
			break
		} else {
			switch err.(type) {
			case *url.Error:
				if strings.HasSuffix(err.Error(), "connection refused") {
					time.Sleep(500 * time.Millisecond)
					continue
				}
				return err
			default:
				return err
				break
			}
		}
	}

	return nil
}
<|endoftext|>