<commit_before>package shards

// These are the endpoints required for
// cheshire sharding to work.
const (
	//router table get endpoint
	// response format
	// {
	//  "strest" :{...}
	//  "router_table" : <the router table>
	// }
	ROUTERTABLE_GET = "/__c/rt/get"

	// Sets the router table on this server
	// @method POST
	// @param router_table The router table
	ROUTERTABLE_SET = "/__c/rt/set"

	PARTITION_LOCK   = "/__c/pt/lock"
	PARTITION_UNLOCK = "/__c/pt/unlock"

	// Delete a partition from this server
	PARTITION_DELETE = "/__c/pt/delete"

	// Is a ping endpoint to check for liveness and
	// to check the revision of the router table.
	// response format
	// {
	//  "strest" :{...}
	//  "ts" : <ISOFORMATED TIMESTAMP>
	//  "rt_revision" : <router table revision>
	// }
	CHECKIN = "/__c/checkin"

	// Creates a stream of data for the given partition
	// @param partition the int partition
	// data is in the key "data"
	PARTITION_EXPORT = "/__c/pt/export"

	// Initializes an import request between two shards
	//
	// @method POST
	// @param partition the partition to import data
	// @param source the http address to pull data from in the form http://address:port
	PARTITION_IMPORT = "/__c/pt/import"
)

//These are the required return error codes for various situations
const (
	// We reserve 630-640 for router table issues

	// return when the requester has an old router table
	E_ROUTER_TABLE_OLD = 632

	// requester has a newer router table then us, request they update ours
	E_SEND_ROUTER_TABLE = 633

	// the requested partition is locked.  requester should try back in a bit
	E_PARTITION_LOCKED = 634

	// The requested partition does not live on this shard
	E_NOT_MY_PARTITION = 635
)

// Param Names
const (
	//The partition val (an integer from 0 to TotalPartitions)
	P_PARTITION = "_p"

	// The version of the router table
	P_REVISION = "_v"

	//The query type.
	// This defines how the request can be handled by the router.
	// Possible values:
	// single : return a single result (the first response received)
	// all : (default) return values for all servers, will make an effort to retry on failure, but will generally return error results.
	// all_q : return values for all servers (queue requests if needed, retry until response).  This would typically be for posting
	// none_q : returns success immediately, queues the request and make best effort to ensure it is delivered (TODO)
	P_QUERY_TYPE = "_qt"
)
<commit_msg>add shard key param<commit_after>package shards

// These are the endpoints required for
// cheshire sharding to work.
const (
	//router table get endpoint
	// response format
	// {
	//  "strest" :{...}
	//  "router_table" : <the router table>
	// }
	ROUTERTABLE_GET = "/__c/rt/get"

	// Sets the router table on this server
	// @method POST
	// @param router_table The router table
	ROUTERTABLE_SET = "/__c/rt/set"

	PARTITION_LOCK   = "/__c/pt/lock"
	PARTITION_UNLOCK = "/__c/pt/unlock"

	// Delete a partition from this server
	PARTITION_DELETE = "/__c/pt/delete"

	// Is a ping endpoint to check for liveness and
	// to check the revision of the router table.
	// response format
	// {
	//  "strest" :{...}
	//  "ts" : <ISOFORMATED TIMESTAMP>
	//  "rt_revision" : <router table revision>
	// }
	CHECKIN = "/__c/checkin"

	// Creates a stream of data for the given partition
	// @param partition the int partition
	// data is in the key "data"
	PARTITION_EXPORT = "/__c/pt/export"

	// Initializes an import request between two shards
	//
	// @method POST
	// @param partition the partition to import data
	// @param source the http address to pull data from in the form http://address:port
	PARTITION_IMPORT = "/__c/pt/import"
)

//These are the required return error codes for various situations
const (
	// We reserve 630-640 for router table issues

	// return when the requester has an old router table
	E_ROUTER_TABLE_OLD = 632

	// requester has a newer router table then us, request they update ours
	E_SEND_ROUTER_TABLE = 633

	// the requested partition is locked.  requester should try back in a bit
	E_PARTITION_LOCKED = 634

	// The requested partition does not live on this shard
	E_NOT_MY_PARTITION = 635
)

// Param Names
const (
	//The partition val (an integer from 0 to TotalPartitions)
	P_PARTITION = "_p"

	// The version of the router table
	P_REVISION = "_v"

	//The shard key, should only be used when passing to a proxy
	P_SHARD_KEY = "_sk"

	//The query type.
	// This defines how the request can be handled by the router.
	// Possible values:
	// single : return a single result (the first response received)
	// all : (default) return values for all servers, will make an effort to retry on failure, but will generally return error results.
	// all_q : return values for all servers (queue requests if needed, retry until response).  This would typically be for posting
	// none_q : returns success immediately, queues the request and make best effort to ensure it is delivered (TODO)
	P_QUERY_TYPE = "_qt"
)
<|endoftext|>


<commit_before>package etcd_test

import (
	"crypto/tls"
	"fmt"
	"github.com/FidelityInternational/etcd-leader-monitor/etcd"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"net/http"
	"net/http/httptest"
	"net/url"
)

var _ = Describe("#NewClient", func() {
	It("creates an etcd client", func() {
		config := &etcd.Config{
			EtcdIP:     "1.1.1.1",
			HTTPClient: &http.Client{},
		}
		client := etcd.NewClient(config)
		Ω(client.Config.HTTPClient).Should(BeAssignableToTypeOf(&http.Client{}))
		Ω(client.Config.EtcdIP).Should(Equal("1.1.1.1"))
	})
})

var _ = Describe("#GetLeaderStats", func() {
	Context("when the http requests raises an error", func() {
		var client *etcd.Client

		BeforeEach(func() {
			server := httptest.NewServer((http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				w.WriteHeader(1)
			})))

			transport := &http.Transport{
				Proxy: func(req *http.Request) (*url.URL, error) {
					return url.Parse(server.URL)
				},
				TLSClientConfig: &tls.Config{},
			}
			httpClient := &http.Client{Transport: transport}

			config := &etcd.Config{
				EtcdIP:     "1.1.1.1",
				HTTPClient: httpClient,
			}
			client = etcd.NewClient(config)
		})

		It("returns the error", func() {
			_, _, err := client.GetLeaderStats()
			Ω(err).Should(MatchError(`Get http://1.1.1.1:4001/v2/stats/leader: malformed HTTP status code "1"`))
		})
	})

	Context("when unmarshalling the json response raises an error", func() {
		var client *etcd.Client

		BeforeEach(func() {
			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				w.WriteHeader(200)
				w.Header().Set("Content-Type", "application/json")
				fmt.Fprintln(w, `leader":"6a0b69a54415a491","followers":{"a0294459200078aa":{"latency":{"current":0.001199,"average":0.0023682517720168754,"standardDeviation":0.4302199179552562,"minimum":0.000654,"maximum":1996.564157},"counts":{"fail":16,"success":21538911}},"b5c352b4495e4195":{"latency":{"current":0.001609,"average":0.002361467019756358,"standardDeviation":0.00506414137059054,"minimum":0.00088,"maximum":5.153269},"counts":{"fail":7,"success":1617908}}}}`)
			}))

			transport := &http.Transport{
				Proxy: func(req *http.Request) (*url.URL, error) {
					return url.Parse(server.URL)
				},
				TLSClientConfig: &tls.Config{},
			}
			httpClient := &http.Client{Transport: transport}

			config := &etcd.Config{
				EtcdIP:     "1.1.1.1",
				HTTPClient: httpClient,
			}
			client = etcd.NewClient(config)
		})

		It("returns the error", func() {
			_, _, err := client.GetLeaderStats()
			Ω(err).Should(MatchError("invalid character 'l' looking for beginning of value"))
		})
	})

	Context("when no errors are raised", func() {
		Context("and the etcd is a leader", func() {
			var client *etcd.Client

			BeforeEach(func() {
				server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					w.WriteHeader(200)
					w.Header().Set("Content-Type", "application/json")
					fmt.Fprintln(w, `{"leader":"6a0b69a54415a491","followers":{"a0294459200078aa":{"latency":{"current":0.001199,"average":0.0023682517720168754,"standardDeviation":0.4302199179552562,"minimum":0.000654,"maximum":1996.564157},"counts":{"fail":16,"success":21538911}},"b5c352b4495e4195":{"latency":{"current":0.001609,"average":0.002361467019756358,"standardDeviation":0.00506414137059054,"minimum":0.00088,"maximum":5.153269},"counts":{"fail":7,"success":1617908}}}}`)
				}))

				transport := &http.Transport{
					Proxy: func(req *http.Request) (*url.URL, error) {
						return url.Parse(server.URL)
					},
					TLSClientConfig: &tls.Config{},
				}
				httpClient := &http.Client{Transport: transport}

				config := &etcd.Config{
					EtcdIP:     "1.1.1.1",
					HTTPClient: httpClient,
				}
				client = etcd.NewClient(config)
			})

			It("returns a count of followers and leader true", func() {
				leader, followers, _ := client.GetLeaderStats()
				Ω(leader).Should(BeTrue())
				Ω(followers).Should(Equal(2))
			})
		})

		Context("and the etcd is not a leader", func() {
			var client *etcd.Client

			BeforeEach(func() {
				server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					w.WriteHeader(200)
					w.Header().Set("Content-Type", "application/json")
					fmt.Fprintln(w, `{"message":"not current leader"}`)
				}))

				transport := &http.Transport{
					Proxy: func(req *http.Request) (*url.URL, error) {
						return url.Parse(server.URL)
					},
					TLSClientConfig: &tls.Config{},
				}
				httpClient := &http.Client{Transport: transport}

				config := &etcd.Config{
					EtcdIP:     "1.1.1.1",
					HTTPClient: httpClient,
				}
				client = etcd.NewClient(config)
			})

			It("returns leader false", func() {
				leader, _, _ := client.GetLeaderStats()
				Ω(leader).Should(BeFalse())
			})
		})
	})
})
<commit_msg>update negative test for http errors to work with go 1.7<commit_after>package etcd_test

import (
	"crypto/tls"
	"fmt"
	"github.com/FidelityInternational/etcd-leader-monitor/etcd"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"net/http"
	"net/http/httptest"
	"net/url"
)

var _ = Describe("#NewClient", func() {
	It("creates an etcd client", func() {
		config := &etcd.Config{
			EtcdIP:     "1.1.1.1",
			HTTPClient: &http.Client{},
		}
		client := etcd.NewClient(config)
		Ω(client.Config.HTTPClient).Should(BeAssignableToTypeOf(&http.Client{}))
		Ω(client.Config.EtcdIP).Should(Equal("1.1.1.1"))
	})
})

var _ = Describe("#GetLeaderStats", func() {
	Context("when the http requests raises an error", func() {
		var client *etcd.Client

		BeforeEach(func() {
			config := &etcd.Config{
				EtcdIP:     "1.1.1.1:1",
				HTTPClient: &http.Client{},
			}
			client = etcd.NewClient(config)
		})

		It("returns the error", func() {
			_, _, err := client.GetLeaderStats()
			Ω(err).Should(MatchError(`Get http://1.1.1.1:1:4001/v2/stats/leader: dial tcp: too many colons in address 1.1.1.1:1:4001`))
		})
	})

	Context("when unmarshalling the json response raises an error", func() {
		var client *etcd.Client

		BeforeEach(func() {
			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				w.WriteHeader(200)
				w.Header().Set("Content-Type", "application/json")
				fmt.Fprintln(w, `leader":"6a0b69a54415a491","followers":{"a0294459200078aa":{"latency":{"current":0.001199,"average":0.0023682517720168754,"standardDeviation":0.4302199179552562,"minimum":0.000654,"maximum":1996.564157},"counts":{"fail":16,"success":21538911}},"b5c352b4495e4195":{"latency":{"current":0.001609,"average":0.002361467019756358,"standardDeviation":0.00506414137059054,"minimum":0.00088,"maximum":5.153269},"counts":{"fail":7,"success":1617908}}}}`)
			}))

			transport := &http.Transport{
				Proxy: func(req *http.Request) (*url.URL, error) {
					return url.Parse(server.URL)
				},
				TLSClientConfig: &tls.Config{},
			}
			httpClient := &http.Client{Transport: transport}

			config := &etcd.Config{
				EtcdIP:     "1.1.1.1",
				HTTPClient: httpClient,
			}
			client = etcd.NewClient(config)
		})

		It("returns the error", func() {
			_, _, err := client.GetLeaderStats()
			Ω(err).Should(MatchError("invalid character 'l' looking for beginning of value"))
		})
	})

	Context("when no errors are raised", func() {
		Context("and the etcd is a leader", func() {
			var client *etcd.Client

			BeforeEach(func() {
				server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					w.WriteHeader(200)
					w.Header().Set("Content-Type", "application/json")
					fmt.Fprintln(w, `{"leader":"6a0b69a54415a491","followers":{"a0294459200078aa":{"latency":{"current":0.001199,"average":0.0023682517720168754,"standardDeviation":0.4302199179552562,"minimum":0.000654,"maximum":1996.564157},"counts":{"fail":16,"success":21538911}},"b5c352b4495e4195":{"latency":{"current":0.001609,"average":0.002361467019756358,"standardDeviation":0.00506414137059054,"minimum":0.00088,"maximum":5.153269},"counts":{"fail":7,"success":1617908}}}}`)
				}))

				transport := &http.Transport{
					Proxy: func(req *http.Request) (*url.URL, error) {
						return url.Parse(server.URL)
					},
					TLSClientConfig: &tls.Config{},
				}
				httpClient := &http.Client{Transport: transport}

				config := &etcd.Config{
					EtcdIP:     "1.1.1.1",
					HTTPClient: httpClient,
				}
				client = etcd.NewClient(config)
			})

			It("returns a count of followers and leader true", func() {
				leader, followers, _ := client.GetLeaderStats()
				Ω(leader).Should(BeTrue())
				Ω(followers).Should(Equal(2))
			})
		})

		Context("and the etcd is not a leader", func() {
			var client *etcd.Client

			BeforeEach(func() {
				server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					w.WriteHeader(200)
					w.Header().Set("Content-Type", "application/json")
					fmt.Fprintln(w, `{"message":"not current leader"}`)
				}))

				transport := &http.Transport{
					Proxy: func(req *http.Request) (*url.URL, error) {
						return url.Parse(server.URL)
					},
					TLSClientConfig: &tls.Config{},
				}
				httpClient := &http.Client{Transport: transport}

				config := &etcd.Config{
					EtcdIP:     "1.1.1.1",
					HTTPClient: httpClient,
				}
				client = etcd.NewClient(config)
			})

			It("returns leader false", func() {
				leader, _, _ := client.GetLeaderStats()
				Ω(leader).Should(BeFalse())
			})
		})
	})
})
<|endoftext|>


<commit_before>package ethchain

import (
	"bytes"
	"fmt"
	"math/big"
	"sort"
	_ "strconv"
	"time"

	"github.com/ethereum/eth-go/ethcrypto"
	"github.com/ethereum/eth-go/ethstate"
	"github.com/ethereum/eth-go/ethtrie"
	"github.com/ethereum/eth-go/ethutil"
)

type BlockInfo struct {
	Number uint64
	Hash   []byte
	Parent []byte
	TD     *big.Int
}

func (bi *BlockInfo) RlpDecode(data []byte) {
	decoder := ethutil.NewValueFromBytes(data)

	bi.Number = decoder.Get(0).Uint()
	bi.Hash = decoder.Get(1).Bytes()
	bi.Parent = decoder.Get(2).Bytes()
	bi.TD = decoder.Get(3).BigInt()
}

func (bi *BlockInfo) RlpEncode() []byte {
	return ethutil.Encode([]interface{}{bi.Number, bi.Hash, bi.Parent, bi.TD})
}

type Blocks []*Block

func (self Blocks) AsSet() ethutil.UniqueSet {
	set := make(ethutil.UniqueSet)
	for _, block := range self {
		set.Insert(block.Hash())
	}

	return set
}

type BlockBy func(b1, b2 *Block) bool

func (self BlockBy) Sort(blocks Blocks) {
	bs := blockSorter{
		blocks: blocks,
		by:     self,
	}
	sort.Sort(bs)
}

type blockSorter struct {
	blocks Blocks
	by     func(b1, b2 *Block) bool
}

func (self blockSorter) Len() int { return len(self.blocks) }
func (self blockSorter) Swap(i, j int) {
	self.blocks[i], self.blocks[j] = self.blocks[j], self.blocks[i]
}
func (self blockSorter) Less(i, j int) bool { return self.by(self.blocks[i], self.blocks[j]) }

func Number(b1, b2 *Block) bool { return b1.Number.Cmp(b2.Number) < 0 }

type Block struct {
	// Hash to the previous block
	PrevHash ethutil.Bytes
	// Uncles of this block
	Uncles   Blocks
	UncleSha []byte
	// The coin base address
	Coinbase []byte
	// Block Trie state
	//state *ethutil.Trie
	state *ethstate.State
	// Difficulty for the current block
	Difficulty *big.Int
	// Creation time
	Time int64
	// The block number
	Number *big.Int
	// Minimum Gas Price
	MinGasPrice *big.Int
	// Gas limit
	GasLimit *big.Int
	// Gas used
	GasUsed *big.Int
	// Extra data
	Extra string
	// Block Nonce for verification
	Nonce ethutil.Bytes
	// List of transactions and/or contracts
	transactions []*Transaction
	receipts     []*Receipt
	TxSha        []byte
}

func NewBlockFromBytes(raw []byte) *Block {
	block := &Block{}
	block.RlpDecode(raw)

	return block
}

// New block takes a raw encoded string
func NewBlockFromRlpValue(rlpValue *ethutil.Value) *Block {
	block := &Block{}
	block.RlpValueDecode(rlpValue)

	return block
}

func CreateBlock(root interface{},
	prevHash []byte,
	base []byte,
	Difficulty *big.Int,
	Nonce []byte,
	extra string) *Block {

	block := &Block{
		PrevHash:    prevHash,
		Coinbase:    base,
		Difficulty:  Difficulty,
		Nonce:       Nonce,
		Time:        time.Now().Unix(),
		Extra:       extra,
		UncleSha:    EmptyShaList,
		GasUsed:     new(big.Int),
		MinGasPrice: new(big.Int),
		GasLimit:    new(big.Int),
	}
	block.SetUncles([]*Block{})

	block.state = ethstate.New(ethtrie.New(ethutil.Config.Db, root))

	return block
}

// Returns a hash of the block
func (block *Block) Hash() ethutil.Bytes {
	return ethcrypto.Sha3(ethutil.NewValue(block.header()).Encode())
	//return ethcrypto.Sha3(block.Value().Encode())
}

func (block *Block) HashNoNonce() []byte {
	return ethcrypto.Sha3(ethutil.Encode([]interface{}{block.PrevHash,
		block.UncleSha, block.Coinbase, block.state.Trie.Root,
		block.TxSha, block.Difficulty, block.Number, block.MinGasPrice,
		block.GasLimit, block.GasUsed, block.Time, block.Extra}))
}

func (block *Block) State() *ethstate.State {
	return block.state
}

func (block *Block) Transactions() []*Transaction {
	return block.transactions
}

func (block *Block) CalcGasLimit(parent *Block) *big.Int {
	if block.Number.Cmp(big.NewInt(0)) == 0 {
		return ethutil.BigPow(10, 6)
	}

	// ((1024-1) * parent.gasLimit + (gasUsed * 6 / 5)) / 1024

	previous := new(big.Int).Mul(big.NewInt(1024-1), parent.GasLimit)
	current := new(big.Rat).Mul(new(big.Rat).SetInt(parent.GasUsed), big.NewRat(6, 5))
	curInt := new(big.Int).Div(current.Num(), current.Denom())

	result := new(big.Int).Add(previous, curInt)
	result.Div(result, big.NewInt(1024))

	min := big.NewInt(125000)

	return ethutil.BigMax(min, result)
}

func (block *Block) BlockInfo() BlockInfo {
	bi := BlockInfo{}
	data, _ := ethutil.Config.Db.Get(append(block.Hash(), []byte("Info")...))
	bi.RlpDecode(data)

	return bi
}

func (self *Block) GetTransaction(hash []byte) *Transaction {
	for _, receipt := range self.receipts {
		if bytes.Compare(receipt.Tx.Hash(), hash) == 0 {
			return receipt.Tx
		}
	}

	return nil
}

// Sync the block's state and contract respectively
func (block *Block) Sync() {
	block.state.Sync()
}

func (block *Block) Undo() {
	// Sync the block state itself
	block.state.Reset()
}

/////// Block Encoding
func (block *Block) rlpReceipts() interface{} {
	// Marshal the transactions of this block
	encR := make([]interface{}, len(block.receipts))
	for i, r := range block.receipts {
		// Cast it to a string (safe)
		encR[i] = r.RlpData()
	}

	return encR
}

func (block *Block) rlpUncles() interface{} {
	// Marshal the transactions of this block
	uncles := make([]interface{}, len(block.Uncles))
	for i, uncle := range block.Uncles {
		// Cast it to a string (safe)
		uncles[i] = uncle.header()
	}

	return uncles
}

func (block *Block) SetUncles(uncles []*Block) {
	block.Uncles = uncles

	// Sha of the concatenated uncles
	block.UncleSha = ethcrypto.Sha3(ethutil.Encode(block.rlpUncles()))
}

func (self *Block) SetReceipts(receipts []*Receipt, txs []*Transaction) {
	self.receipts = receipts
	self.setTransactions(txs)
}

func (block *Block) setTransactions(txs []*Transaction) {
	block.transactions = txs
}

func CreateTxSha(receipts Receipts) (sha []byte) {
	trie := ethtrie.New(ethutil.Config.Db, "")
	for i, receipt := range receipts {
		trie.Update(string(ethutil.NewValue(i).Encode()), string(ethutil.NewValue(receipt.RlpData()).Encode()))
	}

	switch trie.Root.(type) {
	case string:
		sha = []byte(trie.Root.(string))
	case []byte:
		sha = trie.Root.([]byte)
	default:
		panic(fmt.Sprintf("invalid root type %T", trie.Root))
	}

	return sha
}

func (self *Block) SetTxHash(receipts Receipts) {
	self.TxSha = CreateTxSha(receipts)
}

func (block *Block) Value() *ethutil.Value {
	return ethutil.NewValue([]interface{}{block.header(), block.rlpReceipts(), block.rlpUncles()})
}

func (block *Block) RlpEncode() []byte {
	// Encode a slice interface which contains the header and the list of
	// transactions.
	return block.Value().Encode()
}

func (block *Block) RlpDecode(data []byte) {
	rlpValue := ethutil.NewValueFromBytes(data)
	block.RlpValueDecode(rlpValue)
}

func (block *Block) RlpValueDecode(decoder *ethutil.Value) {
	header := decoder.Get(0)

	block.PrevHash = header.Get(0).Bytes()
	block.UncleSha = header.Get(1).Bytes()
	block.Coinbase = header.Get(2).Bytes()
	block.state = ethstate.New(ethtrie.New(ethutil.Config.Db, header.Get(3).Val))
	block.TxSha = header.Get(4).Bytes()
	block.Difficulty = header.Get(5).BigInt()
	block.Number = header.Get(6).BigInt()
	//fmt.Printf("#%v : %x\n", block.Number, block.Coinbase)
	block.MinGasPrice = header.Get(7).BigInt()
	block.GasLimit = header.Get(8).BigInt()
	block.GasUsed = header.Get(9).BigInt()
	block.Time = int64(header.Get(10).BigInt().Uint64())
	block.Extra = header.Get(11).Str()
	block.Nonce = header.Get(12).Bytes()

	// Tx list might be empty if this is an uncle. Uncles only have their
	// header set.
	if decoder.Get(1).IsNil() == false { // Yes explicitness
		receipts := decoder.Get(1)
		block.transactions = make([]*Transaction, receipts.Len())
		block.receipts = make([]*Receipt, receipts.Len())
		for i := 0; i < receipts.Len(); i++ {
			receipt := NewRecieptFromValue(receipts.Get(i))
			block.transactions[i] = receipt.Tx
			block.receipts[i] = receipt
		}

	}

	if decoder.Get(2).IsNil() == false { // Yes explicitness
		uncles := decoder.Get(2)
		block.Uncles = make([]*Block, uncles.Len())
		for i := 0; i < uncles.Len(); i++ {
			block.Uncles[i] = NewUncleBlockFromValue(uncles.Get(i))
		}
	}

}

func NewUncleBlockFromValue(header *ethutil.Value) *Block {
	block := &Block{}

	block.PrevHash = header.Get(0).Bytes()
	block.UncleSha = header.Get(1).Bytes()
	block.Coinbase = header.Get(2).Bytes()
	block.state = ethstate.New(ethtrie.New(ethutil.Config.Db, header.Get(3).Val))
	block.TxSha = header.Get(4).Bytes()
	block.Difficulty = header.Get(5).BigInt()
	block.Number = header.Get(6).BigInt()
	block.MinGasPrice = header.Get(7).BigInt()
	block.GasLimit = header.Get(8).BigInt()
	block.GasUsed = header.Get(9).BigInt()
	block.Time = int64(header.Get(10).BigInt().Uint64())
	block.Extra = header.Get(11).Str()
	block.Nonce = header.Get(12).Bytes()

	return block
}

func (block *Block) GetRoot() interface{} {
	return block.state.Trie.Root
}

func (self *Block) Receipts() []*Receipt {
	return self.receipts
}

func (block *Block) header() []interface{} {
	return []interface{}{
		// Sha of the previous block
		block.PrevHash,
		// Sha of uncles
		block.UncleSha,
		// Coinbase address
		block.Coinbase,
		// root state
		block.state.Trie.Root,
		// Sha of tx
		block.TxSha,
		// Current block Difficulty
		block.Difficulty,
		// The block number
		block.Number,
		// Block minimum gas price
		block.MinGasPrice,
		// Block upper gas bound
		block.GasLimit,
		// Block gas used
		block.GasUsed,
		// Time the block was found?
		block.Time,
		// Extra data
		block.Extra,
		// Block's Nonce for validation
		block.Nonce,
	}
}

func (block *Block) String() string {
	return fmt.Sprintf(`
	BLOCK(%x): Size: %v
	PrevHash:   %x
	UncleSha:   %x
	Coinbase:   %x
	Root:       %x
	TxSha:      %x
	Difficulty: %v
	Number:     %v
	MinGas:     %v
	MaxLimit:   %v
	GasUsed:    %v
	Time:       %v
	Extra:      %v
	Nonce:      %x
	NumTx:      %v
`,
		block.Hash(),
		block.Size(),
		block.PrevHash,
		block.UncleSha,
		block.Coinbase,
		block.state.Trie.Root,
		block.TxSha,
		block.Difficulty,
		block.Number,
		block.MinGasPrice,
		block.GasLimit,
		block.GasUsed,
		block.Time,
		block.Extra,
		block.Nonce,
		len(block.transactions),
	)
}

func (self *Block) Size() ethutil.StorageSize {
	return ethutil.StorageSize(len(self.RlpEncode()))
}
<commit_msg>Added some methods to comply to the PoW block interface<commit_after>package ethchain

import (
	"bytes"
	"fmt"
	"math/big"
	"sort"
	_ "strconv"
	"time"

	"github.com/ethereum/eth-go/ethcrypto"
	"github.com/ethereum/eth-go/ethstate"
	"github.com/ethereum/eth-go/ethtrie"
	"github.com/ethereum/eth-go/ethutil"
)

type BlockInfo struct {
	Number uint64
	Hash   []byte
	Parent []byte
	TD     *big.Int
}

func (bi *BlockInfo) RlpDecode(data []byte) {
	decoder := ethutil.NewValueFromBytes(data)

	bi.Number = decoder.Get(0).Uint()
	bi.Hash = decoder.Get(1).Bytes()
	bi.Parent = decoder.Get(2).Bytes()
	bi.TD = decoder.Get(3).BigInt()
}

func (bi *BlockInfo) RlpEncode() []byte {
	return ethutil.Encode([]interface{}{bi.Number, bi.Hash, bi.Parent, bi.TD})
}

type Blocks []*Block

func (self Blocks) AsSet() ethutil.UniqueSet {
	set := make(ethutil.UniqueSet)
	for _, block := range self {
		set.Insert(block.Hash())
	}

	return set
}

type BlockBy func(b1, b2 *Block) bool

func (self BlockBy) Sort(blocks Blocks) {
	bs := blockSorter{
		blocks: blocks,
		by:     self,
	}
	sort.Sort(bs)
}

type blockSorter struct {
	blocks Blocks
	by     func(b1, b2 *Block) bool
}

func (self blockSorter) Len() int { return len(self.blocks) }
func (self blockSorter) Swap(i, j int) {
	self.blocks[i], self.blocks[j] = self.blocks[j], self.blocks[i]
}
func (self blockSorter) Less(i, j int) bool { return self.by(self.blocks[i], self.blocks[j]) }

func Number(b1, b2 *Block) bool { return b1.Number.Cmp(b2.Number) < 0 }

type Block struct {
	// Hash to the previous block
	PrevHash ethutil.Bytes
	// Uncles of this block
	Uncles   Blocks
	UncleSha []byte
	// The coin base address
	Coinbase []byte
	// Block Trie state
	//state *ethutil.Trie
	state *ethstate.State
	// Difficulty for the current block
	Difficulty *big.Int
	// Creation time
	Time int64
	// The block number
	Number *big.Int
	// Minimum Gas Price
	MinGasPrice *big.Int
	// Gas limit
	GasLimit *big.Int
	// Gas used
	GasUsed *big.Int
	// Extra data
	Extra string
	// Block Nonce for verification
	Nonce ethutil.Bytes
	// List of transactions and/or contracts
	transactions []*Transaction
	receipts     []*Receipt
	TxSha        []byte
}

func NewBlockFromBytes(raw []byte) *Block {
	block := &Block{}
	block.RlpDecode(raw)

	return block
}

// New block takes a raw encoded string
func NewBlockFromRlpValue(rlpValue *ethutil.Value) *Block {
	block := &Block{}
	block.RlpValueDecode(rlpValue)

	return block
}

func CreateBlock(root interface{},
	prevHash []byte,
	base []byte,
	Difficulty *big.Int,
	Nonce []byte,
	extra string) *Block {

	block := &Block{
		PrevHash:    prevHash,
		Coinbase:    base,
		Difficulty:  Difficulty,
		Nonce:       Nonce,
		Time:        time.Now().Unix(),
		Extra:       extra,
		UncleSha:    EmptyShaList,
		GasUsed:     new(big.Int),
		MinGasPrice: new(big.Int),
		GasLimit:    new(big.Int),
	}
	block.SetUncles([]*Block{})

	block.state = ethstate.New(ethtrie.New(ethutil.Config.Db, root))

	return block
}

// Returns a hash of the block
func (block *Block) Hash() ethutil.Bytes {
	return ethcrypto.Sha3(ethutil.NewValue(block.header()).Encode())
	//return ethcrypto.Sha3(block.Value().Encode())
}

func (block *Block) HashNoNonce() []byte {
	return ethcrypto.Sha3(ethutil.Encode([]interface{}{block.PrevHash,
		block.UncleSha, block.Coinbase, block.state.Trie.Root,
		block.TxSha, block.Difficulty, block.Number, block.MinGasPrice,
		block.GasLimit, block.GasUsed, block.Time, block.Extra}))
}

func (block *Block) State() *ethstate.State {
	return block.state
}

func (block *Block) Transactions() []*Transaction {
	return block.transactions
}

func (block *Block) CalcGasLimit(parent *Block) *big.Int {
	if block.Number.Cmp(big.NewInt(0)) == 0 {
		return ethutil.BigPow(10, 6)
	}

	// ((1024-1) * parent.gasLimit + (gasUsed * 6 / 5)) / 1024

	previous := new(big.Int).Mul(big.NewInt(1024-1), parent.GasLimit)
	current := new(big.Rat).Mul(new(big.Rat).SetInt(parent.GasUsed), big.NewRat(6, 5))
	curInt := new(big.Int).Div(current.Num(), current.Denom())

	result := new(big.Int).Add(previous, curInt)
	result.Div(result, big.NewInt(1024))

	min := big.NewInt(125000)

	return ethutil.BigMax(min, result)
}

func (block *Block) BlockInfo() BlockInfo {
	bi := BlockInfo{}
	data, _ := ethutil.Config.Db.Get(append(block.Hash(), []byte("Info")...))
	bi.RlpDecode(data)

	return bi
}

func (self *Block) GetTransaction(hash []byte) *Transaction {
	for _, receipt := range self.receipts {
		if bytes.Compare(receipt.Tx.Hash(), hash) == 0 {
			return receipt.Tx
		}
	}

	return nil
}

// Sync the block's state and contract respectively
func (block *Block) Sync() {
	block.state.Sync()
}

func (block *Block) Undo() {
	// Sync the block state itself
	block.state.Reset()
}

/////// Block Encoding
func (block *Block) rlpReceipts() interface{} {
	// Marshal the transactions of this block
	encR := make([]interface{}, len(block.receipts))
	for i, r := range block.receipts {
		// Cast it to a string (safe)
		encR[i] = r.RlpData()
	}

	return encR
}

func (block *Block) rlpUncles() interface{} {
	// Marshal the transactions of this block
	uncles := make([]interface{}, len(block.Uncles))
	for i, uncle := range block.Uncles {
		// Cast it to a string (safe)
		uncles[i] = uncle.header()
	}

	return uncles
}

func (block *Block) SetUncles(uncles []*Block) {
	block.Uncles = uncles

	// Sha of the concatenated uncles
	block.UncleSha = ethcrypto.Sha3(ethutil.Encode(block.rlpUncles()))
}

func (self *Block) SetReceipts(receipts []*Receipt, txs []*Transaction) {
	self.receipts = receipts
	self.setTransactions(txs)
}

func (block *Block) setTransactions(txs []*Transaction) {
	block.transactions = txs
}

func CreateTxSha(receipts Receipts) (sha []byte) {
	trie := ethtrie.New(ethutil.Config.Db, "")
	for i, receipt := range receipts {
		trie.Update(string(ethutil.NewValue(i).Encode()), string(ethutil.NewValue(receipt.RlpData()).Encode()))
	}

	switch trie.Root.(type) {
	case string:
		sha = []byte(trie.Root.(string))
	case []byte:
		sha = trie.Root.([]byte)
	default:
		panic(fmt.Sprintf("invalid root type %T", trie.Root))
	}

	return sha
}

func (self *Block) SetTxHash(receipts Receipts) {
	self.TxSha = CreateTxSha(receipts)
}

func (block *Block) Value() *ethutil.Value {
	return ethutil.NewValue([]interface{}{block.header(), block.rlpReceipts(), block.rlpUncles()})
}

func (block *Block) RlpEncode() []byte {
	// Encode a slice interface which contains the header and the list of
	// transactions.
	return block.Value().Encode()
}

func (block *Block) RlpDecode(data []byte) {
	rlpValue := ethutil.NewValueFromBytes(data)
	block.RlpValueDecode(rlpValue)
}

func (block *Block) RlpValueDecode(decoder *ethutil.Value) {
	header := decoder.Get(0)

	block.PrevHash = header.Get(0).Bytes()
	block.UncleSha = header.Get(1).Bytes()
	block.Coinbase = header.Get(2).Bytes()
	block.state = ethstate.New(ethtrie.New(ethutil.Config.Db, header.Get(3).Val))
	block.TxSha = header.Get(4).Bytes()
	block.Difficulty = header.Get(5).BigInt()
	block.Number = header.Get(6).BigInt()
	//fmt.Printf("#%v : %x\n", block.Number, block.Coinbase)
	block.MinGasPrice = header.Get(7).BigInt()
	block.GasLimit = header.Get(8).BigInt()
	block.GasUsed = header.Get(9).BigInt()
	block.Time = int64(header.Get(10).BigInt().Uint64())
	block.Extra = header.Get(11).Str()
	block.Nonce = header.Get(12).Bytes()

	// Tx list might be empty if this is an uncle. Uncles only have their
	// header set.
	if decoder.Get(1).IsNil() == false { // Yes explicitness
		receipts := decoder.Get(1)
		block.transactions = make([]*Transaction, receipts.Len())
		block.receipts = make([]*Receipt, receipts.Len())
		for i := 0; i < receipts.Len(); i++ {
			receipt := NewRecieptFromValue(receipts.Get(i))
			block.transactions[i] = receipt.Tx
			block.receipts[i] = receipt
		}

	}

	if decoder.Get(2).IsNil() == false { // Yes explicitness
		uncles := decoder.Get(2)
		block.Uncles = make([]*Block, uncles.Len())
		for i := 0; i < uncles.Len(); i++ {
			block.Uncles[i] = NewUncleBlockFromValue(uncles.Get(i))
		}
	}

}

func NewUncleBlockFromValue(header *ethutil.Value) *Block {
	block := &Block{}

	block.PrevHash = header.Get(0).Bytes()
	block.UncleSha = header.Get(1).Bytes()
	block.Coinbase = header.Get(2).Bytes()
	block.state = ethstate.New(ethtrie.New(ethutil.Config.Db, header.Get(3).Val))
	block.TxSha = header.Get(4).Bytes()
	block.Difficulty = header.Get(5).BigInt()
	block.Number = header.Get(6).BigInt()
	block.MinGasPrice = header.Get(7).BigInt()
	block.GasLimit = header.Get(8).BigInt()
	block.GasUsed = header.Get(9).BigInt()
	block.Time = int64(header.Get(10).BigInt().Uint64())
	block.Extra = header.Get(11).Str()
	block.Nonce = header.Get(12).Bytes()

	return block
}

func (block *Block) Trie() *ethrie.Trie {
	return block.state.Trie
}

func (block *Block) GetRoot() interface{} {
	return block.state.Trie.Root
}

func (block *Block) Diff() *big.Int {
	return block.Difficulty
}

func (self *Block) Receipts() []*Receipt {
	return self.receipts
}

func (block *Block) header() []interface{} {
	return []interface{}{
		// Sha of the previous block
		block.PrevHash,
		// Sha of uncles
		block.UncleSha,
		// Coinbase address
		block.Coinbase,
		// root state
		block.state.Trie.Root,
		// Sha of tx
		block.TxSha,
		// Current block Difficulty
		block.Difficulty,
		// The block number
		block.Number,
		// Block minimum gas price
		block.MinGasPrice,
		// Block upper gas bound
		block.GasLimit,
		// Block gas used
		block.GasUsed,
		// Time the block was found?
		block.Time,
		// Extra data
		block.Extra,
		// Block's Nonce for validation
		block.Nonce,
	}
}

func (block *Block) String() string {
	return fmt.Sprintf(`
	BLOCK(%x): Size: %v
	PrevHash:   %x
	UncleSha:   %x
	Coinbase:   %x
	Root:       %x
	TxSha:      %x
	Difficulty: %v
	Number:     %v
	MinGas:     %v
	MaxLimit:   %v
	GasUsed:    %v
	Time:       %v
	Extra:      %v
	Nonce:      %x
	NumTx:      %v
`,
		block.Hash(),
		block.Size(),
		block.PrevHash,
		block.UncleSha,
		block.Coinbase,
		block.state.Trie.Root,
		block.TxSha,
		block.Difficulty,
		block.Number,
		block.MinGasPrice,
		block.GasLimit,
		block.GasUsed,
		block.Time,
		block.Extra,
		block.Nonce,
		len(block.transactions),
	)
}

func (self *Block) Size() ethutil.StorageSize {
	return ethutil.StorageSize(len(self.RlpEncode()))
}
<|endoftext|>


<commit_before>package ethutil

import (
	"fmt"
	"log"
	"os"
	"os/user"
	"path"
	"runtime"
)

// Config struct
type config struct {
	Db Database

	Log          *Logger
	ExecPath     string
	Debug        bool
	Ver          string
	ClientString string
	Pubkey       []byte
	Identifier   string
}

const defaultConf = `
id = ""
port = 30303
upnp = true
maxpeer = 10
rpc = false
rpcport = 8080
`

var Config *config

func ApplicationFolder(base string) string {
	usr, _ := user.Current()
	p := path.Join(usr.HomeDir, base)

	if len(base) > 0 {
		//Check if the logging directory already exists, create it if not
		_, err := os.Stat(p)
		if err != nil {
			if os.IsNotExist(err) {
				log.Printf("Debug logging directory %s doesn't exist, creating it\n", p)
				os.Mkdir(p, 0777)

			}
		}

		iniFilePath := path.Join(p, "conf.ini")
		_, err = os.Stat(iniFilePath)
		if err != nil && os.IsNotExist(err) {
			file, err := os.Create(iniFilePath)
			if err != nil {
				fmt.Println(err)
			} else {
				assetPath := path.Join(os.Getenv("GOPATH"), "src", "github.com", "ethereum", "go-ethereum", "ethereal", "assets")
				file.Write([]byte(defaultConf + "\nasset_path = " + assetPath))
			}
		}
	}

	return p
}

// Read config
//
// Initialize the global Config variable with default settings
func ReadConfig(base string, logTypes LoggerType, id string) *config {
	if Config == nil {
		path := ApplicationFolder(base)

		Config = &config{ExecPath: path, Debug: true, Ver: "0.5.0 RC11"}
		Config.Identifier = id
		Config.Log = NewLogger(logTypes, LogLevelDebug)
		Config.SetClientString("/Ethereum(G)")
	}

	return Config
}

// Set client string
//
func (c *config) SetClientString(str string) {
	id := runtime.GOOS
	if len(c.Identifier) > 0 {
		id = c.Identifier
	}
	Config.ClientString = fmt.Sprintf("%s nv%s/%s", str, c.Ver, id)
}

type LoggerType byte

const (
	LogFile = 0x1
	LogStd  = 0x2
)

type LogSystem interface {
	Println(v ...interface{})
	Printf(format string, v ...interface{})
}

type Logger struct {
	logSys   []LogSystem
	logLevel int
}

func NewLogger(flag LoggerType, level int) *Logger {
	var loggers []LogSystem

	flags := log.LstdFlags

	if flag&LogFile > 0 {
		file, err := os.OpenFile(path.Join(Config.ExecPath, "debug.log"), os.O_RDWR|os.O_CREATE|os.O_APPEND, os.ModePerm)
		if err != nil {
			log.Panic("unable to create file logger", err)
		}

		log := log.New(file, "", flags)

		loggers = append(loggers, log)
	}
	if flag&LogStd > 0 {
		log := log.New(os.Stdout, "", flags)
		loggers = append(loggers, log)
	}

	return &Logger{logSys: loggers, logLevel: level}
}

func (log *Logger) AddLogSystem(logger LogSystem) {
	log.logSys = append(log.logSys, logger)
}

const (
	LogLevelDebug = iota
	LogLevelInfo
)

func (log *Logger) Debugln(v ...interface{}) {
	if log.logLevel != LogLevelDebug {
		return
	}

	for _, logger := range log.logSys {
		logger.Println(v...)
	}
}

func (log *Logger) Debugf(format string, v ...interface{}) {
	if log.logLevel != LogLevelDebug {
		return
	}

	for _, logger := range log.logSys {
		logger.Printf(format, v...)
	}
}

func (log *Logger) Infoln(v ...interface{}) {
	if log.logLevel > LogLevelInfo {
		return
	}

	for _, logger := range log.logSys {
		logger.Println(v...)
	}
}

func (log *Logger) Infof(format string, v ...interface{}) {
	if log.logLevel > LogLevelInfo {
		return
	}

	for _, logger := range log.logSys {
		logger.Printf(format, v...)
	}
}

func (log *Logger) Fatal(v ...interface{}) {
	if log.logLevel > LogLevelInfo {
		return
	}

	for _, logger := range log.logSys {
		logger.Println(v...)
	}

	os.Exit(1)
}
<commit_msg>woops<commit_after>package ethutil

import (
	"flag"
	"fmt"
	"github.com/rakyll/globalconf"
	"log"
	"os"
	"os/user"
	"path"
	"runtime"
)

// Config struct
type config struct {
	Db Database

	Log          *Logger
	ExecPath     string
	Debug        bool
	Ver          string
	ClientString string
	Pubkey       []byte
	Identifier   string

	conf *globalconf.GlobalConf
}

const defaultConf = `
id = ""
port = 30303
upnp = true
maxpeer = 10
rpc = false
rpcport = 8080
`

var Config *config

func ApplicationFolder(base string) string {
	usr, _ := user.Current()
	p := path.Join(usr.HomeDir, base)

	if len(base) > 0 {
		//Check if the logging directory already exists, create it if not
		_, err := os.Stat(p)
		if err != nil {
			if os.IsNotExist(err) {
				log.Printf("Debug logging directory %s doesn't exist, creating it\n", p)
				os.Mkdir(p, 0777)

			}
		}

		iniFilePath := path.Join(p, "conf.ini")
		_, err = os.Stat(iniFilePath)
		if err != nil && os.IsNotExist(err) {
			file, err := os.Create(iniFilePath)
			if err != nil {
				fmt.Println(err)
			} else {
				assetPath := path.Join(os.Getenv("GOPATH"), "src", "github.com", "ethereum", "go-ethereum", "ethereal", "assets")
				file.Write([]byte(defaultConf + "\nasset_path = " + assetPath))
			}
		}
	}

	return p
}

// Read config
//
// Initialize the global Config variable with default settings
func ReadConfig(base string, logTypes LoggerType, g *globalconf.GlobalConf, id string) *config {
	if Config == nil {
		path := ApplicationFolder(base)

		Config = &config{ExecPath: path, Debug: true, Ver: "0.5.0 RC11"}
		Config.conf = g
		Config.Identifier = id
		Config.Log = NewLogger(logTypes, LogLevelDebug)
		Config.SetClientString("/Ethereum(G)")
	}

	return Config
}

// Set client string
//
func (c *config) SetClientString(str string) {
	id := runtime.GOOS
	if len(c.Identifier) > 0 {
		id = c.Identifier
	}
	Config.ClientString = fmt.Sprintf("%s nv%s/%s", str, c.Ver, id)
}

func (c *config) SetIdentifier(id string) {
	c.Identifier = id
	c.Set("id", id)
}

func (c *config) Set(key, value string) {
	f := &flag.Flag{Name: key, Value: &confValue{value}}
	c.conf.Set("", f)
}

type LoggerType byte

const (
	LogFile = 0x1
	LogStd  = 0x2
)

type LogSystem interface {
	Println(v ...interface{})
	Printf(format string, v ...interface{})
}

type Logger struct {
	logSys   []LogSystem
	logLevel int
}

func NewLogger(flag LoggerType, level int) *Logger {
	var loggers []LogSystem

	flags := log.LstdFlags

	if flag&LogFile > 0 {
		file, err := os.OpenFile(path.Join(Config.ExecPath, "debug.log"), os.O_RDWR|os.O_CREATE|os.O_APPEND, os.ModePerm)
		if err != nil {
			log.Panic("unable to create file logger", err)
		}

		log := log.New(file, "", flags)

		loggers = append(loggers, log)
	}
	if flag&LogStd > 0 {
		log := log.New(os.Stdout, "", flags)
		loggers = append(loggers, log)
	}

	return &Logger{logSys: loggers, logLevel: level}
}

func (log *Logger) AddLogSystem(logger LogSystem) {
	log.logSys = append(log.logSys, logger)
}

const (
	LogLevelDebug = iota
	LogLevelInfo
)

func (log *Logger) Debugln(v ...interface{}) {
	if log.logLevel != LogLevelDebug {
		return
	}

	for _, logger := range log.logSys {
		logger.Println(v...)
	}
}

func (log *Logger) Debugf(format string, v ...interface{}) {
	if log.logLevel != LogLevelDebug {
		return
	}

	for _, logger := range log.logSys {
		logger.Printf(format, v...)
	}
}

func (log *Logger) Infoln(v ...interface{}) {
	if log.logLevel > LogLevelInfo {
		return
	}

	for _, logger := range log.logSys {
		logger.Println(v...)
	}
}

func (log *Logger) Infof(format string, v ...interface{}) {
	if log.logLevel > LogLevelInfo {
		return
	}

	for _, logger := range log.logSys {
		logger.Printf(format, v...)
	}
}

func (log *Logger) Fatal(v ...interface{}) {
	if log.logLevel > LogLevelInfo {
		return
	}

	for _, logger := range log.logSys {
		logger.Println(v...)
	}

	os.Exit(1)
}

type confValue struct {
	value string
}

func (self confValue) String() string     { return self.value }
func (self confValue) Set(s string) error { self.value = s; return nil }
<|endoftext|>


<commit_before>// Copyright (c) 2014-2015 by Michael Dvorkin. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package donna

type PawnEntry struct {
	id       uint64 	// Pawn hash key.
	score    Score 		// Static score for the given pawn structure.
	king     [2]uint8 	// King square for both sides.
	cover    [2]Score 	// King cover penalties for both sides.
	passers  [2]Bitmask 	// Passed pawn bitmasks for both sides.
}

type PawnCache [8192*2]PawnEntry

func (e *Evaluation) analyzePawns() {
	key := e.position.pawnId

	// Since pawn hash is fairly small we can use much faster 32-bit index.
	index := uint32(key) % uint32(len(game.pawnCache))
	e.pawns = &game.pawnCache[index]

	// Bypass pawns cache if evaluation tracing is enabled.
	if e.pawns.id != key || engine.trace {
		white, black := e.pawnStructure(White), e.pawnStructure(Black)
		e.pawns.score.clear().add(white).sub(black).apply(weightPawnStructure)
		e.pawns.id = key

		// Force full king shelter evaluation since any legit king square
		// will be viewed as if the king has moved.
		e.pawns.king[White], e.pawns.king[Black] = 0xFF, 0xFF

		if engine.trace {
			e.checkpoint(`Pawns`, Total{white, black})
		}
	}

	e.score.add(e.pawns.score)
}

func (e *Evaluation) analyzePassers() {
	var white, black, score Score

	if engine.trace {
		defer func() {
			e.checkpoint(`Passers`, Total{white, black})
		}()
	}

	white, black = e.pawnPassers(White), e.pawnPassers(Black)
	score.add(white).sub(black).apply(weightPassedPawns)
	e.score.add(score)
}

// Calculates extra bonus and penalty based on pawn structure. Specifically,
// a bonus is awarded for passed pawns, and penalty applied for isolated and
// doubled pawns.
func (e *Evaluation) pawnStructure(color uint8) (score Score) {
	rival := color ^ 1
	hisPawns := e.position.outposts[pawn(color)]
	herPawns := e.position.outposts[pawn(rival)]
	e.pawns.passers[color] = 0

	// Encourage center pawn moves in the opening.
	pawns := hisPawns

	for pawns.any() {
		square := pawns.pop()
		row, col := coordinate(square)

		isolated := (maskIsolated[col] & hisPawns).empty()
		exposed := (maskInFront[color][square] & herPawns).empty()
		doubled := (maskInFront[color][square] & hisPawns).any()
		supported := (maskIsolated[col] & (maskRank[row] | maskRank[row].up(rival)) & hisPawns).any()

		// The pawn is passed if a) there are no enemy pawns in the same
		// and adjacent columns; and b) there are no same color pawns in
		// front of us.
		passed := !doubled && (maskPassed[color][square] & herPawns).empty()
		if passed {
			e.pawns.passers[color] |= bit[square]
		}

		// Penalty if the pawn is isolated, i.e. has no friendly pawns
		// on adjacent files. The penalty goes up if isolated pawn is
		// exposed on semi-open file.
		if isolated {
			if !exposed {
				score.sub(penaltyIsolatedPawn[col])
			} else {
				score.sub(penaltyWeakIsolatedPawn[col])
			}
		} else if !supported {
			// Small penalty if the pawn is not supported by a fiendly pawn.
			score.sub(Score{10, 5})
		}

		// Penalty if the pawn is doubled, i.e. there is another friendly
		// pawn in front of us.
		if doubled {
			score.sub(penaltyDoubledPawn[col])
		}

		// Penalty if the pawn is backward.
		backward := false
		if (!passed && !supported && !isolated) {

			// Backward pawn should not be attacking enemy pawns.
			if (pawnAttacks[color][square] & herPawns).empty() {

				// Backward pawn should not have friendly pawns behind.
				if (maskPassed[rival][square] & maskIsolated[col] & hisPawns).empty() {

					// Backward pawn should face enemy pawns on the next two ranks
					// preventing its advance.
					enemy := pawnAttacks[color][square].up(color)
					if ((enemy | enemy.up(color)) & herPawns).any() {
						backward = true
						if !exposed {
							score.sub(penaltyBackwardPawn[col])
						} else {
							score.sub(penaltyWeakBackwardPawn[col])
						}
					}
				}
			}
		}

		// Bonus if the pawn has good chance to become a passed pawn.
		if exposed && !isolated && !passed && !backward {
			his := maskPassed[rival][square + up[color]] & maskIsolated[col] & hisPawns
			her := maskPassed[color][square] & maskIsolated[col] & herPawns
			if his.count() >= her.count() {
				score.add(bonusSemiPassedPawn[rank(color, square)])
			}
		}

		//\\ Encourage center pawn moves.
		//\\ if maskCenter.on(square) {
		//\\ 	score.midgame += bonusPawn[0][flip(color, square)] / 2
		//\\ }
	}

	return
}

func (e *Evaluation) pawnPassers(color uint8) (score Score) {
	p := e.position
	rival := color ^ 1

	// If opposing side has no pieces other than pawns then need to check if passers are unstoppable.
	chase := (p.outposts[rival] ^ p.outposts[pawn(rival)] ^ p.outposts[king(rival)]).empty()

	pawns := e.pawns.passers[color]
	for pawns.any() {
		square := pawns.pop()
		rank := rank(color, square)
		bonus := bonusPassedPawn[rank]

		if rank > A2H2 {
			extra := extraPassedPawn[rank]
			nextSquare := square + up[color]

			// Adjust endgame bonus based on how close the kings are from the
			// step forward square.
			bonus.endgame += (distance[p.king[rival]][nextSquare] * 5 - distance[p.king[color]][nextSquare] * 2) * extra

			// Check if the pawn can step forward.
			if p.board.off(nextSquare) {
				boost := 0

				// Assume all squares in front of the pawn are under attack.
				attacked := maskInFront[color][square]
				protected := attacked & e.attacks[color]

				// Boost the bonus if squares in front of the pawn are protected.
				if protected == attacked {
					boost += 6 // All squares.
				} else if protected.on(nextSquare) {
					boost += 4 // Next square only.
				}

				// Check who is attacking the squares in front of the pawn including
				// queen and rook x-ray attacks from behind.
				enemy := maskInFront[rival][square] & (p.outposts[queen(rival)] | p.outposts[rook(rival)])
				if enemy == 0 || enemy & p.rookMoves(square) == 0 {

					// Since nobody attacks the pawn from behind adjust the attacked
					// bitmask to only include squares attacked or occupied by the enemy.
					attacked &= (e.attacks[rival] | p.outposts[rival])
				}

				// Boost the bonus if passed pawn is free to advance to the 8th rank
				// or at least safely step forward.
				if attacked == 0 {
					boost += 15 // Remaining squares are not under attack.
				} else if attacked.off(nextSquare) {
					boost += 9  // Next square is not under attack.
				}

				if boost > 0 {
					bonus.adjust(extra * boost)
				}
			}
		}

		// Before chasing the unstoppable make sure own pieces are not blocking the passer.
		if chase && (p.outposts[color] & maskInFront[color][square]).empty() {
			// Pick square rule bitmask for the pawn. If defending king has the right
			// to move then pick extended square mask.
			mask := Bitmask(0)
			if p.color == color {
				mask = maskSquare[color][square]
			} else {
				mask = maskSquareEx[color][square]
			}
			if (mask & p.outposts[king(rival)]).empty() {
				bonus.endgame += unstoppablePawn
			}
		}

		score.add(bonus)
	}

	return
}

<commit_msg>Remove commented out code<commit_after>// Copyright (c) 2014-2015 by Michael Dvorkin. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package donna

type PawnEntry struct {
	id       uint64 	// Pawn hash key.
	score    Score 		// Static score for the given pawn structure.
	king     [2]uint8 	// King square for both sides.
	cover    [2]Score 	// King cover penalties for both sides.
	passers  [2]Bitmask 	// Passed pawn bitmasks for both sides.
}

type PawnCache [8192*2]PawnEntry

func (e *Evaluation) analyzePawns() {
	key := e.position.pawnId

	// Since pawn hash is fairly small we can use much faster 32-bit index.
	index := uint32(key) % uint32(len(game.pawnCache))
	e.pawns = &game.pawnCache[index]

	// Bypass pawns cache if evaluation tracing is enabled.
	if e.pawns.id != key || engine.trace {
		white, black := e.pawnStructure(White), e.pawnStructure(Black)
		e.pawns.score.clear().add(white).sub(black).apply(weightPawnStructure)
		e.pawns.id = key

		// Force full king shelter evaluation since any legit king square
		// will be viewed as if the king has moved.
		e.pawns.king[White], e.pawns.king[Black] = 0xFF, 0xFF

		if engine.trace {
			e.checkpoint(`Pawns`, Total{white, black})
		}
	}

	e.score.add(e.pawns.score)
}

func (e *Evaluation) analyzePassers() {
	var white, black, score Score

	if engine.trace {
		defer func() {
			e.checkpoint(`Passers`, Total{white, black})
		}()
	}

	white, black = e.pawnPassers(White), e.pawnPassers(Black)
	score.add(white).sub(black).apply(weightPassedPawns)
	e.score.add(score)
}

// Calculates extra bonus and penalty based on pawn structure. Specifically,
// a bonus is awarded for passed pawns, and penalty applied for isolated and
// doubled pawns.
func (e *Evaluation) pawnStructure(color uint8) (score Score) {
	rival := color ^ 1
	hisPawns := e.position.outposts[pawn(color)]
	herPawns := e.position.outposts[pawn(rival)]
	e.pawns.passers[color] = 0

	// Encourage center pawn moves in the opening.
	pawns := hisPawns

	for pawns.any() {
		square := pawns.pop()
		row, col := coordinate(square)

		isolated := (maskIsolated[col] & hisPawns).empty()
		exposed := (maskInFront[color][square] & herPawns).empty()
		doubled := (maskInFront[color][square] & hisPawns).any()
		supported := (maskIsolated[col] & (maskRank[row] | maskRank[row].up(rival)) & hisPawns).any()

		// The pawn is passed if a) there are no enemy pawns in the same
		// and adjacent columns; and b) there are no same color pawns in
		// front of us.
		passed := !doubled && (maskPassed[color][square] & herPawns).empty()
		if passed {
			e.pawns.passers[color] |= bit[square]
		}

		// Penalty if the pawn is isolated, i.e. has no friendly pawns
		// on adjacent files. The penalty goes up if isolated pawn is
		// exposed on semi-open file.
		if isolated {
			if !exposed {
				score.sub(penaltyIsolatedPawn[col])
			} else {
				score.sub(penaltyWeakIsolatedPawn[col])
			}
		} else if !supported {
			// Small penalty if the pawn is not supported by a fiendly pawn.
			score.sub(Score{10, 5})
		}

		// Penalty if the pawn is doubled, i.e. there is another friendly
		// pawn in front of us.
		if doubled {
			score.sub(penaltyDoubledPawn[col])
		}

		// Penalty if the pawn is backward.
		backward := false
		if (!passed && !supported && !isolated) {

			// Backward pawn should not be attacking enemy pawns.
			if (pawnAttacks[color][square] & herPawns).empty() {

				// Backward pawn should not have friendly pawns behind.
				if (maskPassed[rival][square] & maskIsolated[col] & hisPawns).empty() {

					// Backward pawn should face enemy pawns on the next two ranks
					// preventing its advance.
					enemy := pawnAttacks[color][square].up(color)
					if ((enemy | enemy.up(color)) & herPawns).any() {
						backward = true
						if !exposed {
							score.sub(penaltyBackwardPawn[col])
						} else {
							score.sub(penaltyWeakBackwardPawn[col])
						}
					}
				}
			}
		}

		// Bonus if the pawn has good chance to become a passed pawn.
		if exposed && !isolated && !passed && !backward {
			his := maskPassed[rival][square + up[color]] & maskIsolated[col] & hisPawns
			her := maskPassed[color][square] & maskIsolated[col] & herPawns
			if his.count() >= her.count() {
				score.add(bonusSemiPassedPawn[rank(color, square)])
			}
		}
	}

	return
}

func (e *Evaluation) pawnPassers(color uint8) (score Score) {
	p := e.position
	rival := color ^ 1

	// If opposing side has no pieces other than pawns then need to check if passers are unstoppable.
	chase := (p.outposts[rival] ^ p.outposts[pawn(rival)] ^ p.outposts[king(rival)]).empty()

	pawns := e.pawns.passers[color]
	for pawns.any() {
		square := pawns.pop()
		rank := rank(color, square)
		bonus := bonusPassedPawn[rank]

		if rank > A2H2 {
			extra := extraPassedPawn[rank]
			nextSquare := square + up[color]

			// Adjust endgame bonus based on how close the kings are from the
			// step forward square.
			bonus.endgame += (distance[p.king[rival]][nextSquare] * 5 - distance[p.king[color]][nextSquare] * 2) * extra

			// Check if the pawn can step forward.
			if p.board.off(nextSquare) {
				boost := 0

				// Assume all squares in front of the pawn are under attack.
				attacked := maskInFront[color][square]
				protected := attacked & e.attacks[color]

				// Boost the bonus if squares in front of the pawn are protected.
				if protected == attacked {
					boost += 6 // All squares.
				} else if protected.on(nextSquare) {
					boost += 4 // Next square only.
				}

				// Check who is attacking the squares in front of the pawn including
				// queen and rook x-ray attacks from behind.
				enemy := maskInFront[rival][square] & (p.outposts[queen(rival)] | p.outposts[rook(rival)])
				if enemy == 0 || enemy & p.rookMoves(square) == 0 {

					// Since nobody attacks the pawn from behind adjust the attacked
					// bitmask to only include squares attacked or occupied by the enemy.
					attacked &= (e.attacks[rival] | p.outposts[rival])
				}

				// Boost the bonus if passed pawn is free to advance to the 8th rank
				// or at least safely step forward.
				if attacked == 0 {
					boost += 15 // Remaining squares are not under attack.
				} else if attacked.off(nextSquare) {
					boost += 9  // Next square is not under attack.
				}

				if boost > 0 {
					bonus.adjust(extra * boost)
				}
			}
		}

		// Before chasing the unstoppable make sure own pieces are not blocking the passer.
		if chase && (p.outposts[color] & maskInFront[color][square]).empty() {
			// Pick square rule bitmask for the pawn. If defending king has the right
			// to move then pick extended square mask.
			mask := Bitmask(0)
			if p.color == color {
				mask = maskSquare[color][square]
			} else {
				mask = maskSquareEx[color][square]
			}
			if (mask & p.outposts[king(rival)]).empty() {
				bonus.endgame += unstoppablePawn
			}
		}

		score.add(bonus)
	}

	return
}

<|endoftext|>


<commit_before>package main

// This short script is used for comparison of the golang-package results
// with those of the Perl module Geo::Ellipsoid.
// It is obsolete and superseded by the builtin tests and only
// here for reference.

import "github.com/StefanSchroeder/Golang-Ellipsoid/ellipsoid"
import "fmt"

func main() {
	lat1, lon1 := 37.619002, -122.374843
	lon2, lat2 := 33.942536, -118.408074
	// To
	{
		geo1 := ellipsoid.Init("WGS84", ellipsoid.Degrees, ellipsoid.Meter, ellipsoid.Longitude_is_symmetric, ellipsoid.Bearing_is_symmetric)
		dist, bear := geo1.To(lat1, lon1, lon2, lat2)
		fmt.Printf("1 dist = %v bear = %v\n", dist, bear)
	}
	{
		geo1 := ellipsoid.Init("AIRY", ellipsoid.Degrees, ellipsoid.Meter, ellipsoid.Longitude_is_symmetric, ellipsoid.Bearing_is_symmetric)
		dist, bear := geo1.To(lat1, lon1, lon2, lat2)
		fmt.Printf("2 dist = %v bear = %v\n", dist, bear)
	}
	{
		geo1 := ellipsoid.Init("AIRY", ellipsoid.Radians, ellipsoid.Meter, ellipsoid.Longitude_is_symmetric, ellipsoid.Bearing_not_symmetric)
		dist, bear := geo1.To(lat1, lon1, lon2, lat2)
		fmt.Printf("3 dist = %v bear = %v\n", dist, bear)
	}
	{
		geo1 := ellipsoid.Init("AIRY", ellipsoid.Radians, ellipsoid.Meter, ellipsoid.Longitude_is_symmetric, ellipsoid.Bearing_is_symmetric)
		dist, bear := geo1.To(lat1, lon1, lon2, lat2)
		fmt.Printf("4 dist = %v bear = %v\n", dist, bear)
	}
	{
		geo1 := ellipsoid.Init("AIRY", ellipsoid.Degrees, ellipsoid.Kilometer, ellipsoid.Longitude_is_symmetric, ellipsoid.Bearing_is_symmetric)
		dist, bear := geo1.To(lat1, lon1, lon2, lat2)
		fmt.Printf("5 dist = %v bear = %v\n", dist, bear)
	}
	{
		geo1 := ellipsoid.Init("AIRY", ellipsoid.Degrees, ellipsoid.Foot, ellipsoid.Longitude_is_symmetric, ellipsoid.Bearing_is_symmetric)
		dist, bear := geo1.To(lat1, lon1, lon2, lat2)
		fmt.Printf("6 dist = %v bear = %v\n", dist, bear)
	}
	// At
	{
		geo1 := ellipsoid.Init("AIRY", ellipsoid.Degrees, ellipsoid.Foot, ellipsoid.Longitude_is_symmetric, ellipsoid.Bearing_is_symmetric)
		lat2, lon2 := geo1.At(lat1, lon1, 2000.0, 45.0)
		fmt.Printf("7 lat  = %v lon = %v\n", lat2, lon2)
	}
	{
		geo1 := ellipsoid.Init("AIRY", ellipsoid.Degrees, ellipsoid.Meter, ellipsoid.Longitude_is_symmetric, ellipsoid.Bearing_is_symmetric)
		lat2, lon2 := geo1.At(lat1, lon1, 2000.0, 45.0)
		fmt.Printf("8 lat  = %v lon = %v\n", lat2, lon2)
	}
	{
		geo1 := ellipsoid.Init("AIRY", ellipsoid.Degrees, ellipsoid.Meter, ellipsoid.Longitude_is_symmetric, ellipsoid.Bearing_is_symmetric)
		lat2, lon2 := geo1.At(90.0, 90.0, 1000.0, 90.0)
		fmt.Printf("9 lat  = %v lon = %v\n", lat2, lon2)
	}
	// To
	{
		geo1 := ellipsoid.Init("WGS84", ellipsoid.Degrees, ellipsoid.Nm, ellipsoid.Longitude_is_symmetric, ellipsoid.Bearing_is_symmetric)
		lat3, lon3 := 73.06, 19.11 // Mumbai
		lat4, lon4 := 4.89, 52.37  // Amsterdam
		dist, bear := geo1.To(lat3, lon3, lat4, lon4)
		fmt.Printf("10 dist  = %v bear = %v\n", dist, bear)
	}
}
<commit_msg>Update ellipsoid_perlcompare.go<commit_after>package main

// This short script is used for comparison of the golang-package results
// with those of the Perl module Geo::Ellipsoid.
// It is obsolete and superseded by the builtin tests and only
// here for reference.
import "github.com/StefanSchroeder/Golang-Ellipsoid/ellipsoid"
import "fmt"

func main() {
	lat1, lon1 := 37.619002, -122.374843
	lon2, lat2 := 33.942536, -118.408074
	// To
	{
		geo1 := ellipsoid.Init("WGS84", ellipsoid.Degrees, ellipsoid.Meter, ellipsoid.Longitude_is_symmetric, ellipsoid.Bearing_is_symmetric)
		dist, bear := geo1.To(lat1, lon1, lon2, lat2)
		fmt.Printf("1 dist = %v bear = %v\n", dist, bear)
	}
	{
		geo1 := ellipsoid.Init("AIRY", ellipsoid.Degrees, ellipsoid.Meter, ellipsoid.Longitude_is_symmetric, ellipsoid.Bearing_is_symmetric)
		dist, bear := geo1.To(lat1, lon1, lon2, lat2)
		fmt.Printf("2 dist = %v bear = %v\n", dist, bear)
	}
	{
		geo1 := ellipsoid.Init("AIRY", ellipsoid.Radians, ellipsoid.Meter, ellipsoid.Longitude_is_symmetric, ellipsoid.Bearing_not_symmetric)
		dist, bear := geo1.To(lat1, lon1, lon2, lat2)
		fmt.Printf("3 dist = %v bear = %v\n", dist, bear)
	}
	{
		geo1 := ellipsoid.Init("AIRY", ellipsoid.Radians, ellipsoid.Meter, ellipsoid.Longitude_is_symmetric, ellipsoid.Bearing_is_symmetric)
		dist, bear := geo1.To(lat1, lon1, lon2, lat2)
		fmt.Printf("4 dist = %v bear = %v\n", dist, bear)
	}
	{
		geo1 := ellipsoid.Init("AIRY", ellipsoid.Degrees, ellipsoid.Kilometer, ellipsoid.Longitude_is_symmetric, ellipsoid.Bearing_is_symmetric)
		dist, bear := geo1.To(lat1, lon1, lon2, lat2)
		fmt.Printf("5 dist = %v bear = %v\n", dist, bear)
	}
	{
		geo1 := ellipsoid.Init("AIRY", ellipsoid.Degrees, ellipsoid.Foot, ellipsoid.Longitude_is_symmetric, ellipsoid.Bearing_is_symmetric)
		dist, bear := geo1.To(lat1, lon1, lon2, lat2)
		fmt.Printf("6 dist = %v bear = %v\n", dist, bear)
	}
	// At
	{
		geo1 := ellipsoid.Init("AIRY", ellipsoid.Degrees, ellipsoid.Foot, ellipsoid.Longitude_is_symmetric, ellipsoid.Bearing_is_symmetric)
		lat2, lon2 := geo1.At(lat1, lon1, 2000.0, 45.0)
		fmt.Printf("7 lat  = %v lon = %v\n", lat2, lon2)
	}
	{
		geo1 := ellipsoid.Init("AIRY", ellipsoid.Degrees, ellipsoid.Meter, ellipsoid.Longitude_is_symmetric, ellipsoid.Bearing_is_symmetric)
		lat2, lon2 := geo1.At(lat1, lon1, 2000.0, 45.0)
		fmt.Printf("8 lat  = %v lon = %v\n", lat2, lon2)
	}
	{
		geo1 := ellipsoid.Init("AIRY", ellipsoid.Degrees, ellipsoid.Meter, ellipsoid.Longitude_is_symmetric, ellipsoid.Bearing_is_symmetric)
		lat2, lon2 := geo1.At(90.0, 90.0, 1000.0, 90.0)
		fmt.Printf("9 lat  = %v lon = %v\n", lat2, lon2)
	}
	// To
	{
		geo1 := ellipsoid.Init("WGS84", ellipsoid.Degrees, ellipsoid.Nm, ellipsoid.Longitude_is_symmetric, ellipsoid.Bearing_is_symmetric)
		lat3, lon3 := 73.06, 19.11 // Mumbai
		lat4, lon4 := 4.89, 52.37  // Amsterdam
		dist, bear := geo1.To(lat3, lon3, lat4, lon4)
		fmt.Printf("10 dist  = %v bear = %v\n", dist, bear)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strings"
	"time"

	"strconv"

	"github.com/izzulhaziq/glow/flow"
)

func aggregate(param aggrParam) <-chan map[string]interface{} {
	aggrOut := make(chan map[string]interface{})
	f := flow.New().Source(func(out chan map[string]interface{}) {
		if err := cfg.src.read(out); err != nil {
			panic(err)
		}
	}, 5).Map(func(data map[string]interface{}) flow.KeyValue {
		key := groupKey(param.GroupBy, param.Interval, data)
		val, ok := data[param.AggregatedField].(int)
		if !ok {
			val, _ = strconv.Atoi(data[param.AggregatedField].(string))
		}
		return flow.KeyValue{Key: key, Value: val}
	}).ReduceByKey(func(x int, y int) int {
		return x + y
	}).Map(func(group string, count int) flow.KeyValue {
		k := strings.Split(group, ",")
		v := map[string]int{
			strings.Join(k[:len(k)-1], ","): count,
		}
		// key = date, value = [group]
		return flow.KeyValue{Key: k[len(k)-1], Value: v}
	}).GroupByKey().Map(func(group string, values []map[string]int) map[string]interface{} {
		flatten := map[string]interface{}{
			"date": group,
		}
		for _, item := range values {
			for k, v := range item {
				flatten[k] = v
			}
		}
		// { date, group1, group2, ... }
		return flatten
	}).AddOutput(aggrOut)

	go f.Run()
	return aggrOut
}

func closeFlow() {
	copy(flow.Contexts[0:], flow.Contexts[1:])
	flow.Contexts[len(flow.Contexts)-1] = nil
	flow.Contexts = flow.Contexts[:len(flow.Contexts)-1]
}

func groupKey(groupBy []string, interval string, data map[string]interface{}) (key string) {
	var keys []string
	for _, g := range groupBy {
		keys = append(keys, data[g].(string))
	}

	t, ok := data[cfg.dateKey].(time.Time)
	if !ok {
		parsed, err := time.Parse(cfg.dateFormat, data[cfg.dateKey].(string))
		if err != nil {
			panic(err)
		}
		t = parsed
	}

	keys = append(keys, fromInterval(t, interval))
	key = strings.Join(keys, ",")
	return
}

func fromInterval(t time.Time, interval string) string {
	var timeKey string
	switch interval {
	case "daily":
		timeKey = fmt.Sprintf("%04d-%02d-%02d", t.Year(), t.Month(), t.Day())
	case "monthly":
		timeKey = fmt.Sprintf("%04d-%02d", t.Year(), t.Month())
	}
	return timeKey
}
<commit_msg>handle malformed data and allow count aggr<commit_after>package main

import (
	"strings"
	"time"

	"strconv"

	"github.com/izzulhaziq/glow/flow"
)

func aggregate(param aggrParam) <-chan map[string]interface{} {
	aggrOut := make(chan map[string]interface{})
	f := flow.New().Source(func(out chan map[string]interface{}) {
		if err := cfg.src.read(out); err != nil {
			panic(err)
		}
	}, 5).Map(func(data map[string]interface{}) flow.KeyValue {
		key, val := mapToGroup(param, data)
		return flow.KeyValue{Key: key, Value: val}
	}).ReduceByKey(func(x int, y int) int {
		return x + y
	}).Map(func(group string, count int) flow.KeyValue {
		k := strings.Split(group, ",")
		v := map[string]int{
			strings.Join(k[:len(k)-1], ","): count,
		}
		// key = date, value = [group]
		return flow.KeyValue{Key: k[len(k)-1], Value: v}
	}).GroupByKey().Map(func(group string, values []map[string]int) map[string]interface{} {
		flatten := map[string]interface{}{
			"date": group,
		}
		for _, item := range values {
			for k, v := range item {
				flatten[k] = v
			}
		}
		// { date, group1, group2, ... }
		return flatten
	}).AddOutput(aggrOut)

	go f.Run()
	return aggrOut
}

func mapToGroup(param aggrParam, data map[string]interface{}) (string, int) {
	key := getKey(param.GroupBy, param.Interval, data)
	var val int
	if param.AggregatedField == "" {
		val = 1
	} else {
		parsed, ok := data[param.AggregatedField].(int)
		if !ok {
			parsed, _ = strconv.Atoi(data[param.AggregatedField].(string))
		}
		val = parsed
	}

	return key, val
}

func closeFlow() {
	copy(flow.Contexts[0:], flow.Contexts[1:])
	flow.Contexts[len(flow.Contexts)-1] = nil
	flow.Contexts = flow.Contexts[:len(flow.Contexts)-1]
}

func getKey(groupBy []string, interval string, data map[string]interface{}) (key string) {
	var keys []string
	for _, g := range groupBy {
		k, ok := data[g].(string)
		if !ok {
			k = "null"
		}
		keys = append(keys, k)
	}

	t, ok := data[cfg.dateKey].(time.Time)
	if !ok {
		parsed, err := time.Parse(cfg.dateFormat, data[cfg.dateKey].(string))
		if err != nil {
			panic(err)
		}
		t = parsed
	}

	keys = append(keys, fromInterval(t, interval))
	key = strings.Join(keys, ",")
	return
}

func fromInterval(t time.Time, interval string) string {
	var timeKey string
	switch interval {
	case "daily":
		timeKey = t.Format("2006-01-02")
		//timeKey = fmt.Sprintf("%04d-%02d-%02d", t.Year(), t.Month(), t.Day())
	case "monthly":
		timeKey = t.Format("2006-01")
		//timeKey = fmt.Sprintf("%04d-%02d", t.Year(), t.Month())
	case "yearly":
		timeKey = t.Format("2006")
	}
	return timeKey
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"

	"github.com/pbnjay/db"
)

func init() {
	db.Schema = []string{
		`CREATE TABLE hotels (
        id serial primary key,
        name varchar
     );`,
		`CREATE TABLE rooms (
        id serial primary key,
        number integer,
        hotel int references hotels(id)
     );`,
	}
}

func main() {
	flag.Parse()
	db.MustInit()

	args := flag.Args()
	if len(args) == 0 {
		fmt.Println("USAGE: ./hotels 'Hotel Name' 101 102 103 ...")
		return
	}

	var hotelID int
	err := db.DB.QueryRow(`SELECT id FROM hotels WHERE name=$1`, args[0]).Scan(&hotelID)
	if err == db.ErrNotFound {
		fmt.Printf("'%s' doesn't exist, creating...\n", args[0])
		err = db.DB.QueryRow(`INSERT INTO hotels (name) VALUES ($1) RETURNING id`,
			args[0]).Scan(&hotelID)
	}
	if err != nil {
		fmt.Println("error creating hotel: ", err)
		return
	}

	for _, roomNum := range args[1:] {
		// no error checking for brevity of example
		db.DB.Exec(`INSERT INTO rooms (number, hotel) VALUES ($1,$2)`, roomNum, hotelID)
	}
}
<commit_msg>adding addresses to hotels example for an example<commit_after>package main

import (
	"flag"
	"fmt"

	"github.com/pbnjay/db"
)

func init() {
	db.Schema = []string{
		`CREATE TABLE hotels (
        id serial primary key,
        name varchar
     );`,
		`CREATE TABLE rooms (
        id serial primary key,
        number integer,
        hotel int references hotels(id)
     );`,
		`ALTER TABLE hotels ADD COLUMN address varchar;`,
	}
}

func main() {
	flag.Parse()
	db.MustInit()

	args := flag.Args()
	if len(args) == 0 {
		fmt.Println("USAGE: ./hotels 'Hotel Name' 'Hotel Address' 101 102 103 ...")
		return
	}

	var hotelID int
	err := db.DB.QueryRow(`SELECT id FROM hotels WHERE name=$1`, args[0]).Scan(&hotelID)
	if err == nil {
		db.DB.Exec("UPDATE hotels SET address=$2 WHERE id=$1", hotelID, args[1])
	}
	if err == db.ErrNotFound {
		fmt.Printf("'%s' doesn't exist, creating...\n", args[0])
		err = db.DB.QueryRow(`INSERT INTO hotels (name,address) VALUES ($1,$2) RETURNING id`,
			args[0], args[1]).Scan(&hotelID)
	}
	if err != nil {
		fmt.Println("error creating hotel: ", err)
		return
	}

	for _, roomNum := range args[2:] {
		// no error checking for brevity of example
		db.DB.Exec(`INSERT INTO rooms (number, hotel) VALUES ($1,$2)`, roomNum, hotelID)
	}
}
<|endoftext|>


<commit_before>package example

import (
	"log"
	"net/http"

	"github.com/adams-sarah/test2doc/example/foos"
	"github.com/adams-sarah/test2doc/example/widgets"
	"github.com/gorilla/mux"
)

func main() {
	router := NewRouter()
	log.Fatal(http.ListenAndServe(":8080", router))
}

func NewRouter() *mux.Router {
	r := mux.NewRouter()
	r.HandleFunc("/foos", foos.GetFoos).Methods("GET").Name("GetFoos")
	r.HandleFunc("/foos/{key}", foos.GetFoo).Methods("GET").Name("GetFoo")

	r.HandleFunc("/widgets", widgets.GetWidgets).Methods("GET").Name("GetWidgets")
	r.HandleFunc("/widgets", widgets.PostWidget).Methods("POST").Name("PostWidget")
	r.HandleFunc("/widgets/{id}", widgets.GetWidget).Methods("GET").Name("GetWidget")

	return r
}
<commit_msg>Mark example package as a binary<commit_after>package main

import (
	"log"
	"net/http"

	"github.com/adams-sarah/test2doc/example/foos"
	"github.com/adams-sarah/test2doc/example/widgets"
	"github.com/gorilla/mux"
)

func main() {
	router := NewRouter()
	log.Fatal(http.ListenAndServe(":8080", router))
}

func NewRouter() *mux.Router {
	r := mux.NewRouter()
	r.HandleFunc("/foos", foos.GetFoos).Methods("GET").Name("GetFoos")
	r.HandleFunc("/foos/{key}", foos.GetFoo).Methods("GET").Name("GetFoo")

	r.HandleFunc("/widgets", widgets.GetWidgets).Methods("GET").Name("GetWidgets")
	r.HandleFunc("/widgets", widgets.PostWidget).Methods("POST").Name("PostWidget")
	r.HandleFunc("/widgets/{id}", widgets.GetWidget).Methods("GET").Name("GetWidget")

	return r
}
<|endoftext|>


<commit_before>package analytics

//
// dependencies
//

import "github.com/nu7hatch/gouuid"
import . "encoding/json"
import "net/http"
import "bytes"
import "time"
import "log"

//
// Library version
//

const Version = "0.0.1"

//
// Default API end-point
//

const api = "https://api.segment.io"

//
// Segment.io client
//

type client struct {
	debug      bool
	key        string
	url        string
	flushAt    int
	flushAfter time.Duration
	buffer     []*interface{}
}

//
// Message context library
//

type contextLibrary struct {
	Name    string `json:"name"`
	Version string `json:"version"`
}

//
// Message context
//

type context struct {
	Library contextLibrary `json:"library"`
}

//
// Identify message
//

type identify struct {
	Action string      `json:"action"`
	Traits interface{} `json:"trailts"`
}

//
// Alias message
//

type alias struct {
	Action     string `json:"action"`
	PreviousId string `json:"previousId"`
}

//
// Track message
//

type track struct {
	Action     string      `json:"action"`
	Event      string      `json:"event"`
	Properties interface{} `json:"properties"`
}

//
// Group message
//

type group struct {
	Action  string      `json:"action"`
	GroupId string      `json:"groupId"`
	Traits  interface{} `json:"trailts"`
}

//
// Page message
//

type page struct {
	Action     string      `json:"action"`
	Category   string      `json:"category"`
	Name       string      `json:"name"`
	Properties interface{} `json:"properties"`
}

//
// Batch message
//

type batch struct {
	Context   context        `json:"context"`
	RequestId string         `json:"requestId"`
	Messages  []*interface{} `json:"batch"`
}

//
// Return a new Segment.io client
// with the given write key.
//

func Client(key string) *client {
	return &client{
		key:        key,
		url:        api,
		flushAt:    500,
		flushAfter: 10 * time.Second,
		buffer:     make([]*interface{}, 0),
	}
}

//
// Set buffer max.
//

func (c *client) FlushAt(n int) {
	c.flushAt = n
}

//
// Set buffer flush interal.
//

func (c *client) FlushAfter(interval time.Duration) {
	c.flushAfter = interval
}

//
// Enable debug mode.
//

func (c *client) Debug() {
	c.debug = true
}

//
// Set target url
//

func (c *client) URL(url string) {
	c.url = url
}

// Return a batch message primed
// with context properties
//

func createBatch(msgs []*interface{}) (*batch, error) {
	uid, err := uuid.NewV4()

	if err != nil {
		return nil, err
	}

	batch := &batch{
		RequestId: uid.String(),
		Messages:  msgs,
		Context: context{
			Library: contextLibrary{
				Name:    "analytics-go",
				Version: Version,
			},
		},
	}

	return batch, nil
}

//
// Flush the buffered messages.
//

func (c *client) flush() error {
	b, err := createBatch(c.buffer)

	if err != nil {
		return err
	}

	j, err := Marshal(b)

	if err != nil {
		return err
	}

	c.buffer = nil

	client := &http.Client{}
	req, err := http.NewRequest("POST", c.url+"/v1/batch", bytes.NewBuffer(j))

	if err != nil {
		return err
	}

	req.Header.Add("User-Agent", "analytics-go (version: "+Version+")")
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("Content-Length", string(len(j)))
	req.SetBasicAuth(c.key, "")

	res, err := client.Do(req)

	if res != nil {
		// TODO: how the fuck do you ignore res ^
	}

	return err
}

//
// Buffer the given message and flush
// when the buffer exceeds .flushAt.
//

func (c *client) bufferMessage(msg interface{}) error {
	c.buffer = append(c.buffer, &msg)

	if c.debug {
		log.Printf("buffer (%d/%d) %v", len(c.buffer), c.flushAt, msg)
	}

	if len(c.buffer) >= c.flushAt {
		log.Printf("flushing %d messages", len(c.buffer))
		return c.flush()
	}

	return nil
}

//
// Buffer an alias message
//

func (c *client) Alias(previousId string) error {
	return c.bufferMessage(&alias{"Alias", previousId})
}

//
// Buffer a page message
//

func (c *client) Page(name string, category string, properties interface{}) error {
	return c.bufferMessage(&page{"Page", name, category, properties})
}

//
// Buffer a screen message
//

func (c *client) Screen(name string, category string, properties interface{}) error {
	return c.bufferMessage(&page{"Screen", name, category, properties})
}

//
// Buffer a group message
//

func (c *client) Group(id string, traits interface{}) error {
	return c.bufferMessage(&group{"Group", id, traits})
}

//
// Buffer an identify message
//

func (c *client) Identify(traits interface{}) error {
	return c.bufferMessage(&identify{"Identify", traits})
}

//
// Buffer a track message
//

func (c *client) Track(event string, properties interface{}) error {
	// TODO: .timestamp ISO-8601-formatted string.
	return c.bufferMessage(&track{"Track", event, properties})
}
<commit_msg>add .timestamp<commit_after>package analytics

//
// dependencies
//

import "github.com/jehiah/go-strftime"
import "github.com/nu7hatch/gouuid"
import . "encoding/json"
import "net/http"
import "bytes"
import "time"
import "log"

//
// Library version
//

const Version = "0.0.1"

//
// Default API end-point
//

const api = "https://api.segment.io"

//
// Segment.io client
//

type client struct {
	debug      bool
	key        string
	url        string
	flushAt    int
	flushAfter time.Duration
	buffer     []*interface{}
}

//
// Message context library
//

type contextLibrary struct {
	Name    string `json:"name"`
	Version string `json:"version"`
}

//
// Message context
//

type context struct {
	Library contextLibrary `json:"library"`
}

//
// Identify message
//

type identify struct {
	Action string      `json:"action"`
	Traits interface{} `json:"trailts"`
}

//
// Alias message
//

type alias struct {
	Action     string `json:"action"`
	PreviousId string `json:"previousId"`
}

//
// Track message
//

type track struct {
	Action     string      `json:"action"`
	Event      string      `json:"event"`
	Properties interface{} `json:"properties"`
}

//
// Group message
//

type group struct {
	Action  string      `json:"action"`
	GroupId string      `json:"groupId"`
	Traits  interface{} `json:"trailts"`
}

//
// Page message
//

type page struct {
	Action     string      `json:"action"`
	Category   string      `json:"category"`
	Name       string      `json:"name"`
	Properties interface{} `json:"properties"`
}

//
// Batch message
//

type batch struct {
	Timestamp string         `json:"timestamp"`
	Context   context        `json:"context"`
	RequestId string         `json:"requestId"`
	Messages  []*interface{} `json:"batch"`
}

//
// Return a new Segment.io client
// with the given write key.
//

func Client(key string) *client {
	return &client{
		key:        key,
		url:        api,
		flushAt:    500,
		flushAfter: 10 * time.Second,
		buffer:     make([]*interface{}, 0),
	}
}

//
// Set buffer max.
//

func (c *client) FlushAt(n int) {
	c.flushAt = n
}

//
// Set buffer flush interal.
//

func (c *client) FlushAfter(interval time.Duration) {
	c.flushAfter = interval
}

//
// Enable debug mode.
//

func (c *client) Debug() {
	c.debug = true
}

//
// Set target url
//

func (c *client) URL(url string) {
	c.url = url
}

// Return a batch message primed
// with context properties
//

func createBatch(msgs []*interface{}) (*batch, error) {
	uid, err := uuid.NewV4()

	if err != nil {
		return nil, err
	}

	batch := &batch{
		Timestamp: strftime.Format("%Y-%m-%dT%H:%M:%S%z", time.Now()),
		RequestId: uid.String(),
		Messages:  msgs,
		Context: context{
			Library: contextLibrary{
				Name:    "analytics-go",
				Version: Version,
			},
		},
	}

	return batch, nil
}

//
// Flush the buffered messages.
//

func (c *client) flush() error {
	b, err := createBatch(c.buffer)

	if err != nil {
		return err
	}

	j, err := Marshal(b)

	if err != nil {
		return err
	}

	c.buffer = nil

	client := &http.Client{}
	req, err := http.NewRequest("POST", c.url+"/v1/batch", bytes.NewBuffer(j))

	if err != nil {
		return err
	}

	req.Header.Add("User-Agent", "analytics-go (version: "+Version+")")
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("Content-Length", string(len(j)))
	req.SetBasicAuth(c.key, "")

	res, err := client.Do(req)

	if res != nil {
		// TODO: how the fuck do you ignore res ^
	}

	return err
}

//
// Buffer the given message and flush
// when the buffer exceeds .flushAt.
//

func (c *client) bufferMessage(msg interface{}) error {
	c.buffer = append(c.buffer, &msg)

	if c.debug {
		log.Printf("buffer (%d/%d) %v", len(c.buffer), c.flushAt, msg)
	}

	if len(c.buffer) >= c.flushAt {
		log.Printf("flushing %d messages", len(c.buffer))
		return c.flush()
	}

	return nil
}

//
// Buffer an alias message
//

func (c *client) Alias(previousId string) error {
	return c.bufferMessage(&alias{"Alias", previousId})
}

//
// Buffer a page message
//

func (c *client) Page(name string, category string, properties interface{}) error {
	return c.bufferMessage(&page{"Page", name, category, properties})
}

//
// Buffer a screen message
//

func (c *client) Screen(name string, category string, properties interface{}) error {
	return c.bufferMessage(&page{"Screen", name, category, properties})
}

//
// Buffer a group message
//

func (c *client) Group(id string, traits interface{}) error {
	return c.bufferMessage(&group{"Group", id, traits})
}

//
// Buffer an identify message
//

func (c *client) Identify(traits interface{}) error {
	return c.bufferMessage(&identify{"Identify", traits})
}

//
// Buffer a track message
//

func (c *client) Track(event string, properties interface{}) error {
	return c.bufferMessage(&track{"Track", event, properties})
}
<|endoftext|>


<commit_before>package analytics

import (
	"io/ioutil"
	"os"

	"bytes"
	"encoding/json"
	"errors"
	"log"
	"net/http"
	"time"

	"github.com/jehiah/go-strftime"
	"github.com/xtgo/uuid"
)

// Version of the client.
const Version = "2.0.0"

// Endpoint for the Segment API.
const Endpoint = "https://api.segment.io"

// DefaultContext of message batches.
var DefaultContext = map[string]interface{}{
	"library": map[string]interface{}{
		"name":    "analytics-go",
		"version": Version,
	},
}

// Message interface.
type message interface {
	setMessageId(string)
	setTimestamp(string)
}

// Message fields common to all.
type Message struct {
	Type      string `json:"type,omitempty"`
	MessageId string `json:"messageId,omitempty"`
	Timestamp string `json:"timestamp,omitempty"`
	SentAt    string `json:"sentAt,omitempty"`
}

// Batch message.
type Batch struct {
	Context  map[string]interface{} `json:"context,omitempty"`
	Messages []interface{}          `json:"batch"`
	Message
}

// Identify message.
type Identify struct {
	Context      map[string]interface{} `json:"context,omitempty"`
	Integrations map[string]interface{} `json:"integrations,omitempty"`
	Traits       map[string]interface{} `json:"traits,omitempty"`
	AnonymousId  string                 `json:"anonymousId,omitempty"`
	UserId       string                 `json:"userId,omitempty"`
	Message
}

// Group message.
type Group struct {
	Context      map[string]interface{} `json:"context,omitempty"`
	Integrations map[string]interface{} `json:"integrations,omitempty"`
	Traits       map[string]interface{} `json:"traits,omitempty"`
	AnonymousId  string                 `json:"anonymousId,omitempty"`
	UserId       string                 `json:"userId,omitempty"`
	GroupId      string                 `json:"groupId"`
	Message
}

// Track message.
type Track struct {
	Context      map[string]interface{} `json:"context,omitempty"`
	Integrations map[string]interface{} `json:"integrations,omitempty"`
	Properties   map[string]interface{} `json:"properties,omitempty"`
	AnonymousId  string                 `json:"anonymousId,omitempty"`
	UserId       string                 `json:"userId,omitempty"`
	Event        string                 `json:"event"`
	Message
}

// Page message.
type Page struct {
	Context      map[string]interface{} `json:"context,omitempty"`
	Integrations map[string]interface{} `json:"integrations,omitempty"`
	Traits       map[string]interface{} `json:"properties,omitempty"`
	AnonymousId  string                 `json:"anonymousId,omitempty"`
	UserId       string                 `json:"userId,omitempty"`
	Category     string                 `json:"category,omitempty"`
	Name         string                 `json:"name,omitempty"`
	Message
}

// Alias message.
type Alias struct {
	PreviousId string `json:"previousId"`
	UserId     string `json:"userId"`
	Message
}

// Client which batches messages and flushes at the given Interval or
// when the Size limit is exceeded. Set Verbose to true to enable
// logging output.
type Client struct {
	Endpoint string
	Interval time.Duration
	Size     int
	Logger   *log.Logger
	Verbose  bool
	Client   http.Client
	key      string
	msgs     chan interface{}
	quit     chan struct{}
	shutdown chan struct{}
	uid      func() string
	now      func() time.Time
}

// New client with write key.
func New(key string) *Client {
	c := &Client{
		Endpoint: Endpoint,
		Interval: 5 * time.Second,
		Size:     250,
		Logger:   log.New(os.Stderr, "segment ", log.LstdFlags),
		Verbose:  false,
		Client:   *http.DefaultClient,
		key:      key,
		msgs:     make(chan interface{}, 100),
		quit:     make(chan struct{}),
		shutdown: make(chan struct{}),
		now:      time.Now,
		uid:      uid,
	}

	go c.loop()

	return c
}

// Alias buffers an "alias" message.
func (c *Client) Alias(msg *Alias) error {
	if msg.UserId == "" {
		return errors.New("You must pass a 'userId'.")
	}

	if msg.PreviousId == "" {
		return errors.New("You must pass a 'previousId'.")
	}

	msg.Type = "alias"
	c.queue(msg)

	return nil
}

// Page buffers an "page" message.
func (c *Client) Page(msg *Page) error {
	if msg.UserId == "" && msg.AnonymousId == "" {
		return errors.New("You must pass either an 'anonymousId' or 'userId'.")
	}

	msg.Type = "page"
	c.queue(msg)

	return nil
}

// Group buffers an "group" message.
func (c *Client) Group(msg *Group) error {
	if msg.GroupId == "" {
		return errors.New("You must pass a 'groupId'.")
	}

	if msg.UserId == "" && msg.AnonymousId == "" {
		return errors.New("You must pass either an 'anonymousId' or 'userId'.")
	}

	msg.Type = "group"
	c.queue(msg)

	return nil
}

// Identify buffers an "identify" message.
func (c *Client) Identify(msg *Identify) error {
	if msg.UserId == "" && msg.AnonymousId == "" {
		return errors.New("You must pass either an 'anonymousId' or 'userId'.")
	}

	msg.Type = "identify"
	c.queue(msg)

	return nil
}

// Track buffers an "track" message.
func (c *Client) Track(msg *Track) error {
	if msg.Event == "" {
		return errors.New("You must pass 'event'.")
	}

	if msg.UserId == "" && msg.AnonymousId == "" {
		return errors.New("You must pass either an 'anonymousId' or 'userId'.")
	}

	msg.Type = "track"
	c.queue(msg)

	return nil
}

// Queue message.
func (c *Client) queue(msg message) {
	msg.setMessageId(c.uid())
	msg.setTimestamp(timestamp(c.now()))
	c.msgs <- msg
}

// Close and flush metrics.
func (c *Client) Close() error {
	c.quit <- struct{}{}
	close(c.msgs)
	<-c.shutdown
	return nil
}

// Send batch request.
func (c *Client) send(msgs []interface{}) {
	if len(msgs) == 0 {
		return
	}

	batch := new(Batch)
	batch.Messages = msgs
	batch.MessageId = c.uid()
	batch.SentAt = timestamp(c.now())
	batch.Context = DefaultContext

	b, err := json.Marshal(batch)
	if err != nil {
		c.log("error marshalling msgs: %s", err)
		return
	}

	url := c.Endpoint + "/v1/batch"
	req, err := http.NewRequest("POST", url, bytes.NewReader(b))
	if err != nil {
		c.log("error creating request: %s", err)
		return
	}

	req.Header.Add("User-Agent", "analytics-go (version: "+Version+")")
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("Content-Length", string(len(b)))
	req.SetBasicAuth(c.key, "")

	res, err := c.Client.Do(req)
	if err != nil {
		c.log("error sending request: %s", err)
		return
	}
	defer res.Body.Close()

	c.report(res)
}

// Report on response body.
func (c *Client) report(res *http.Response) {
	if res.StatusCode < 400 {
		c.verbose("response %s", res.Status)
		return
	}

	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		c.log("error reading response body: %s", err)
		return
	}

	c.log("response %s: %s – %s", res.Status, res.StatusCode, body)
}

// Batch loop.
func (c *Client) loop() {
	var msgs []interface{}
	tick := time.NewTicker(c.Interval)

	for {
		select {
		case msg := <-c.msgs:
			c.verbose("buffer (%d/%d) %v", len(msgs), c.Size, msg)
			msgs = append(msgs, msg)
			if len(msgs) == c.Size {
				c.verbose("exceeded %d messages – flushing", c.Size)
				c.send(msgs)
				msgs = nil
			}
		case <-tick.C:
			if len(msgs) > 0 {
				c.verbose("interval reached - flushing %d", len(msgs))
				c.send(msgs)
				msgs = nil
			} else {
				c.verbose("interval reached – nothing to send")
			}
		case <-c.quit:
			c.verbose("exit requested – draining msgs")
			// drain the msg channel.
			for msg := range c.msgs {
				c.verbose("buffer (%d/%d) %v", len(msgs), c.Size, msg)
				msgs = append(msgs, msg)
			}
			c.verbose("exit requested – flushing %d", len(msgs))
			c.send(msgs)
			c.verbose("exit")
			c.shutdown <- struct{}{}
			return
		}
	}
}

// Verbose log.
func (c *Client) verbose(msg string, args ...interface{}) {
	if c.Verbose {
		c.Logger.Printf(msg, args...)
	}
}

// Unconditional log.
func (c *Client) log(msg string, args ...interface{}) {
	c.Logger.Printf(msg, args...)
}

// Set message timestamp if one is not already set.
func (m *Message) setTimestamp(s string) {
	if m.Timestamp == "" {
		m.Timestamp = s
	}
}

// Set message id.
func (m *Message) setMessageId(s string) {
	m.MessageId = s
}

// Return formatted timestamp.
func timestamp(t time.Time) string {
	return strftime.Format("%Y-%m-%dT%H:%M:%S%z", t)
}

// Return uuid string.
func uid() string {
	return uuid.NewRandom().String()
}
<commit_msg>Fixed resource leak with ticker<commit_after>package analytics

import (
	"io/ioutil"
	"os"

	"bytes"
	"encoding/json"
	"errors"
	"log"
	"net/http"
	"time"

	"github.com/jehiah/go-strftime"
	"github.com/xtgo/uuid"
)

// Version of the client.
const Version = "2.0.0"

// Endpoint for the Segment API.
const Endpoint = "https://api.segment.io"

// DefaultContext of message batches.
var DefaultContext = map[string]interface{}{
	"library": map[string]interface{}{
		"name":    "analytics-go",
		"version": Version,
	},
}

// Message interface.
type message interface {
	setMessageId(string)
	setTimestamp(string)
}

// Message fields common to all.
type Message struct {
	Type      string `json:"type,omitempty"`
	MessageId string `json:"messageId,omitempty"`
	Timestamp string `json:"timestamp,omitempty"`
	SentAt    string `json:"sentAt,omitempty"`
}

// Batch message.
type Batch struct {
	Context  map[string]interface{} `json:"context,omitempty"`
	Messages []interface{}          `json:"batch"`
	Message
}

// Identify message.
type Identify struct {
	Context      map[string]interface{} `json:"context,omitempty"`
	Integrations map[string]interface{} `json:"integrations,omitempty"`
	Traits       map[string]interface{} `json:"traits,omitempty"`
	AnonymousId  string                 `json:"anonymousId,omitempty"`
	UserId       string                 `json:"userId,omitempty"`
	Message
}

// Group message.
type Group struct {
	Context      map[string]interface{} `json:"context,omitempty"`
	Integrations map[string]interface{} `json:"integrations,omitempty"`
	Traits       map[string]interface{} `json:"traits,omitempty"`
	AnonymousId  string                 `json:"anonymousId,omitempty"`
	UserId       string                 `json:"userId,omitempty"`
	GroupId      string                 `json:"groupId"`
	Message
}

// Track message.
type Track struct {
	Context      map[string]interface{} `json:"context,omitempty"`
	Integrations map[string]interface{} `json:"integrations,omitempty"`
	Properties   map[string]interface{} `json:"properties,omitempty"`
	AnonymousId  string                 `json:"anonymousId,omitempty"`
	UserId       string                 `json:"userId,omitempty"`
	Event        string                 `json:"event"`
	Message
}

// Page message.
type Page struct {
	Context      map[string]interface{} `json:"context,omitempty"`
	Integrations map[string]interface{} `json:"integrations,omitempty"`
	Traits       map[string]interface{} `json:"properties,omitempty"`
	AnonymousId  string                 `json:"anonymousId,omitempty"`
	UserId       string                 `json:"userId,omitempty"`
	Category     string                 `json:"category,omitempty"`
	Name         string                 `json:"name,omitempty"`
	Message
}

// Alias message.
type Alias struct {
	PreviousId string `json:"previousId"`
	UserId     string `json:"userId"`
	Message
}

// Client which batches messages and flushes at the given Interval or
// when the Size limit is exceeded. Set Verbose to true to enable
// logging output.
type Client struct {
	Endpoint string
	Interval time.Duration
	Size     int
	Logger   *log.Logger
	Verbose  bool
	Client   http.Client
	key      string
	msgs     chan interface{}
	quit     chan struct{}
	shutdown chan struct{}
	uid      func() string
	now      func() time.Time
}

// New client with write key.
func New(key string) *Client {
	c := &Client{
		Endpoint: Endpoint,
		Interval: 5 * time.Second,
		Size:     250,
		Logger:   log.New(os.Stderr, "segment ", log.LstdFlags),
		Verbose:  false,
		Client:   *http.DefaultClient,
		key:      key,
		msgs:     make(chan interface{}, 100),
		quit:     make(chan struct{}),
		shutdown: make(chan struct{}),
		now:      time.Now,
		uid:      uid,
	}

	go c.loop()

	return c
}

// Alias buffers an "alias" message.
func (c *Client) Alias(msg *Alias) error {
	if msg.UserId == "" {
		return errors.New("You must pass a 'userId'.")
	}

	if msg.PreviousId == "" {
		return errors.New("You must pass a 'previousId'.")
	}

	msg.Type = "alias"
	c.queue(msg)

	return nil
}

// Page buffers an "page" message.
func (c *Client) Page(msg *Page) error {
	if msg.UserId == "" && msg.AnonymousId == "" {
		return errors.New("You must pass either an 'anonymousId' or 'userId'.")
	}

	msg.Type = "page"
	c.queue(msg)

	return nil
}

// Group buffers an "group" message.
func (c *Client) Group(msg *Group) error {
	if msg.GroupId == "" {
		return errors.New("You must pass a 'groupId'.")
	}

	if msg.UserId == "" && msg.AnonymousId == "" {
		return errors.New("You must pass either an 'anonymousId' or 'userId'.")
	}

	msg.Type = "group"
	c.queue(msg)

	return nil
}

// Identify buffers an "identify" message.
func (c *Client) Identify(msg *Identify) error {
	if msg.UserId == "" && msg.AnonymousId == "" {
		return errors.New("You must pass either an 'anonymousId' or 'userId'.")
	}

	msg.Type = "identify"
	c.queue(msg)

	return nil
}

// Track buffers an "track" message.
func (c *Client) Track(msg *Track) error {
	if msg.Event == "" {
		return errors.New("You must pass 'event'.")
	}

	if msg.UserId == "" && msg.AnonymousId == "" {
		return errors.New("You must pass either an 'anonymousId' or 'userId'.")
	}

	msg.Type = "track"
	c.queue(msg)

	return nil
}

// Queue message.
func (c *Client) queue(msg message) {
	msg.setMessageId(c.uid())
	msg.setTimestamp(timestamp(c.now()))
	c.msgs <- msg
}

// Close and flush metrics.
func (c *Client) Close() error {
	c.quit <- struct{}{}
	close(c.msgs)
	<-c.shutdown
	return nil
}

// Send batch request.
func (c *Client) send(msgs []interface{}) {
	if len(msgs) == 0 {
		return
	}

	batch := new(Batch)
	batch.Messages = msgs
	batch.MessageId = c.uid()
	batch.SentAt = timestamp(c.now())
	batch.Context = DefaultContext

	b, err := json.Marshal(batch)
	if err != nil {
		c.log("error marshalling msgs: %s", err)
		return
	}

	url := c.Endpoint + "/v1/batch"
	req, err := http.NewRequest("POST", url, bytes.NewReader(b))
	if err != nil {
		c.log("error creating request: %s", err)
		return
	}

	req.Header.Add("User-Agent", "analytics-go (version: "+Version+")")
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("Content-Length", string(len(b)))
	req.SetBasicAuth(c.key, "")

	res, err := c.Client.Do(req)
	if err != nil {
		c.log("error sending request: %s", err)
		return
	}
	defer res.Body.Close()

	c.report(res)
}

// Report on response body.
func (c *Client) report(res *http.Response) {
	if res.StatusCode < 400 {
		c.verbose("response %s", res.Status)
		return
	}

	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		c.log("error reading response body: %s", err)
		return
	}

	c.log("response %s: %s – %s", res.Status, res.StatusCode, body)
}

// Batch loop.
func (c *Client) loop() {
	var msgs []interface{}
	tick := time.NewTicker(c.Interval)

	for {
		select {
		case msg := <-c.msgs:
			c.verbose("buffer (%d/%d) %v", len(msgs), c.Size, msg)
			msgs = append(msgs, msg)
			if len(msgs) == c.Size {
				c.verbose("exceeded %d messages – flushing", c.Size)
				c.send(msgs)
				msgs = nil
			}
		case <-tick.C:
			if len(msgs) > 0 {
				c.verbose("interval reached - flushing %d", len(msgs))
				c.send(msgs)
				msgs = nil
			} else {
				c.verbose("interval reached – nothing to send")
			}
		case <-c.quit:
			tick.Stop()
			c.verbose("exit requested – draining msgs")
			// drain the msg channel.
			for msg := range c.msgs {
				c.verbose("buffer (%d/%d) %v", len(msgs), c.Size, msg)
				msgs = append(msgs, msg)
			}
			c.verbose("exit requested – flushing %d", len(msgs))
			c.send(msgs)
			c.verbose("exit")
			c.shutdown <- struct{}{}
			return
		}
	}
}

// Verbose log.
func (c *Client) verbose(msg string, args ...interface{}) {
	if c.Verbose {
		c.Logger.Printf(msg, args...)
	}
}

// Unconditional log.
func (c *Client) log(msg string, args ...interface{}) {
	c.Logger.Printf(msg, args...)
}

// Set message timestamp if one is not already set.
func (m *Message) setTimestamp(s string) {
	if m.Timestamp == "" {
		m.Timestamp = s
	}
}

// Set message id.
func (m *Message) setMessageId(s string) {
	m.MessageId = s
}

// Return formatted timestamp.
func timestamp(t time.Time) string {
	return strftime.Format("%Y-%m-%dT%H:%M:%S%z", t)
}

// Return uuid string.
func uid() string {
	return uuid.NewRandom().String()
}
<|endoftext|>