<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"math"
	"os"
	"strconv"
	"strings"

	"github.com/cespare/argf"
	"github.com/cespare/stats/b"
)

func summarize(args []string) {
	fs := flag.NewFlagSet("summarize", flag.ExitOnError)
	quantStr := fs.String("quantiles", "0.5,0.9,0.99", "Quantiles to record")
	printHist := fs.Bool("hist", false, "Print a histogram")
	histBuckets := fs.Int("buckets", 10, "How many buckets for the histogram")
	fs.Parse(args)

	if *histBuckets <= 1 {
		fmt.Fprintf(os.Stderr, "%d is an invalid number of buckets\n", *histBuckets)
	}

	var quants []float64
	for _, qs := range strings.Split(*quantStr, ",") {
		qs = strings.TrimSpace(qs)
		f, err := strconv.ParseFloat(qs, 64)
		if err != nil {
			log.Fatal(err)
		}
		if f <= 0 || f >= 1 {
			log.Fatal(fmt.Errorf("quantile values must be in (0, 1); got %f", f))
		}
		quants = append(quants, f)
	}

	btree := NewBTree()
	var nonNumericFound int64
	argf.Init(flag.Args())
	for argf.Scan() {
		s := argf.String()
		if s == "" {
			continue
		}
		v, err := strconv.ParseFloat(s, 64)
		if err != nil {
			nonNumericFound++
			continue
		}
		btree.Put(v, func(c uint, _ bool) (newC uint, write bool) { return c + 1, true })
	}
	if err := argf.Error(); err != nil {
		log.Fatal(err)
	}
	if nonNumericFound > 0 {
		fmt.Fprintf(os.Stderr, "Warning: found %d non-numeric lines of input\n", nonNumericFound)
	}
	if btree.Len() == 0 {
		fmt.Fprintln(os.Stderr, "No numbers given")
		return
	}
	stats := StatsFromBtree(btree)
	printStat("count", stats.Count)
	printStat("min", stats.min)
	printStat("max", stats.max)
	printStat("mean", stats.Mean())
	printStat("std. dev.", stats.Stdev())
	for _, q := range quants {
		name := fmt.Sprintf("quantile %f", q)
		name = strings.TrimRight(name, "0")
		printStat(name, stats.Quant(q))
	}
	if *printHist {
		fmt.Println(stats.Hist(*histBuckets))
	}
}

type Stats struct {
	Count      float64
	min        float64
	max        float64
	sum        float64
	sumSquares float64
	sorted     []float64
}

func (s *Stats) Mean() float64 {
	return s.sum / s.Count
}

func (s *Stats) Stdev() float64 {
	return math.Sqrt(s.Count*s.sumSquares-(s.sum*s.sum)) / s.Count
}

func (s *Stats) Quant(q float64) float64 {
	if q < 0 || q > 1 {
		panic("bad quantile")
	}
	i := round((s.Count - 1) * q)
	return s.sorted[i]
}

func round(f float64) int { return int(f + 0.5) }

func printStat(name string, value float64) {
	fmt.Printf("%-15s %7.3f\n", name, value)
}

func StatsFromBtree(btree *b.Tree) *Stats {
	enum, err := btree.SeekFirst()
	if err != nil {
		panic(err)
	}
	s := &Stats{sorted: make([]float64, 0, btree.Len())}
	for {
		k, c, err := enum.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			panic(err)
		}
		if s.Count == 0 || k < s.min {
			s.min = k
		}
		if s.Count == 0 || k > s.max {
			s.max = k
		}
		for i := 0; i < int(c); i++ {
			s.sorted = append(s.sorted, k)
			s.Count++
			s.sum += k
			s.sumSquares += k * k
		}
	}
	return s
}

func NewBTree() *b.Tree {
	return b.TreeNew(func(a, b float64) int {
		if a < b {
			return -1
		}
		if a == b {
			return 0
		}
		return 1
	})
}

type Bucket struct {
	start float64
	count uint64
}

type Hist struct {
	bucketSize float64
	buckets    []Bucket
}

func (s *Stats) Hist(n int) *Hist {
	h := &Hist{buckets: make([]Bucket, n)}
	rnge := s.max - s.min
	h.bucketSize = rnge / float64(n)
	i := 0
	limit := s.min + h.bucketSize
	h.buckets[0].start = s.min
	for j := 0; j < len(s.sorted); {
		v := s.sorted[j]
		if v >= limit && i < len(h.buckets)-1 {
			i++
			h.buckets[i].start = limit
			limit = s.min + float64(i+1)*(rnge/float64(n))
			continue
		}
		h.buckets[i].count++
		j++
	}
	return h
}

const histBlocks = 70

func (h *Hist) String() string {
	// TODO: if the range is large, expand the bucketsize and start/end a bit to get integer boundaries.
	labels := make([]string, len(h.buckets))
	labelSpaceBefore := 0
	labelSpaceAfter := 0
	var maxCount, sum float64
	for i, b := range h.buckets {
		sum += float64(b.count)
		s := "<"
		if i == len(h.buckets)-1 {
			s = "≤"
		}
		label := fmt.Sprintf("%.3g ≤ x %s %.3g", b.start, s, b.start+h.bucketSize)
		xPos := runeIndex(label, 'x')
		if xPos > labelSpaceBefore {
			labelSpaceBefore = xPos
		}
		if after := runeLen(label) - xPos - 1; after > labelSpaceAfter {
			labelSpaceAfter = after
		}
		labels[i] = label
		if f := float64(b.count); f > maxCount {
			maxCount = f
		}
	}

	var buf bytes.Buffer
	for i, b := range h.buckets {
		xPos := runeIndex(labels[i], 'x')
		before := labelSpaceBefore - xPos
		after := labelSpaceAfter - runeLen(labels[i]) + xPos + 1
		fmt.Fprintf(&buf, " %*s%s%*s │", before, "", labels[i], after, "")
		fmt.Fprint(&buf, makeBar((float64(b.count)/float64(maxCount))*histBlocks))
		fmt.Fprintf(&buf, " %d (%.3f%%)\n", b.count, 100*float64(b.count)/sum)
	}
	b := buf.Bytes()
	return string(b[:len(b)-1]) // drop the \n
}

func runeLen(s string) int { return len([]rune(s)) }

func runeIndex(s string, r rune) int {
	for i, r2 := range []rune(s) {
		if r2 == r {
			return i
		}
	}
	return -1
}

var barEighths = [9]rune{
	' ', // empty
	'▏',
	'▎',
	'▍',
	'▌',
	'▋',
	'▊',
	'▉',
	'█', // full
}

func makeBar(n float64) string {
	eighths := round(n * 8)
	full := eighths / 8
	rem := eighths % 8
	return strings.Repeat(string(barEighths[8]), full) + string(barEighths[rem])
}
<commit_msg>Clean up logging code<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"math"
	"strconv"
	"strings"

	"github.com/cespare/argf"
	"github.com/cespare/stats/b"
)

func summarize(args []string) {
	fs := flag.NewFlagSet("summarize", flag.ExitOnError)
	quantStr := fs.String("quantiles", "0.5,0.9,0.99", "Quantiles to record")
	printHist := fs.Bool("hist", false, "Print a histogram")
	histBuckets := fs.Int("buckets", 10, "How many buckets for the histogram")
	fs.Parse(args)

	if *histBuckets <= 1 {
		log.Fatalf("%d is an invalid number of buckets", *histBuckets)
	}

	var quants []float64
	for _, qs := range strings.Split(*quantStr, ",") {
		qs = strings.TrimSpace(qs)
		f, err := strconv.ParseFloat(qs, 64)
		if err != nil {
			log.Fatal(err)
		}
		if f <= 0 || f >= 1 {
			log.Fatalf("quantile values must be in (0, 1); got %g", f)
		}
		quants = append(quants, f)
	}

	btree := NewBTree()
	var nonNumericFound int64
	argf.Init(flag.Args())
	for argf.Scan() {
		s := argf.String()
		if s == "" {
			continue
		}
		v, err := strconv.ParseFloat(s, 64)
		if err != nil {
			nonNumericFound++
			continue
		}
		btree.Put(v, func(c uint, _ bool) (newC uint, write bool) { return c + 1, true })
	}
	if err := argf.Error(); err != nil {
		log.Fatal(err)
	}
	if nonNumericFound > 0 {
		log.Printf("warning: found %d non-numeric lines of input", nonNumericFound)
	}
	if btree.Len() == 0 {
		log.Println("no numbers given")
		return
	}
	stats := StatsFromBtree(btree)
	printStat("count", stats.Count)
	printStat("min", stats.min)
	printStat("max", stats.max)
	printStat("mean", stats.Mean())
	printStat("std. dev.", stats.Stdev())
	for _, q := range quants {
		name := fmt.Sprintf("quantile %f", q)
		name = strings.TrimRight(name, "0")
		printStat(name, stats.Quant(q))
	}
	if *printHist {
		fmt.Println(stats.Hist(*histBuckets))
	}
}

type Stats struct {
	Count      float64
	min        float64
	max        float64
	sum        float64
	sumSquares float64
	sorted     []float64
}

func (s *Stats) Mean() float64 {
	return s.sum / s.Count
}

func (s *Stats) Stdev() float64 {
	return math.Sqrt(s.Count*s.sumSquares-(s.sum*s.sum)) / s.Count
}

func (s *Stats) Quant(q float64) float64 {
	if q < 0 || q > 1 {
		panic("bad quantile")
	}
	i := round((s.Count - 1) * q)
	return s.sorted[i]
}

func round(f float64) int { return int(f + 0.5) }

func printStat(name string, value float64) {
	fmt.Printf("%-15s %7.3f\n", name, value)
}

func StatsFromBtree(btree *b.Tree) *Stats {
	enum, err := btree.SeekFirst()
	if err != nil {
		panic(err)
	}
	s := &Stats{sorted: make([]float64, 0, btree.Len())}
	for {
		k, c, err := enum.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			panic(err)
		}
		if s.Count == 0 || k < s.min {
			s.min = k
		}
		if s.Count == 0 || k > s.max {
			s.max = k
		}
		for i := 0; i < int(c); i++ {
			s.sorted = append(s.sorted, k)
			s.Count++
			s.sum += k
			s.sumSquares += k * k
		}
	}
	return s
}

func NewBTree() *b.Tree {
	return b.TreeNew(func(a, b float64) int {
		if a < b {
			return -1
		}
		if a == b {
			return 0
		}
		return 1
	})
}

type Bucket struct {
	start float64
	count uint64
}

type Hist struct {
	bucketSize float64
	buckets    []Bucket
}

func (s *Stats) Hist(n int) *Hist {
	h := &Hist{buckets: make([]Bucket, n)}
	rnge := s.max - s.min
	h.bucketSize = rnge / float64(n)
	i := 0
	limit := s.min + h.bucketSize
	h.buckets[0].start = s.min
	for j := 0; j < len(s.sorted); {
		v := s.sorted[j]
		if v >= limit && i < len(h.buckets)-1 {
			i++
			h.buckets[i].start = limit
			limit = s.min + float64(i+1)*(rnge/float64(n))
			continue
		}
		h.buckets[i].count++
		j++
	}
	return h
}

const histBlocks = 70

func (h *Hist) String() string {
	// TODO: if the range is large, expand the bucketsize and start/end a bit to get integer boundaries.
	labels := make([]string, len(h.buckets))
	labelSpaceBefore := 0
	labelSpaceAfter := 0
	var maxCount, sum float64
	for i, b := range h.buckets {
		sum += float64(b.count)
		s := "<"
		if i == len(h.buckets)-1 {
			s = "≤"
		}
		label := fmt.Sprintf("%.3g ≤ x %s %.3g", b.start, s, b.start+h.bucketSize)
		xPos := runeIndex(label, 'x')
		if xPos > labelSpaceBefore {
			labelSpaceBefore = xPos
		}
		if after := runeLen(label) - xPos - 1; after > labelSpaceAfter {
			labelSpaceAfter = after
		}
		labels[i] = label
		if f := float64(b.count); f > maxCount {
			maxCount = f
		}
	}

	var buf bytes.Buffer
	for i, b := range h.buckets {
		xPos := runeIndex(labels[i], 'x')
		before := labelSpaceBefore - xPos
		after := labelSpaceAfter - runeLen(labels[i]) + xPos + 1
		fmt.Fprintf(&buf, " %*s%s%*s │", before, "", labels[i], after, "")
		fmt.Fprint(&buf, makeBar((float64(b.count)/float64(maxCount))*histBlocks))
		fmt.Fprintf(&buf, " %d (%.3f%%)\n", b.count, 100*float64(b.count)/sum)
	}
	b := buf.Bytes()
	return string(b[:len(b)-1]) // drop the \n
}

func runeLen(s string) int { return len([]rune(s)) }

func runeIndex(s string, r rune) int {
	for i, r2 := range []rune(s) {
		if r2 == r {
			return i
		}
	}
	return -1
}

var barEighths = [9]rune{
	' ', // empty
	'▏',
	'▎',
	'▍',
	'▌',
	'▋',
	'▊',
	'▉',
	'█', // full
}

func makeBar(n float64) string {
	eighths := round(n * 8)
	full := eighths / 8
	rem := eighths % 8
	return strings.Repeat(string(barEighths[8]), full) + string(barEighths[rem])
}
<|endoftext|>


<commit_before>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package elastigo

import (
	"bytes"
	"crypto/rand"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"log"
	"strconv"
	"testing"
	"time"

	"github.com/araddon/gou"
	"github.com/bmizerany/assert"
)

//  go test -bench=".*"
//  go test -bench="Bulk"

func init() {
	flag.Parse()
	if testing.Verbose() {
		gou.SetupLogging("debug")
	}
}

// take two ints, compare, need to be within 5%
func closeInt(a, b int) bool {
	c := float64(a) / float64(b)
	if c >= .95 && c <= 1.05 {
		return true
	}
	return false
}

func TestBulkIndexerBasic(t *testing.T) {
	testIndex := "users"
	var (
		buffers        = make([]*bytes.Buffer, 0)
		totalBytesSent int
		messageSets    int
	)

	InitTests(true)
	c := NewTestConn()

	c.DeleteIndex(testIndex)

	indexer := c.NewBulkIndexer(3)
	indexer.BufferDelayMax = time.Second
	indexer.Sender = func(buf *bytes.Buffer) error {
		messageSets += 1
		totalBytesSent += buf.Len()
		buffers = append(buffers, buf)
		// log.Printf("buffer:%s", string(buf.Bytes()))
		return indexer.Send(buf)
	}
	indexer.Start()

	date := time.Unix(1257894000, 0)
	data := map[string]interface{}{
		"name": "smurfs",
		"age":  22,
		"date": "yesterday",
	}

	if err := indexer.Index(testIndex, "user", "1", "", &date, data, true); err != nil {
		t.Fatal(err)
	}

	waitFor(func() bool {
		return len(buffers) > 0
	}, 2)
	// part of request is url, so lets factor that in
	//totalBytesSent = totalBytesSent - len(*eshost)
	assert.T(t, len(buffers) == 1, fmt.Sprintf("Should have sent one operation but was %d", len(buffers)))
	assert.T(t, indexer.NumErrors() == 0, fmt.Sprintf("Should not have any errors. NumErrors: %v", indexer.NumErrors()))
	expectedBytes := 144
	assert.T(t, totalBytesSent == expectedBytes, fmt.Sprintf("Should have sent %v bytes but was %v", expectedBytes, totalBytesSent))

	if err := indexer.Index(testIndex, "user", "2", "", nil, data, true); err != nil {
		t.Fatal(err)
	}
	<-time.After(time.Millisecond * 10) // we need to wait for doc to hit send channel
	if err := indexer.Stop(); err != nil {
		t.Fatal(err)
	}
	totalBytesSent = totalBytesSent - len(*eshost)
	assert.T(t, len(buffers) == 2, fmt.Sprintf("Should have another buffer ct=%d", len(buffers)))

	assert.T(t, indexer.NumErrors() == 0, fmt.Sprintf("Should not have any errors %d", indexer.NumErrors()))
	expectedBytes = 250 // with refresh
	assert.T(t, closeInt(totalBytesSent, expectedBytes), fmt.Sprintf("Should have sent %v bytes but was %v", expectedBytes, totalBytesSent))
}

// currently broken in drone.io
func XXXTestBulkUpdate(t *testing.T) {
	var (
		buffers        = make([]*bytes.Buffer, 0)
		totalBytesSent int
		messageSets    int
	)

	InitTests(true)
	c := NewTestConn()
	c.Port = "9200"
	indexer := c.NewBulkIndexer(3)
	indexer.Sender = func(buf *bytes.Buffer) error {
		messageSets += 1
		totalBytesSent += buf.Len()
		buffers = append(buffers, buf)
		return indexer.Send(buf)
	}
	indexer.Start()

	date := time.Unix(1257894000, 0)
	user := map[string]interface{}{
		"name": "smurfs", "age": 22, "date": date, "count": 1,
	}

	// Lets make sure the data is in the index ...
	if _, err := c.Index("users", "user", "5", nil, user); err != nil {
		t.Fatal(err)
	}

	// script and params
	data := map[string]interface{}{
		"script": "ctx._source.count += 2",
	}
	if err := indexer.Update("users", "user", "5", "", &date, data, true); err != nil {
		t.Fatal(err)
	}
	// So here's the deal. Flushing does seem to work, you just have to give the
	// channel a moment to recieve the message ...
	//	<- time.After(time.Millisecond * 20)
	//	indexer.Flush()

	waitFor(func() bool {
		return len(buffers) > 0
	}, 5)

	if err := indexer.Stop(); err != nil {
		t.Fatal(err)
	}

	assert.T(t, indexer.NumErrors() == 0, fmt.Sprintf("Should not have any errors, bulkErrorCt:%v", indexer.NumErrors()))

	response, err := c.Get("users", "user", "5", nil)
	assert.T(t, err == nil, fmt.Sprintf("Should not have any errors  %v", err))
	m := make(map[string]interface{})
	json.Unmarshal([]byte(*response.Source), &m)
	newCount := m["count"]
	assert.T(t, newCount.(float64) == 3,
		fmt.Sprintf("Should have update count: %#v ... %#v", m["count"], response))
}

func TestBulkSmallBatch(t *testing.T) {
	var (
		messageSets int
	)

	InitTests(true)
	c := NewTestConn()

	date := time.Unix(1257894000, 0)
	data := map[string]interface{}{"name": "smurfs", "age": 22, "date": date}

	// Now tests small batches
	indexer := c.NewBulkIndexer(1)
	indexer.BufferDelayMax = 100 * time.Millisecond
	indexer.BulkMaxDocs = 2
	messageSets = 0
	indexer.Sender = func(buf *bytes.Buffer) error {
		messageSets += 1
		return indexer.Send(buf)
	}
	indexer.Start()
	<-time.After(time.Millisecond * 20)

	indexer.Index("users", "user", "2", "", &date, data, true)
	indexer.Index("users", "user", "3", "", &date, data, true)
	indexer.Index("users", "user", "4", "", &date, data, true)
	<-time.After(time.Millisecond * 200)
	//	indexer.Flush()
	if err := indexer.Stop(); err != nil {
		t.Fatal(err)
	}
	assert.T(t, messageSets == 2, fmt.Sprintf("Should have sent 2 message sets %d", messageSets))

}

func TestBulkDelete(t *testing.T) {
	InitTests(true)

	c := NewTestConn()
	indexer := c.NewBulkIndexer(1)
	sentBytes := []byte{}

	indexer.Sender = func(buf *bytes.Buffer) error {
		sentBytes = append(sentBytes, buf.Bytes()...)
		return nil
	}

	indexer.Start()

	indexer.Delete("fake", "fake_type", "1", true)

	indexer.Flush()
	if err := indexer.Stop(); err != nil {
		t.Fatal(err)
	}

	sent := string(sentBytes)

	expected := `{"delete":{"_index":"fake","_type":"fake_type","_id":"1","refresh":true}}
`
	asExpected := sent == expected
	assert.T(t, asExpected, fmt.Sprintf("Should have sent '%s' but actually sent '%s'", expected, sent))
}

func TestBulkErrors(t *testing.T) {
	c := NewTestConn()
	indexer := c.NewBulkIndexerRetry(10, 1)
	indexer.Sender = func(_ *bytes.Buffer) error {
		return errors.New("FAIL")
	}
	indexer.Start()
	for i := 0; i < 20; i++ {
		date := time.Unix(1257894000, 0)
		data := map[string]interface{}{"name": "smurfs", "age": 22, "date": date}
		indexer.Index("users", "user", strconv.Itoa(i), "", &date, data, true)
	}
	err := indexer.Stop()
	assert.NotEqual(t, nil, err, fmt.Sprintf("error should not be nil"))
	assert.Equal(t, "FAIL", err.Error(), "error should be the expected one")
}

/*
BenchmarkSend	18:33:00 bulk_test.go:131: Sent 1 messages in 0 sets totaling 0 bytes
18:33:00 bulk_test.go:131: Sent 100 messages in 1 sets totaling 145889 bytes
18:33:01 bulk_test.go:131: Sent 10000 messages in 100 sets totaling 14608888 bytes
18:33:05 bulk_test.go:131: Sent 20000 messages in 99 sets totaling 14462790 bytes
   20000	    234526 ns/op

*/
func BenchmarkSend(b *testing.B) {
	InitTests(true)
	c := NewTestConn()
	b.StartTimer()
	totalBytes := 0
	sets := 0
	indexer := c.NewBulkIndexer(1)
	indexer.Sender = func(buf *bytes.Buffer) error {
		totalBytes += buf.Len()
		sets += 1
		//log.Println("got bulk")
		return indexer.Send(buf)
	}
	for i := 0; i < b.N; i++ {
		about := make([]byte, 1000)
		rand.Read(about)
		data := map[string]interface{}{"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0), "about": about}
		indexer.Index("users", "user", strconv.Itoa(i), "", nil, data, true)
	}
	log.Printf("Sent %d messages in %d sets totaling %d bytes \n", b.N, sets, totalBytes)
	if indexer.NumErrors() != 0 {
		b.Fail()
	}
}

/*
TODO:  this should be faster than above

BenchmarkSendBytes	18:33:05 bulk_test.go:169: Sent 1 messages in 0 sets totaling 0 bytes
18:33:05 bulk_test.go:169: Sent 100 messages in 2 sets totaling 292299 bytes
18:33:09 bulk_test.go:169: Sent 10000 messages in 99 sets totaling 14473800 bytes
   10000	    373529 ns/op

*/
func BenchmarkSendBytes(b *testing.B) {
	InitTests(true)
	c := NewTestConn()
	about := make([]byte, 1000)
	rand.Read(about)
	data := map[string]interface{}{"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0), "about": about}
	body, _ := json.Marshal(data)
	b.StartTimer()
	totalBytes := 0
	sets := 0
	indexer := c.NewBulkIndexer(1)
	indexer.Sender = func(buf *bytes.Buffer) error {
		totalBytes += buf.Len()
		sets += 1
		return indexer.Send(buf)
	}
	for i := 0; i < b.N; i++ {
		indexer.Index("users", "user", strconv.Itoa(i), "", nil, body, true)
	}
	log.Printf("Sent %d messages in %d sets totaling %d bytes \n", b.N, sets, totalBytes)
	if indexer.NumErrors() != 0 {
		b.Fail()
	}
}
<commit_msg>fix all races for go test -run TestBulk -race<commit_after>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package elastigo

import (
	"bytes"
	"crypto/rand"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"log"
	"strconv"
	"sync"
	"testing"
	"time"

	"github.com/araddon/gou"
	"github.com/bmizerany/assert"
)

//  go test -bench=".*"
//  go test -bench="Bulk"

func init() {
	flag.Parse()
	if testing.Verbose() {
		gou.SetupLogging("debug")
	}
}

// take two ints, compare, need to be within 5%
func closeInt(a, b int) bool {
	c := float64(a) / float64(b)
	if c >= .95 && c <= 1.05 {
		return true
	}
	return false
}

func TestBulkIndexerBasic(t *testing.T) {
	testIndex := "users"
	var (
		mu             sync.Mutex // guards following fields
		buffers        = make([]*bytes.Buffer, 0)
		totalBytesSent int
		messageSets    int
	)

	InitTests(true)
	c := NewTestConn()

	c.DeleteIndex(testIndex)

	indexer := c.NewBulkIndexer(3)
	indexer.BufferDelayMax = time.Second
	indexer.Sender = func(buf *bytes.Buffer) error {
		mu.Lock()
		messageSets++
		totalBytesSent += buf.Len()
		buffers = append(buffers, buf)
		mu.Unlock()
		// log.Printf("buffer:%s", string(buf.Bytes()))
		return indexer.Send(buf)
	}
	indexer.Start()

	date := time.Unix(1257894000, 0)
	data := map[string]interface{}{
		"name": "smurfs",
		"age":  22,
		"date": "yesterday",
	}

	if err := indexer.Index(testIndex, "user", "1", "", &date, data, true); err != nil {
		t.Fatal(err)
	}

	waitFor(func() bool {
		mu.Lock()
		defer mu.Unlock()
		return len(buffers) > 0
	}, 2)
	// part of request is url, so lets factor that in
	//totalBytesSent = totalBytesSent - len(*eshost)
	assert.T(t, len(buffers) == 1, fmt.Sprintf("Should have sent one operation but was %d", len(buffers)))
	assert.T(t, indexer.NumErrors() == 0, fmt.Sprintf("Should not have any errors. NumErrors: %v", indexer.NumErrors()))
	expectedBytes := 144
	assert.T(t, totalBytesSent == expectedBytes, fmt.Sprintf("Should have sent %v bytes but was %v", expectedBytes, totalBytesSent))

	if err := indexer.Index(testIndex, "user", "2", "", nil, data, true); err != nil {
		t.Fatal(err)
	}
	<-time.After(time.Millisecond * 10) // we need to wait for doc to hit send channel
	if err := indexer.Stop(); err != nil {
		t.Fatal(err)
	}
	totalBytesSent = totalBytesSent - len(*eshost)
	assert.T(t, len(buffers) == 2, fmt.Sprintf("Should have another buffer ct=%d", len(buffers)))

	assert.T(t, indexer.NumErrors() == 0, fmt.Sprintf("Should not have any errors %d", indexer.NumErrors()))
	expectedBytes = 250 // with refresh
	assert.T(t, closeInt(totalBytesSent, expectedBytes), fmt.Sprintf("Should have sent %v bytes but was %v", expectedBytes, totalBytesSent))
}

// currently broken in drone.io
func XXXTestBulkUpdate(t *testing.T) {
	var (
		buffers        = make([]*bytes.Buffer, 0)
		totalBytesSent int
		messageSets    int
	)

	InitTests(true)
	c := NewTestConn()
	c.Port = "9200"
	indexer := c.NewBulkIndexer(3)
	indexer.Sender = func(buf *bytes.Buffer) error {
		messageSets += 1
		totalBytesSent += buf.Len()
		buffers = append(buffers, buf)
		return indexer.Send(buf)
	}
	indexer.Start()

	date := time.Unix(1257894000, 0)
	user := map[string]interface{}{
		"name": "smurfs", "age": 22, "date": date, "count": 1,
	}

	// Lets make sure the data is in the index ...
	if _, err := c.Index("users", "user", "5", nil, user); err != nil {
		t.Fatal(err)
	}

	// script and params
	data := map[string]interface{}{
		"script": "ctx._source.count += 2",
	}
	if err := indexer.Update("users", "user", "5", "", &date, data, true); err != nil {
		t.Fatal(err)
	}
	// So here's the deal. Flushing does seem to work, you just have to give the
	// channel a moment to recieve the message ...
	//	<- time.After(time.Millisecond * 20)
	//	indexer.Flush()

	waitFor(func() bool {
		return len(buffers) > 0
	}, 5)

	if err := indexer.Stop(); err != nil {
		t.Fatal(err)
	}

	assert.T(t, indexer.NumErrors() == 0, fmt.Sprintf("Should not have any errors, bulkErrorCt:%v", indexer.NumErrors()))

	response, err := c.Get("users", "user", "5", nil)
	assert.T(t, err == nil, fmt.Sprintf("Should not have any errors  %v", err))
	m := make(map[string]interface{})
	json.Unmarshal([]byte(*response.Source), &m)
	newCount := m["count"]
	assert.T(t, newCount.(float64) == 3,
		fmt.Sprintf("Should have update count: %#v ... %#v", m["count"], response))
}

func TestBulkSmallBatch(t *testing.T) {
	var (
		messageSets int
	)

	InitTests(true)
	c := NewTestConn()

	date := time.Unix(1257894000, 0)
	data := map[string]interface{}{"name": "smurfs", "age": 22, "date": date}

	// Now tests small batches
	indexer := c.NewBulkIndexer(1)
	indexer.BufferDelayMax = 100 * time.Millisecond
	indexer.BulkMaxDocs = 2
	messageSets = 0
	indexer.Sender = func(buf *bytes.Buffer) error {
		messageSets += 1
		return indexer.Send(buf)
	}
	indexer.Start()
	<-time.After(time.Millisecond * 20)

	indexer.Index("users", "user", "2", "", &date, data, true)
	indexer.Index("users", "user", "3", "", &date, data, true)
	indexer.Index("users", "user", "4", "", &date, data, true)
	<-time.After(time.Millisecond * 200)
	//	indexer.Flush()
	if err := indexer.Stop(); err != nil {
		t.Fatal(err)
	}
	assert.T(t, messageSets == 2, fmt.Sprintf("Should have sent 2 message sets %d", messageSets))

}

func TestBulkDelete(t *testing.T) {
	InitTests(true)

	c := NewTestConn()
	indexer := c.NewBulkIndexer(1)
	sentBytes := []byte{}

	indexer.Sender = func(buf *bytes.Buffer) error {
		sentBytes = append(sentBytes, buf.Bytes()...)
		return nil
	}

	indexer.Start()

	indexer.Delete("fake", "fake_type", "1", true)

	indexer.Flush()
	if err := indexer.Stop(); err != nil {
		t.Fatal(err)
	}

	sent := string(sentBytes)

	expected := `{"delete":{"_index":"fake","_type":"fake_type","_id":"1","refresh":true}}
`
	asExpected := sent == expected
	assert.T(t, asExpected, fmt.Sprintf("Should have sent '%s' but actually sent '%s'", expected, sent))
}

func TestBulkErrors(t *testing.T) {
	c := NewTestConn()
	indexer := c.NewBulkIndexerRetry(10, 1)
	indexer.Sender = func(_ *bytes.Buffer) error {
		return errors.New("FAIL")
	}
	indexer.Start()
	for i := 0; i < 20; i++ {
		date := time.Unix(1257894000, 0)
		data := map[string]interface{}{"name": "smurfs", "age": 22, "date": date}
		indexer.Index("users", "user", strconv.Itoa(i), "", &date, data, true)
	}
	err := indexer.Stop()
	assert.NotEqual(t, nil, err, fmt.Sprintf("error should not be nil"))
	assert.Equal(t, "FAIL", err.Error(), "error should be the expected one")
}

/*
BenchmarkSend	18:33:00 bulk_test.go:131: Sent 1 messages in 0 sets totaling 0 bytes
18:33:00 bulk_test.go:131: Sent 100 messages in 1 sets totaling 145889 bytes
18:33:01 bulk_test.go:131: Sent 10000 messages in 100 sets totaling 14608888 bytes
18:33:05 bulk_test.go:131: Sent 20000 messages in 99 sets totaling 14462790 bytes
   20000	    234526 ns/op

*/
func BenchmarkSend(b *testing.B) {
	InitTests(true)
	c := NewTestConn()
	b.StartTimer()
	totalBytes := 0
	sets := 0
	indexer := c.NewBulkIndexer(1)
	indexer.Sender = func(buf *bytes.Buffer) error {
		totalBytes += buf.Len()
		sets += 1
		//log.Println("got bulk")
		return indexer.Send(buf)
	}
	for i := 0; i < b.N; i++ {
		about := make([]byte, 1000)
		rand.Read(about)
		data := map[string]interface{}{"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0), "about": about}
		indexer.Index("users", "user", strconv.Itoa(i), "", nil, data, true)
	}
	log.Printf("Sent %d messages in %d sets totaling %d bytes \n", b.N, sets, totalBytes)
	if indexer.NumErrors() != 0 {
		b.Fail()
	}
}

/*
TODO:  this should be faster than above

BenchmarkSendBytes	18:33:05 bulk_test.go:169: Sent 1 messages in 0 sets totaling 0 bytes
18:33:05 bulk_test.go:169: Sent 100 messages in 2 sets totaling 292299 bytes
18:33:09 bulk_test.go:169: Sent 10000 messages in 99 sets totaling 14473800 bytes
   10000	    373529 ns/op

*/
func BenchmarkSendBytes(b *testing.B) {
	InitTests(true)
	c := NewTestConn()
	about := make([]byte, 1000)
	rand.Read(about)
	data := map[string]interface{}{"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0), "about": about}
	body, _ := json.Marshal(data)
	b.StartTimer()
	totalBytes := 0
	sets := 0
	indexer := c.NewBulkIndexer(1)
	indexer.Sender = func(buf *bytes.Buffer) error {
		totalBytes += buf.Len()
		sets += 1
		return indexer.Send(buf)
	}
	for i := 0; i < b.N; i++ {
		indexer.Index("users", "user", strconv.Itoa(i), "", nil, body, true)
	}
	log.Printf("Sent %d messages in %d sets totaling %d bytes \n", b.N, sets, totalBytes)
	if indexer.NumErrors() != 0 {
		b.Fail()
	}
}
<|endoftext|>


<commit_before>package forecast

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"math"
	"net/http"
	"regexp"
	"time"

	"github.com/jteeuwen/go-pkg-xmlx"
	"golang.org/x/net/html/charset"
)

const (
	fcUrlFmt    = "http://forecast.weather.gov/MapClick.php?lat=%s&lon=%s&FcstType=digitalDWML"
	locUrlFmt   = "http://nominatim.openstreetmap.org/search?format=json&limit=1&q=%s"
	arrows      = "↑↗→↘↓↙←↖"
	firstOctile = 0x2581
	maxHours    = 48
)

var (
	errFc  = errors.New("Error retrieving forecast.")
	errLoc = errors.New("I had a problem finding that location.")
)

type location struct {
	Name string `json:"display_name"`
	Lat  string `json:"lat"`
	Lon  string `json:"lon"`
}

func Forecast(text string) (string, error) {
	loc := regexp.MustCompile("\\s+").ReplaceAllLiteralString(text, "+")
	resp, err := http.Get(fmt.Sprintf(locUrlFmt, loc))
	if err != nil {
		return "", errLoc
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return "", errLoc
	}
	dec := json.NewDecoder(resp.Body)
	locs := []location{}
	dec.Decode(&locs)
	if len(locs) == 0 {
		return "", errLoc
	}
	fc, err := forecast(locs[0])
	if err != nil {
		return "", errFc
	}
	return fc, nil
}

func forecast(loc location) (string, error) {
	doc := xmlx.New()
	url := fmt.Sprintf(fcUrlFmt, loc.Lat, loc.Lon)
	err := doc.LoadUri(url, func(str string, rdr io.Reader) (io.Reader, error) {
		return charset.NewReader(rdr, str)
	})
	if err != nil {
		return "", errFc
	}

	startTimeNodes := doc.SelectNodes("", "start-valid-time")
	endTimeNodes := doc.SelectNodes("", "end-valid-time")
	if len(startTimeNodes) == 0 || len(endTimeNodes) == 0 {
		return "", errFc
	}
	if len(endTimeNodes) > maxHours {
		endTimeNodes = endTimeNodes[:maxHours]
	}
	startTime, _ := time.Parse(time.RFC3339, startTimeNodes[0].Value)
	endTime, _ := time.Parse(time.RFC3339, endTimeNodes[len(endTimeNodes)-1].Value)

	temps := findVals("temperature", "hourly", doc)
	humids := findVals("humidity", "", doc)
	precips := findVals("probability-of-precipitation", "", doc)
	speeds := findVals("wind-speed", "sustained", doc)
	dirs := findVals("direction", "", doc)

	minTemp, maxTemp, tempGraph := makeGraph(temps)
	minHumid, maxHumid, humidGraph := makeGraph(humids)
	minPrecip, maxPrecip, precipGraph := makeGraph(precips)
	minSpeed, maxSpeed, speedGraph := makeGraph(speeds)

	dirGraph := ""
	for _, dir := range dirs {
		idx := dirIndex(dir)
		dirGraph += string([]rune(arrows)[idx])
	}

	timeFmt := "2006-01-02 15:04"
	start, end := startTime.Format(timeFmt), endTime.Format(timeFmt)

	tempRange := fmt.Sprintf("%3d %3d", minTemp, maxTemp)
	humidRange := fmt.Sprintf("%3d %3d", minHumid, maxHumid)
	precipRange := fmt.Sprintf("%3d %3d", minPrecip, maxPrecip)
	speedRange := fmt.Sprintf("%3d %3d", minSpeed, maxSpeed)

	out := fmt.Sprintf("Forecast for %s\n", loc.Name)
	out += fmt.Sprintf("         min max %-24s%24s\n", start, end)
	out += fmt.Sprintf("Temp °F  %7s %s\n", tempRange, tempGraph)
	out += fmt.Sprintf("Humid %%  %7s %s\n", humidRange, humidGraph) // esc % 2X for later fmt use
	out += fmt.Sprintf("Precip %% %7s %s\n", precipRange, precipGraph)
	out += fmt.Sprintf("Wind mph %7s %s\n", speedRange, speedGraph)
	out += fmt.Sprintf("Wind dir         %s\n", dirGraph)

	return out, nil
}

func minmax(vals []int) (int, int) {
	min, max := math.MaxInt32, math.MinInt32
	for _, v := range vals {
		if v < min {
			min = v
		}
		if v > max {
			max = v
		}
	}
	return min, max
}

func rescale(val, min, max, bins int) int {
	if min >= max {
		return 0
	}
	v := (val - min) * bins / (max - min)
	if v < 0 {
		v = 0
	} else if v > bins-1 {
		v = bins - 1
	}
	return v
}

func dirIndex(dir int) int {
	return ((dir + 360/16) * 8 / 360) % 8
}

func findVals(name, typ string, doc *xmlx.Document) []int {
	vals := []int{}
	nodes := doc.SelectNodes("", name)
	for _, node := range nodes {
		if typ == "" || node.As("", "type") == typ {
			for _, kid := range node.Children {
				vals = append(vals, kid.I("", "value"))
				if len(vals) >= maxHours {
					break
				}
			}
			break // just use the first set
		}
	}
	return vals
}

func makeGraph(vals []int) (int, int, string) {
	if len(vals) == 0 {
		return 0, 0, ""
	}
	graph := ""
	min, max := minmax(vals)
	for _, val := range vals {
		octile := rescale(val, min, max, 8)
		graph += string(firstOctile + octile)
	}
	return min, max, graph
}
<commit_msg>Fixed forecast time extraction.<commit_after>package forecast

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"math"
	"net/http"
	"regexp"
	"time"

	"github.com/jteeuwen/go-pkg-xmlx"
	"golang.org/x/net/html/charset"
)

const (
	fcUrlFmt    = "http://forecast.weather.gov/MapClick.php?lat=%s&lon=%s&FcstType=digitalDWML"
	locUrlFmt   = "http://nominatim.openstreetmap.org/search?format=json&limit=1&q=%s"
	arrows      = "↑↗→↘↓↙←↖"
	firstOctile = 0x2581
	maxHours    = 48
)

var (
	errFc  = errors.New("Error retrieving forecast.")
	errLoc = errors.New("I had a problem finding that location.")
)

type location struct {
	Name string `json:"display_name"`
	Lat  string `json:"lat"`
	Lon  string `json:"lon"`
}

func Forecast(text string) (string, error) {
	loc := regexp.MustCompile("\\s+").ReplaceAllLiteralString(text, "+")
	resp, err := http.Get(fmt.Sprintf(locUrlFmt, loc))
	if err != nil {
		return "", errLoc
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return "", errLoc
	}
	dec := json.NewDecoder(resp.Body)
	locs := []location{}
	dec.Decode(&locs)
	if len(locs) == 0 {
		return "", errLoc
	}
	fc, err := forecast(locs[0])
	if err != nil {
		return "", errFc
	}
	return fc, nil
}

func forecast(loc location) (string, error) {
	doc := xmlx.New()
	url := fmt.Sprintf(fcUrlFmt, loc.Lat, loc.Lon)
	err := doc.LoadUri(url, func(str string, rdr io.Reader) (io.Reader, error) {
		return charset.NewReader(rdr, str)
	})
	if err != nil {
		return "", errFc
	}

	startTimeNodes := doc.SelectNodes("", "start-valid-time")
	endTimeNodes := doc.SelectNodes("", "end-valid-time")
	if len(startTimeNodes) == 0 || len(endTimeNodes) == 0 {
		return "", errFc
	}
	if len(endTimeNodes) > maxHours {
		endTimeNodes = endTimeNodes[:maxHours]
	}
	startTime, _ := time.Parse(time.RFC3339, startTimeNodes[0].GetValue())
	endTime, _ := time.Parse(time.RFC3339, endTimeNodes[len(endTimeNodes)-1].GetValue())

	temps := findVals("temperature", "hourly", doc)
	humids := findVals("humidity", "", doc)
	precips := findVals("probability-of-precipitation", "", doc)
	speeds := findVals("wind-speed", "sustained", doc)
	dirs := findVals("direction", "", doc)

	minTemp, maxTemp, tempGraph := makeGraph(temps)
	minHumid, maxHumid, humidGraph := makeGraph(humids)
	minPrecip, maxPrecip, precipGraph := makeGraph(precips)
	minSpeed, maxSpeed, speedGraph := makeGraph(speeds)

	dirGraph := ""
	for _, dir := range dirs {
		idx := dirIndex(dir)
		dirGraph += string([]rune(arrows)[idx])
	}

	timeFmt := "2006-01-02 15:04"
	start, end := startTime.Format(timeFmt), endTime.Format(timeFmt)

	tempRange := fmt.Sprintf("%3d %3d", minTemp, maxTemp)
	humidRange := fmt.Sprintf("%3d %3d", minHumid, maxHumid)
	precipRange := fmt.Sprintf("%3d %3d", minPrecip, maxPrecip)
	speedRange := fmt.Sprintf("%3d %3d", minSpeed, maxSpeed)

	out := fmt.Sprintf("Forecast for %s\n", loc.Name)
	out += fmt.Sprintf("         min max %-24s%24s\n", start, end)
	out += fmt.Sprintf("Temp °F  %7s %s\n", tempRange, tempGraph)
	out += fmt.Sprintf("Humid %%  %7s %s\n", humidRange, humidGraph) // esc % 2X for later fmt use
	out += fmt.Sprintf("Precip %% %7s %s\n", precipRange, precipGraph)
	out += fmt.Sprintf("Wind mph %7s %s\n", speedRange, speedGraph)
	out += fmt.Sprintf("Wind dir         %s\n", dirGraph)

	return out, nil
}

func minmax(vals []int) (int, int) {
	min, max := math.MaxInt32, math.MinInt32
	for _, v := range vals {
		if v < min {
			min = v
		}
		if v > max {
			max = v
		}
	}
	return min, max
}

func rescale(val, min, max, bins int) int {
	if min >= max {
		return 0
	}
	v := (val - min) * bins / (max - min)
	if v < 0 {
		v = 0
	} else if v > bins-1 {
		v = bins - 1
	}
	return v
}

func dirIndex(dir int) int {
	return ((dir + 360/16) * 8 / 360) % 8
}

func findVals(name, typ string, doc *xmlx.Document) []int {
	vals := []int{}
	nodes := doc.SelectNodes("", name)
	for _, node := range nodes {
		if typ == "" || node.As("", "type") == typ {
			for _, kid := range node.Children {
				vals = append(vals, kid.I("", "value"))
				if len(vals) >= maxHours {
					break
				}
			}
			break // just use the first set
		}
	}
	return vals
}

func makeGraph(vals []int) (int, int, string) {
	if len(vals) == 0 {
		return 0, 0, ""
	}
	graph := ""
	min, max := minmax(vals)
	for _, val := range vals {
		octile := rescale(val, min, max, 8)
		graph += string(firstOctile + octile)
	}
	return min, max, graph
}
<|endoftext|>


<commit_before>package mobi

import (
	"crypto/sha1"
	"fmt"
	"image"
	"io"
	"os"
	"path/filepath"

	"github.com/geek1011/BookBrowser/booklist"
	"github.com/geek1011/BookBrowser/formats"
	"github.com/pkg/errors"
)

type mobi struct {
	book *booklist.Book
}

func (e *mobi) Book() *booklist.Book {
	return e.book
}

func (e *mobi) HasCover() bool {
	return false
}

func (e *mobi) GetCover() (i image.Image, err error) {
	return nil, errors.New("no cover")
}

func load(filename string) (bi formats.BookInfo, ferr error) {
	defer func() {
		if r := recover(); r != nil {
			bi = nil
			ferr = fmt.Errorf("unknown error: %s", r)
		}
	}()

	m := &mobi{book: &booklist.Book{}}

	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}

	fi, err := f.Stat()
	if err != nil {
		f.Close()
		return nil, errors.Wrapf(err, "could not stat book")
	}
	m.book.FilePath = filename
	m.book.FileSize = fi.Size()
	m.book.ModTime = fi.ModTime()

	s := sha1.New()
	i, err := io.Copy(s, f)
	if err == nil && i != fi.Size() {
		err = errors.New("could not read whole file")
	}
	if err != nil {
		f.Close()
		return nil, errors.Wrap(err, "could not hash book")
	}
	m.book.Hash = fmt.Sprintf("%x", s.Sum(nil))

	f.Close()

	m.book.Title = filepath.Base(filename)

	return m, nil
}

func init() {
	formats.Register("mobi", load)
}
<commit_msg>Added full support for reading metadata from .mobi files<commit_after>package mobi

import (
	"crypto/sha1"
	"fmt"
	"image"
	"io"
	"os"
	"path/filepath"

	"github.com/geek1011/BookBrowser/booklist"
	"github.com/geek1011/BookBrowser/formats"
	"github.com/pkg/errors"

	mobirdr "github.com/sblinch/mobi"
		"encoding/binary"
)

type mobi struct {
	book *booklist.Book
	coverstart int64
	coverend int64
}

func (e *mobi) Book() *booklist.Book {
	return e.book
}

func (e *mobi) HasCover() bool {
	return e.coverstart > 0
}

func (e *mobi) GetCover() (i image.Image, err error) {
	if !e.HasCover() {
		return nil, errors.New("no cover")
	}

	f, err := os.Open(e.book.FilePath)
	if err != nil {
		return nil, errors.Wrap(err, "unable to open book file")
	}
	defer f.Close()

	if _, err := f.Seek(e.coverstart, 0); err != nil {
		return nil, errors.Wrap(err, "unable to see to cover offset")
	}

	if i, _, err = image.Decode(f); err != nil {
		return nil, errors.Wrap(err, "unable to decode book cover")
	}

	return i, nil
}

func load(filename string) (bi formats.BookInfo, ferr error) {
	defer func() {
		if r := recover(); r != nil {
			bi = nil
			ferr = fmt.Errorf("unknown error: %s", r)
		}
	}()

	m := &mobi{book: &booklist.Book{}}

	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}

	fi, err := f.Stat()
	if err != nil {
		f.Close()
		return nil, errors.Wrapf(err, "could not stat book")
	}
	m.book.FilePath = filename
	m.book.FileSize = fi.Size()
	m.book.ModTime = fi.ModTime()

	s := sha1.New()
	i, err := io.Copy(s, f)
	if err == nil && i != fi.Size() {
		err = errors.New("could not read whole file")
	}
	if err != nil {
		f.Close()
		return nil, errors.Wrap(err, "could not hash book")
	}
	m.book.Hash = fmt.Sprintf("%x", s.Sum(nil))
	f.Close()

	r, err := mobirdr.NewReader(filename)
	if err != nil {
		return nil, err
	}

	for _, rec := range r.Exth.Records {
		switch(rec.RecordType) {
		case mobirdr.EXTH_COVEROFFSET:
			v := binary.BigEndian.Uint32([]byte(rec.Value))
			coverPDBOffset := r.Header.FirstImageIndex + v

			n := int(coverPDBOffset)
			if n <= int(r.Pdf.RecordsNum)-1 {
				if n+1 < int(r.Pdf.RecordsNum) {
					m.coverend = int64(r.Offsets[n+1].Offset)
				} else {
					m.coverend = -1
				}
				m.coverstart = int64(r.Offsets[n].Offset)
			}

		case mobirdr.EXTH_TITLE:
			if len(m.book.Title)==0 {
				m.book.Title = string(rec.Value)
			}
		case mobirdr.EXTH_UPDATEDTITLE:
			if len(m.book.Title)==0 {
				m.book.Title = string(rec.Value)
			}
		case mobirdr.EXTH_AUTHOR:
			if len(m.book.Author) == 0 {
				m.book.Author = string(rec.Value)
			}
		case mobirdr.EXTH_DESCRIPTION:
			if len(m.book.Description) == 0 {
				m.book.Description = string(rec.Value)
			}
		case mobirdr.EXTH_PUBLISHER:
			if len(m.book.Publisher) == 0 {
				m.book.Publisher = string(rec.Value)
			}
/* // uncomment after merging ISBN/publishing date pull request :)
		case mobirdr.EXTH_ISBN:
			if len(m.book.ISBN) == 0 {
				m.book.ISBN = string(rec.Value)
			}
		case mobirdr.EXTH_PUBLISHINGDATE:
			if m.book.PublishDate.IsZero() {
				m.book.PublishDate = parsePublishDate(string(rec.Value))
			}
*/
		}
	}

	if len(m.book.Title)==0 {
		m.book.Title = filepath.Base(filename)
	}

	return m, nil
}

func init() {
	formats.Register("mobi", load)
}

/* // uncomment after merging ISBN/publishing date pull request :)
func parsePublishDate(s string) time.Time {
	// handle the various dumb decisions people make when encoding dates
	format := ""
	switch len(s) {
	case 32:
		//2012-02-13T20:20:58.175203+00:00
		format = "2006-01-02T15:04:05.000000-07:00"
	case 25:
		//2000-10-31 00:00:00-06:00
		//2009-04-19T22:00:00+00:00
		format = "2006-01-02" + string(s[10]) + "15:04:05-07:00"
	case 20:
		//2016-08-11T14:09:25Z
		format = "2006-01-02T15:04:05Z"
	case 19:
		//2008-01-28T07:00:00
		//2000-10-31 00:00:00
		format = "2006-01-02" + string(s[10]) + "15:04:05"
	case 10:
		//1998-07-01
		format = "2006-01-02"
	default:
		return time.Time{}
	}

	t, err := time.Parse(format,s)
	if err != nil {
		t = time.Time{}
	}
	return t
}
*/<|endoftext|>


<commit_before>package dense

import (
	"fmt"
)

func rowsShouldBePositiveNumber(rows int) {
	shouldBePositiveNumber(rows, "rows")
}

func columnShouldBePositiveNumber(columns int) {
	shouldBePositiveNumber(columns, "columns")
}

func shouldBePositiveNumber(x int, name string) {
	if x > 0 {
		return
	}

	message := fmt.Sprintf("%q should be a positive number.")
	panic(message)
}
<commit_msg>Fix: sprintf requires "name".<commit_after>package dense

import (
	"fmt"
)

func rowsShouldBePositiveNumber(rows int) {
	shouldBePositiveNumber(rows, "rows")
}

func columnShouldBePositiveNumber(columns int) {
	shouldBePositiveNumber(columns, "columns")
}

func shouldBePositiveNumber(x int, name string) {
	if x > 0 {
		return
	}

	message := fmt.Sprintf("%q should be a positive number.", name)
	panic(message)
}
<|endoftext|>


<commit_before>package rest

import (
	"errors"
	"fmt"
	"log"
	"net/http"
	"os/exec"
	"strings"

	"github.com/emicklei/go-restful"
	"github.com/emicklei/landskape/application"
	"github.com/emicklei/landskape/model"
)

var DotConfig = map[string]string{}

type DiagramResource struct {
	service application.Logic
}

func (d DiagramResource) computeDiagram(req *restful.Request, resp *restful.Response) {
	ctx := req.Request.Context()
	filter := model.ConnectionsFilter{
		Froms:   asFilterParameter(req.QueryParameter("from")),
		Tos:     asFilterParameter(req.QueryParameter("to")),
		Types:   asFilterParameter(req.QueryParameter("type")),
		Centers: asFilterParameter(req.QueryParameter("center"))}

	// TODO optimize
	// if system query parameter is given then first select all systems that match
	// and compute the Centers value of the connection filter.
	systemFilter := req.QueryParameter("system")
	if len(systemFilter) == 0 || !strings.Contains(systemFilter, ":") {
		resp.WriteError(400, errors.New("bad format system query parameter"))
		return
	}
	if len(systemFilter) > 0 {
		all, err := d.service.AllSystems(ctx)
		if err != nil {
			log.Printf("AllSystems failed:%#v", err)
			resp.WriteError(500, err)
			return
		}
		systemAttribute := model.ParseAttribute(systemFilter)
		centers := []string{}
		for _, each := range all {
			if each.HasAttribute(systemAttribute) {
				centers = append(centers, each.ID)
			}
		}
		filter.Centers = centers
	}
	// END optimize

	connections, err := d.service.AllConnections(ctx, filter)
	if err != nil {
		log.Printf("AllConnections failed:%#v", err)
		resp.WriteError(500, err)
		return
	}
	format := req.QueryParameter("format")
	if "" == format {
		format = "svg"
	}
	id, err := model.GenerateUUID()
	if err != nil {
		log.Printf("GenerateUUID failed:%v", err)
		resp.WriteError(500, err)
		return
	}
	input := fmt.Sprintf("%v/%v.dot", DotConfig["tmp"], id)
	output := fmt.Sprintf("%v/%v.%v", DotConfig["tmp"], id, format)

	dotBuilder := application.NewDotBuilder()
	dotBuilder.ClusterBy(req.QueryParameter("cluster"))
	dotBuilder.BuildFromAll(connections)

	dotOnly := req.QueryParameter("format") == "dot"
	if dotOnly {
		resp.AddHeader("Content-Type", "text/plain")
		dotBuilder.WriteDot(resp)
		return
	}
	dotBuilder.WriteDotFile(input)

	cmd := exec.Command(DotConfig["binpath"],
		fmt.Sprintf("-T%v", format),
		fmt.Sprintf("-o%v", output),
		input)
	err = cmd.Start()
	if err != nil {
		log.Printf("Dot command start failed:%v", err)
		resp.WriteError(500, err)
		return
	}
	err = cmd.Wait()
	if err != nil {
		log.Printf("Dot did not complete:%v", err)
		resp.WriteError(500, err)
		return
	}
	// resp.AddHeader("Content-Type", "image/svg+xml")
	http.ServeFile(resp, req.Request, output)
}
<commit_msg>fix system param check<commit_after>package rest

import (
	"errors"
	"fmt"
	"log"
	"net/http"
	"os/exec"
	"strings"

	"github.com/emicklei/go-restful"
	"github.com/emicklei/landskape/application"
	"github.com/emicklei/landskape/model"
)

var DotConfig = map[string]string{}

type DiagramResource struct {
	service application.Logic
}

func (d DiagramResource) computeDiagram(req *restful.Request, resp *restful.Response) {
	ctx := req.Request.Context()
	filter := model.ConnectionsFilter{
		Froms:   asFilterParameter(req.QueryParameter("from")),
		Tos:     asFilterParameter(req.QueryParameter("to")),
		Types:   asFilterParameter(req.QueryParameter("type")),
		Centers: asFilterParameter(req.QueryParameter("center"))}

	// TODO optimize
	// if system query parameter is given then first select all systems that match
	// and compute the Centers value of the connection filter.
	systemFilter := req.QueryParameter("system")
	if len(systemFilter) > 0 {
		if !strings.Contains(systemFilter, ":") {
			resp.WriteError(400, errors.New("bad format system query parameter"))
			return
		}
		all, err := d.service.AllSystems(ctx)
		if err != nil {
			log.Printf("AllSystems failed:%#v", err)
			resp.WriteError(500, err)
			return
		}
		systemAttribute := model.ParseAttribute(systemFilter)
		centers := []string{}
		for _, each := range all {
			if each.HasAttribute(systemAttribute) {
				centers = append(centers, each.ID)
			}
		}
		filter.Centers = centers
	}
	// END optimize

	connections, err := d.service.AllConnections(ctx, filter)
	if err != nil {
		log.Printf("AllConnections failed:%#v", err)
		resp.WriteError(500, err)
		return
	}
	format := req.QueryParameter("format")
	if "" == format {
		format = "svg"
	}
	id, err := model.GenerateUUID()
	if err != nil {
		log.Printf("GenerateUUID failed:%v", err)
		resp.WriteError(500, err)
		return
	}
	input := fmt.Sprintf("%v/%v.dot", DotConfig["tmp"], id)
	output := fmt.Sprintf("%v/%v.%v", DotConfig["tmp"], id, format)

	dotBuilder := application.NewDotBuilder()
	dotBuilder.ClusterBy(req.QueryParameter("cluster"))
	dotBuilder.BuildFromAll(connections)

	dotOnly := req.QueryParameter("format") == "dot"
	if dotOnly {
		resp.AddHeader("Content-Type", "text/plain")
		dotBuilder.WriteDot(resp)
		return
	}
	dotBuilder.WriteDotFile(input)

	cmd := exec.Command(DotConfig["binpath"],
		fmt.Sprintf("-T%v", format),
		fmt.Sprintf("-o%v", output),
		input)
	err = cmd.Start()
	if err != nil {
		log.Printf("Dot command start failed:%v", err)
		resp.WriteError(500, err)
		return
	}
	err = cmd.Wait()
	if err != nil {
		log.Printf("Dot did not complete:%v", err)
		resp.WriteError(500, err)
		return
	}
	// resp.AddHeader("Content-Type", "image/svg+xml")
	http.ServeFile(resp, req.Request, output)
}
<|endoftext|>


<commit_before>package flame

import (
	"image"
	"image/color"
	"image/draw"
  /*
	bin "encoding/binary"
	"fmt"
	"math/rand"
	"os"
	"io"
	"time"
  */
)

func RenderPreview(width, height, xs, ys int, fn Variation) *image.RGBA {
  return lines(width, height, matrix(xs, ys, fn))
}

func matrix(width, height int, fn Variation) [][]Point {
  data := make([][]Point, height)
	// these are our parameters
	var a, b, c, d, e, f float64
	a, b, c, d, e, f = DefaultParams()
  for y := range data {
    data[y] = make([]Point, width)
    for x := range data[y] {
      a, b := fn(frow(x, width), frow(y, height), a, b, c, d, e, f)
      data[y][x] = Point{a,b}
    }
  }
  return data
}

// width & height correspond to the output image
func lines(width, height int, data [][]Point) *image.RGBA {
  img := image.NewRGBA(image.Rect(0, 0, width, height))
  ys := len(data)
  xs := len(data[0])
  for y := range data {
    for x := range data[y] {
      // draw down
      line(width, height, Point{frow(x, xs), frow(y, ys)}, data[y][x], color.RGBA{0, 0, 0, 255}, img)
      if y < ys - 1 {
        // line(width, height, data[y][x], data[y+1][x], color.RGBA{0, 0, 0, 255}, img)
        // line(width, height, Point{frow(x, xs), frow(y, ys)}, Point{frow(x, xs), frow(y+1, ys)}, color.RGBA{0, 0, 255, 100}, img)
      }
      // draw right
      if x < xs - 1 {
        // line(width, height, data[y][x], data[y][x+1], color.RGBA{0, 0, 0, 255}, img)
        // line(width, height, Point{frow(x, xs), frow(y, ys)}, Point{frow(x+1, xs), frow(y, ys)}, color.RGBA{0, 0, 255, 100}, img)
      }
    }
  }
  return img
}

func tow(x float64, w, margin int) int {
  return int((x + 1)/2 * float64(w-margin*2)) + margin
}

func frow(x int, w int) float64 {
  return float64(x*2)/float64(w) - 1
}

func line(width, height int, p1, p2 Point, c color.RGBA, img draw.Image) {
  a := int(c.A)
  x1 := tow(p1.X, width, width/10)
  y1 := tow(p1.Y, height, height/10)
  x2 := tow(p2.X, width, width/10)
  y2 := tow(p2.Y, height, height/10)
  parts := 10
  dx := (x2 - x1)/parts
  dy := (y2 - y1)/parts
  for i := 0; i < parts; i++ {
    c.A = uint8(a*i/parts)
    bresneham(img, c, x1+dx*i, y1+dy*i, x2+dx*(i+1), y2+dy*(i+1))
  }
}

func abs(i int) int {
  if i < 0 { return i*-1 }
  return i
}

// alg taken from wikipedia
func bresneham(image draw.Image, c color.RGBA, x0, y0, x1, y1 int) {
  dx := abs(x1-x0)
  dy := abs(y1-y0)
  a := c.A
  var sx, sy int
  if x0 < x1 {
    sx = 1
  } else {
    sx = -1
  }
  if y0 < y1 {
    sy = 1
  } else{
    sy = -1
  }
  err := dx-dy

  for {
    _, _, _, b := image.At(x0, y0).RGBA()
    c.A = uint8(b*255/0xFFFF) + a
    image.Set(x0,y0,c)
    if x0 == x1 && y0 == y1 { return }
    e2 := 2*err
    if e2 > -dy {
      err = err - dy
      x0 = x0 + sx
    }
    if x0 == x1 && y0 == y1 {
      _, _, _, b := image.At(x0, y0).RGBA()
      c.A = uint8(b*255/0xFFFF) + a
      image.Set(x0,y0, c)
      return
    }
    if e2 <  dx {
      err = err + dx
      y0 = y0 + sy
    }
  }
}

<commit_msg>experimenting with opacity<commit_after>package flame

import (
	"image"
	"image/color"
	"image/draw"
  /*
	bin "encoding/binary"
	"fmt"
	"math/rand"
	"os"
	"io"
	"time"
  */
)

func RenderPreview(width, height, xs, ys int, fn Variation) *image.RGBA {
  return lines(width, height, matrix(xs, ys, fn))
}

func matrix(width, height int, fn Variation) [][]Point {
  data := make([][]Point, height)
	// these are our parameters
	var a, b, c, d, e, f float64
	a, b, c, d, e, f = DefaultParams()
  for y := range data {
    data[y] = make([]Point, width)
    for x := range data[y] {
      a, b := fn(frow(x, width), frow(y, height), a, b, c, d, e, f)
      data[y][x] = Point{a,b}
    }
  }
  return data
}

func interp(i, n int, p1, p2 Point) (Point, Point) {
  dx := (p2.X-p1.X)/float64(n)
  dy := (p2.Y-p1.Y)/float64(n)
  z := float64(i)
  return Point{p1.X + dx*z, p1.Y + dy*z}, Point{p1.X + dx*(z+1), p1.Y + dy*(z+1)}
}

// width & height correspond to the output image
func lines(width, height int, data [][]Point) *image.RGBA {
  img := image.NewRGBA(image.Rect(0, 0, width, height))
  ys := len(data)
  xs := len(data[0])
  parts := 4
  for i := 0; i < parts + 1; i++ {
    for y := range data {
      for x := range data[y] {
        // draw down
        p1, p2 := interp(i, parts, Point{frow(x, xs), frow(y, ys)}, data[y][x])
        line(width, height, p1, p2, color.RGBA{0, 0, 0, uint8(255*i/parts)}, img)
      }
    }
  }
  return img
}

// width & height correspond to the output image
func oldlines(width, height int, data [][]Point) *image.RGBA {
  img := image.NewRGBA(image.Rect(0, 0, width, height))
  ys := len(data)
  xs := len(data[0])
  for y := range data {
    for x := range data[y] {
      // draw down
      o := Point{frow(x, xs), frow(y, ys)}
      line(width, height, o, data[y][x], color.RGBA{0, 0, 0, 255}, img)
    }
  }
  return img
}

func tow(x float64, w, margin int) int {
  return int((x + 1)/2 * float64(w-margin*2)) + margin
}

func frow(x int, w int) float64 {
  return float64(x*2)/float64(w) - 1
}

func line(width, height int, p1, p2 Point, c color.RGBA, img draw.Image) {
  // a := c.A
  x1 := tow(p1.X, width, width/10)
  y1 := tow(p1.Y, height, height/10)
  x2 := tow(p2.X, width, width/10)
  y2 := tow(p2.Y, height, height/10)
  bresneham(img, c, x1, y1, x2, y2)
}

func abs(i int) int {
  if i < 0 { return i*-1 }
  return i
}

// alg taken from wikipedia
func bresneham(image draw.Image, c color.Color, x0, y0, x1, y1 int) {
  dx := abs(x1-x0)
  dy := abs(y1-y0)
  var sx, sy int
  if x0 < x1 {
    sx = 1
  } else {
    sx = -1
  }
  if y0 < y1 {
    sy = 1
  } else{
    sy = -1
  }
  err := dx-dy

  for {
    image.Set(x0,y0,c)
    if x0 == x1 && y0 == y1 { return }
    e2 := 2*err
    if e2 > -dy {
      err = err - dy
      x0 = x0 + sx
    }
    if x0 == x1 && y0 == y1 {
      image.Set(x0,y0, c)
      return
    }
    if e2 <  dx {
      err = err + dx
      y0 = y0 + sy
    }
  }
}

<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/ast"
	"go/format"
	"go/parser"
	"go/token"
	"os"
	"strings"
	"text/template"
)

var bindingsFile = `package {{.packageName}}
/*
This is an autogenerated file by autobindings
*/

import(
	"github.com/mholt/binding"
)

func ({{.variableName}} {{.structName}}) FieldMap() binding.FieldMap {
	return binding.FieldMap{ {{$vname := .variableName}}{{range $field, $mapping := .mappings}}
			&{{$vname}}.{{$field}}: "{{$mapping}}",{{end}}
	}
}`

func main() {

	prnt := flag.Bool("print", false, "Output In Console")
	filename := flag.String("file", "", "Input file")

	flag.Parse()

	if *filename == "" {
		fmt.Println("Usage : bindings {file_name}\nExample: bindings file.go")
		return
	}

	generateFieldMap(*filename, *prnt)
}

func generateFieldMap(fileName string, printOnConsole bool) {
	fset := token.NewFileSet() // positions are relative to fset
	// Parse the file given in arguments
	f, err := parser.ParseFile(fset, fileName, nil, parser.ParseComments)
	if err != nil {
		panic(err)
	}
	structMap := map[string]*ast.FieldList{}
	// range over the structs and fill struct map
	for _, d := range f.Scope.Objects {
		ts, ok := d.Decl.(*ast.TypeSpec)
		if !ok {
			continue
		}
		switch ts.Type.(type) {
		case *ast.StructType:
			x, _ := ts.Type.(*ast.StructType)
			structMap[ts.Name.String()] = x.Fields
		}
	}
	// looping through each struct and creating a bindings file for it
	packageName := f.Name
	for structName, fields := range structMap {
		variableName := strings.ToLower(string(structName[0]))
		mappings := map[string]string{}
		for _, field := range fields.List {
			if len(field.Names) == 0 {
				continue
			}
			name := field.Names[0].String()
			// if tag for field doesn't exists, create one
			if field.Tag == nil {
				mappings[name] = name
			} else if strings.Contains(field.Tag.Value, "json") {
				tags := strings.Replace(field.Tag.Value, "`", "", -1)
				for _, tag := range strings.Split(tags, " ") {
					if strings.Contains(tag, "json") {
						mapping := strings.Replace(tag, "json:\"", "", -1)
						mapping = strings.Replace(mapping, "\"", "", -1)
						if mapping == "-" {
							continue
						}
						mappings[name] = mapping
					}
				}
			} else {
				// I will handle other cases later
				mappings[name] = name
			}
		}
		content := new(bytes.Buffer)
		t := template.Must(template.New("bindings").Parse(bindingsFile))
		err = t.Execute(content, map[string]interface{}{
			"packageName":  packageName,
			"variableName": variableName,
			"structName":   structName,
			"mappings":     mappings})
		if err != nil {
			panic(err)
		}
		finalContent, err := format.Source(content.Bytes())
		if err != nil {
			panic(err)
		}
		if printOnConsole {
			fmt.Println(string(finalContent))
			return
		}
		// opening file for writing content
		writer, err := os.Create(fmt.Sprintf("%s_bindings.go", strings.ToLower(structName)))
		if err != nil {
			fmt.Printf("Error opening file %v", err)
			panic(err)
		}
		writer.WriteString(string(finalContent))
		writer.Close()
	}
}
<commit_msg>embedded structs handled<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/ast"
	"go/format"
	"go/parser"
	"go/token"
	"os"
	"strings"
	"text/template"
)

var bindingsFile = `package {{.packageName}}
/*
This is an autogenerated file by autobindings
*/

import(
	"github.com/mholt/binding"
)

func ({{.variableName}} {{.structName}}) FieldMap() binding.FieldMap {
	b := binding.FieldMap{ {{$vname := .variableName}}{{range $field, $mapping := .mappings}}
			&{{$vname}}.{{$field}}: "{{$mapping}}",{{end}}
			}

	{{$vname := .variableName}}
	{{range $field, $type := .embeds}}
	var i interface{} = {{$vname}}.{{$type}}
	if m, ok := i.(binding.FieldMap); ok {
			for k, v := range m.FieldMap() {
				b[k] = v
			}
	}
	{{end}}
	return b
}`

func main() {

	prnt := flag.Bool("print", false, "Output In Console")
	filename := flag.String("file", "", "Input file")

	flag.Parse()

	if *filename == "" {
		fmt.Println("Usage : bindings {file_name}\nExample: bindings file.go")
		return
	}

	generateFieldMap(*filename, *prnt)
}

func generateFieldMap(fileName string, printOnConsole bool) {
	fset := token.NewFileSet() // positions are relative to fset
	// Parse the file given in arguments
	f, err := parser.ParseFile(fset, fileName, nil, parser.ParseComments)
	if err != nil {
		panic(err)
	}
	structMap := map[string]*ast.FieldList{}
	// range over the structs and fill struct map
	for _, d := range f.Scope.Objects {
		ts, ok := d.Decl.(*ast.TypeSpec)
		if !ok {
			continue
		}
		switch ts.Type.(type) {
		case *ast.StructType:
			x, _ := ts.Type.(*ast.StructType)
			structMap[ts.Name.String()] = x.Fields
		}
	}
	// looping through each struct and creating a bindings file for it
	packageName := f.Name
	for structName, fields := range structMap {
		variableName := strings.ToLower(string(structName[0]))
		mappings := map[string]string{}
		embeds := []ast.Expr{}
		for _, field := range fields.List {
			if len(field.Names) == 0 {
				embeds = append(embeds, field.Type)
				continue
			}
			name := field.Names[0].String()
			// if tag for field doesn't exists, create one
			if field.Tag == nil {
				mappings[name] = name
			} else if strings.Contains(field.Tag.Value, "json") {
				tags := strings.Replace(field.Tag.Value, "`", "", -1)
				for _, tag := range strings.Split(tags, " ") {
					if strings.Contains(tag, "json") {
						mapping := strings.Replace(tag, "json:\"", "", -1)
						mapping = strings.Replace(mapping, "\"", "", -1)
						if mapping == "-" {
							continue
						}
						mappings[name] = mapping
					}
				}
			} else {
				// I will handle other cases later
				mappings[name] = name
			}
		}
		content := new(bytes.Buffer)
		t := template.Must(template.New("bindings").Parse(bindingsFile))
		err = t.Execute(content, map[string]interface{}{
			"packageName":  packageName,
			"variableName": variableName,
			"structName":   structName,
			"mappings":     mappings,
			"embeds":       embeds})
		if err != nil {
			panic(err)
		}
		finalContent, err := format.Source(content.Bytes())
		if err != nil {
			panic(err)
		}
		if printOnConsole {
			fmt.Println(string(finalContent))
			return
		}
		// opening file for writing content
		writer, err := os.Create(fmt.Sprintf("%s_bindings.go", strings.ToLower(structName)))
		if err != nil {
			fmt.Printf("Error opening file %v", err)
			panic(err)
		}
		writer.WriteString(string(finalContent))
		writer.Close()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"html/template"
	"log"
	"net/http"
)

func GeneratorHandler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		t, err := template.ParseFiles("templates/generator.html")

		if err != nil {
			log.Println("Failed to parse files")
		}
		t.Execute(w, nil)

	case "POST":
		if err := r.ParseForm(); err != nil {
			log.Println("Failed to get post values")
		}

		rawText := r.PostFormValue("desc")
		fmt.Println(rawText)
		fmt.Fprintln(w, rawText)

	default:
		http.Error(w, "Method not allowed", 405)

	}
}
<commit_msg>Added date parser<commit_after>package main

import (
	"fmt"
	"html/template"
	"log"
	"time"
	"regexp"
	"net/http"
)

func GeneratorHandler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		t, err := template.ParseFiles("templates/generator.html")

		if err != nil {
			log.Println("Failed to parse files")
		}
		t.Execute(w, nil)

	case "POST":
		if err := r.ParseForm(); err != nil {
			log.Println("Failed to get post values")
		}

		rawText := r.PostFormValue("desc")
		fmt.Fprintln(w, ParseDate(rawText))

	default:
		http.Error(w, "Method not allowed", 405)

	}
}

func ParseDate(sample string) time.Time {
	datePattern, err := regexp.Compile(`(\d{1,2}\b\D{3,9}\b\d{4})|([a-zA-Z]{3,9}\s\d{1,2}\s\d{4})`)

	if err != nil {
		panic(err)
	}

	colloquialPattern, err := regexp.Compile(`(\d{1,2}\b\D{3,9}\b\d{4})`)

	if err != nil {
		panic(err)
	}

	americanPattern, err := regexp.Compile(`([a-zA-Z]{3,9}\s\d{1,2}\s\d{4})`)
	if err != nil {
		panic(err)
	}
	var t time.Time
	dateString:= datePattern.FindString(sample)
	switch {
		case americanPattern.MatchString(dateString):
			t, _ = time.Parse("January 2 2006", datePattern.FindString(sample))

		case colloquialPattern.MatchString(dateString):
			t, _ = time.Parse("2 January 2006", datePattern.FindString(sample))
		}
		return t
}

<|endoftext|>


<commit_before>package main

import "fmt"
import "math/big"

// A BigIntPoly represents a polynomial with big.Int coefficients.
//
// The zero value for a BigIntPoly represents the zero polynomial.
type BigIntPoly struct {
	coeffs []big.Int
}

// Only polynomials built with the same value of N and R may be used
// together in one of the functions below.

// Builds a new BigIntPoly representing the zero polynomial
// mod (N, X^R - 1). R must fit into an int.
func NewBigIntPoly(N, R big.Int) *BigIntPoly {
	rInt := int(R.Int64())
	p := BigIntPoly{make([]big.Int, rInt)}

	// Pre-allocate space for each coefficient. In order to take
	// advantage of this, we must not assign to entries of
	// p.coeffs directly but instead use big.Int.Set().
	var nSq big.Int
	nSq.Mul(&N, &N)
	for i := 0; i < rInt; i++ {
		p.coeffs[i].Set(&nSq)
		p.coeffs[i].Set(&big.Int{})
	}

	return &p
}

// Sets p to X^k + a mod (N, X^R - 1).
func (p *BigIntPoly) Set(a, k, N big.Int) {
	R := len(p.coeffs)
	p.coeffs[0].Mod(&a, &N)
	for i := 1; i < R; i++ {
		p.coeffs[i].Set(&big.Int{})
	}
	var i big.Int
	i.Mod(&k, big.NewInt(int64(R)))
	p.coeffs[int(i.Int64())].Set(big.NewInt(1))
}

// Returns whether p has the same coefficients as q.
func (p *BigIntPoly) Eq(q *BigIntPoly) bool {
	R := len(p.coeffs)
	for i := 0; i < R; i++ {
		if p.coeffs[i].Cmp(&q.coeffs[i]) != 0 {
			return false
		}
	}
	return true
}

// Sets p to the product of p and q mod (N, X^R - 1). tmp1 and tmp2
// must not alias each other or p or q.
func (p *BigIntPoly) mul(q *BigIntPoly, N big.Int, tmp1, tmp2 *BigIntPoly) {
	R := len(tmp1.coeffs)
	for i := 0; i < R; i++ {
		tmp1.coeffs[i].Set(&big.Int{})
	}

	for i := 0; i < R; i++ {
		for j := 0; j < R; j++ {
			k := (i + j) % R
			// Set tmp1.coeffs[k] to (tmp1.coeffs[k] +
			// p.coeffs[i] * q.coeffs[j]) % N.

			tmp2.coeffs[k].Mul(&p.coeffs[i], &q.coeffs[j])

			// Set tmp2.coeffs[k] to tmp2.coeffs[k] +
			// tmp1.coeffs[k], avoid copying if possible.
			if tmp2.coeffs[k].Sign() == 0 {
				tmp2.coeffs[k], tmp1.coeffs[k] =
					tmp1.coeffs[k], tmp2.coeffs[k]
			} else if tmp1.coeffs[k].Sign() != 0 {
				tmp2.coeffs[k].Add(
					&tmp2.coeffs[k], &tmp1.coeffs[k])
			}

			// Set tmp1.coeffs[k] to tmp2.coeffs[k] % N,
			// avoiding copying if possible.
			if tmp2.coeffs[k].Cmp(&N) < 0 {
				tmp2.coeffs[k], tmp1.coeffs[k] =
					tmp1.coeffs[k], tmp2.coeffs[k]
			} else {
				// Use big.Int.QuoRem() instead of
				// big.Int.Mod() since the latter allocates an
				// extra big.Int.
				tmp2.coeffs[k].QuoRem(
					&tmp2.coeffs[k], &N, &tmp1.coeffs[k])
			}
		}
	}
	p.coeffs, tmp1.coeffs = tmp1.coeffs, p.coeffs
}

// Sets p to p^N mod (N, X^R - 1), where R is the size of p. tmp1,
// tmp2, and tmp3 must not alias each other or p.
func (p *BigIntPoly) Pow(N big.Int, tmp1, tmp2, tmp3 *BigIntPoly) {
	R := len(p.coeffs)
	for i := 0; i < R; i++ {
		tmp1.coeffs[i].Set(&p.coeffs[i])
	}

	for i := N.BitLen() - 2; i >= 0; i-- {
		tmp1.mul(tmp1, N, tmp2, tmp3)
		if N.Bit(i) != 0 {
			tmp1.mul(p, N, tmp2, tmp3)
		}
	}
	p.coeffs, tmp1.coeffs = tmp1.coeffs, p.coeffs
}

// fmt.Formatter implementation.
func (p *BigIntPoly) Format(f fmt.State, c rune) {
	i := len(p.coeffs) - 1
	for ; i >= 0 && p.coeffs[i].Sign() == 0; i-- {
	}

	if i < 0 {
		fmt.Fprint(f, "0")
		return
	}

	// Formats coeff*x^deg.
	formatNonZeroMonomial := func(
		f fmt.State, c rune,
		coeff big.Int, deg int) {
		if coeff.Cmp(big.NewInt(1)) != 0 || deg == 0 {
			fmt.Fprint(f, &coeff)
		}
		if deg != 0 {
			fmt.Fprint(f, "x")
			if deg > 1 {
				fmt.Fprint(f, "^", deg)
			}
		}
	}

	formatNonZeroMonomial(f, c, p.coeffs[i], i)

	for i--; i >= 0; i-- {
		if p.coeffs[i].Sign() != 0 {
			fmt.Fprint(f, " + ")
			formatNonZeroMonomial(f, c, p.coeffs[i], i)
		}
	}
}
<commit_msg>Reduce the number of modulo operations in BigIntPoly.mul()<commit_after>package main

import "fmt"
import "math/big"

// A BigIntPoly represents a polynomial with big.Int coefficients.
//
// The zero value for a BigIntPoly represents the zero polynomial.
type BigIntPoly struct {
	coeffs []big.Int
}

// Only polynomials built with the same value of N and R may be used
// together in one of the functions below.

// Builds a new BigIntPoly representing the zero polynomial
// mod (N, X^R - 1). R must fit into an int.
func NewBigIntPoly(N, R big.Int) *BigIntPoly {
	rInt := int(R.Int64())
	p := BigIntPoly{make([]big.Int, rInt)}

	// Pre-allocate space for each coefficient (which can be up to
	// R*(N - 1)^2 in intermediate calculations). In order to take
	// advantage of this, we must not assign to entries of
	// p.coeffs directly but instead use big.Int.Set().
	var maxCoeff big.Int
	maxCoeff.Sub(&N, big.NewInt(1))
	maxCoeff.Mul(&maxCoeff, &maxCoeff)
	maxCoeff.Mul(&maxCoeff, &R)
	for i := 0; i < rInt; i++ {
		p.coeffs[i].Set(&maxCoeff)
		p.coeffs[i].Set(&big.Int{})
	}

	return &p
}

// Sets p to X^k + a mod (N, X^R - 1).
func (p *BigIntPoly) Set(a, k, N big.Int) {
	R := len(p.coeffs)
	p.coeffs[0].Mod(&a, &N)
	for i := 1; i < R; i++ {
		p.coeffs[i].Set(&big.Int{})
	}
	var i big.Int
	i.Mod(&k, big.NewInt(int64(R)))
	p.coeffs[int(i.Int64())].Set(big.NewInt(1))
}

// Returns whether p has the same coefficients as q.
func (p *BigIntPoly) Eq(q *BigIntPoly) bool {
	R := len(p.coeffs)
	for i := 0; i < R; i++ {
		if p.coeffs[i].Cmp(&q.coeffs[i]) != 0 {
			return false
		}
	}
	return true
}

// Sets p to the product of p and q mod (N, X^R - 1). tmp1 and tmp2
// must not alias each other or p or q.
func (p *BigIntPoly) mul(q *BigIntPoly, N big.Int, tmp1, tmp2 *BigIntPoly) {
	R := len(tmp1.coeffs)
	for i := 0; i < R; i++ {
		tmp1.coeffs[i].Set(&big.Int{})
	}

	for i := 0; i < R; i++ {
		for j := 0; j < R; j++ {
			k := (i + j) % R
			// Set tmp1.coeffs[k] to (tmp1.coeffs[k] +
			// p.coeffs[i] * q.coeffs[j]) % N.

			tmp2.coeffs[k].Mul(&p.coeffs[i], &q.coeffs[j])

			// Set tmp1.coeffs[k] to tmp2.coeffs[k] +
			// tmp1.coeffs[k], avoid copying if possible.
			if tmp1.coeffs[k].Sign() == 0 {
				tmp1.coeffs[k], tmp2.coeffs[k] =
					tmp2.coeffs[k], tmp1.coeffs[k]
			} else if tmp2.coeffs[k].Sign() != 0 {
				tmp1.coeffs[k].Add(
					&tmp1.coeffs[k], &tmp2.coeffs[k])
			}
		}
	}

	for i := 0; i < R; i++ {
		// Set p.coeffs[i] to tmp1.coeffs[i] % N,
		// avoiding copying if possible.
		if tmp1.coeffs[i].Cmp(&N) < 0 {
			tmp1.coeffs[i], p.coeffs[i] =
				p.coeffs[i], tmp1.coeffs[i]
		} else {
			// Use big.Int.QuoRem() instead of
			// big.Int.Mod() since the latter allocates an
			// extra big.Int.
			tmp1.coeffs[i].QuoRem(
				&tmp1.coeffs[i], &N, &p.coeffs[i])
		}
	}
}

// Sets p to p^N mod (N, X^R - 1), where R is the size of p. tmp1,
// tmp2, and tmp3 must not alias each other or p.
func (p *BigIntPoly) Pow(N big.Int, tmp1, tmp2, tmp3 *BigIntPoly) {
	R := len(p.coeffs)
	for i := 0; i < R; i++ {
		tmp1.coeffs[i].Set(&p.coeffs[i])
	}

	for i := N.BitLen() - 2; i >= 0; i-- {
		tmp1.mul(tmp1, N, tmp2, tmp3)
		if N.Bit(i) != 0 {
			tmp1.mul(p, N, tmp2, tmp3)
		}
	}
	p.coeffs, tmp1.coeffs = tmp1.coeffs, p.coeffs
}

// fmt.Formatter implementation.
func (p *BigIntPoly) Format(f fmt.State, c rune) {
	i := len(p.coeffs) - 1
	for ; i >= 0 && p.coeffs[i].Sign() == 0; i-- {
	}

	if i < 0 {
		fmt.Fprint(f, "0")
		return
	}

	// Formats coeff*x^deg.
	formatNonZeroMonomial := func(
		f fmt.State, c rune,
		coeff big.Int, deg int) {
		if coeff.Cmp(big.NewInt(1)) != 0 || deg == 0 {
			fmt.Fprint(f, &coeff)
		}
		if deg != 0 {
			fmt.Fprint(f, "x")
			if deg > 1 {
				fmt.Fprint(f, "^", deg)
			}
		}
	}

	formatNonZeroMonomial(f, c, p.coeffs[i], i)

	for i--; i >= 0; i-- {
		if p.coeffs[i].Sign() != 0 {
			fmt.Fprint(f, " + ")
			formatNonZeroMonomial(f, c, p.coeffs[i], i)
		}
	}
}
<|endoftext|>


<commit_before>package conn

import (
	"fmt"
	"math/rand"
	"net"
	"strings"
	"syscall"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
	manet "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr-net"
	reuseport "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-reuseport"

	addrutil "github.com/jbenet/go-ipfs/p2p/net/swarm/addr"
	peer "github.com/jbenet/go-ipfs/p2p/peer"
	debugerror "github.com/jbenet/go-ipfs/util/debugerror"
)

// String returns the string rep of d.
func (d *Dialer) String() string {
	return fmt.Sprintf("<Dialer %s %s ...>", d.LocalPeer, d.LocalAddrs[0])
}

// Dial connects to a peer over a particular address
// Ensures raddr is part of peer.Addresses()
// Example: d.DialAddr(ctx, peer.Addresses()[0], peer)
func (d *Dialer) Dial(ctx context.Context, raddr ma.Multiaddr, remote peer.ID) (Conn, error) {

	maconn, err := d.rawConnDial(ctx, raddr, remote)
	if err != nil {
		return nil, err
	}

	var connOut Conn
	var errOut error
	done := make(chan struct{})

	// do it async to ensure we respect don contexteone
	go func() {
		defer func() { done <- struct{}{} }()

		c, err := newSingleConn(ctx, d.LocalPeer, remote, maconn)
		if err != nil {
			errOut = err
			return
		}

		if d.PrivateKey == nil {
			log.Warning("dialer %s dialing INSECURELY %s at %s!", d, remote, raddr)
			connOut = c
			return
		}
		c2, err := newSecureConn(ctx, d.PrivateKey, c)
		if err != nil {
			errOut = err
			c.Close()
			return
		}

		connOut = c2
	}()

	select {
	case <-ctx.Done():
		maconn.Close()
		return nil, ctx.Err()
	case <-done:
		// whew, finished.
	}

	return connOut, errOut
}

// rawConnDial dials the underlying net.Conn + manet.Conns
func (d *Dialer) rawConnDial(ctx context.Context, raddr ma.Multiaddr, remote peer.ID) (manet.Conn, error) {

	// before doing anything, check we're going to be able to dial.
	// we may not support the given address.
	if _, _, err := manet.DialArgs(raddr); err != nil {
		return nil, err
	}

	if strings.HasPrefix(raddr.String(), "/ip4/0.0.0.0") {
		return nil, debugerror.Errorf("Attempted to connect to zero address: %s", raddr)
	}

	// get local addr to use.
	laddr := pickLocalAddr(d.LocalAddrs, raddr)
	log.Debugf("%s dialing %s -- %s --> %s", d.LocalPeer, remote, laddr, raddr)

	// make a copy of the manet.Dialer, we may need to change its timeout.
	madialer := d.Dialer

	if laddr != nil && reuseport.Available() {
		// we're perhaps going to dial twice. half the timeout, so we can afford to.
		// otherwise our context would expire right after the first dial.
		madialer.Dialer.Timeout = (madialer.Dialer.Timeout / 2)

		// dial using reuseport.Dialer, because we're probably reusing addrs.
		// this is optimistic, as the reuseDial may fail to bind the port.
		if nconn, retry, reuseErr := reuseDial(madialer.Dialer, laddr, raddr); reuseErr == nil {
			// if it worked, wrap the raw net.Conn with our manet.Conn
			log.Debugf("%s reuse worked! %s %s %s", d.LocalPeer, laddr, nconn.RemoteAddr(), nconn)
			return manet.WrapNetConn(nconn)
		} else if !retry {
			// reuseDial is sure this is a legitimate dial failure, not a reuseport failure.
			return nil, reuseErr
		} else {
			// this is a failure to reuse port. log it.
			log.Debugf("%s port reuse failed: %s --> %s -- %s", d.LocalPeer, laddr, raddr, reuseErr)
		}
	}

	return madialer.Dial(raddr)
}

func reuseDial(dialer net.Dialer, laddr, raddr ma.Multiaddr) (conn net.Conn, retry bool, err error) {
	if laddr == nil {
		// if we're given no local address no sense in using reuseport to dial, dial out as usual.
		return nil, true, reuseport.ErrReuseFailed
	}

	// give reuse.Dialer the manet.Dialer's Dialer.
	// (wow, Dialer should've so been an interface...)
	rd := reuseport.Dialer{dialer}

	// get the local net.Addr manually
	rd.D.LocalAddr, err = manet.ToNetAddr(laddr)
	if err != nil {
		return nil, true, err // something wrong with laddr. retry without.
	}

	// get the raddr dial args for rd.dial
	network, netraddr, err := manet.DialArgs(raddr)
	if err != nil {
		return nil, true, err // something wrong with laddr. retry without.
	}

	// rd.Dial gets us a net.Conn with SO_REUSEPORT and SO_REUSEADDR set.
	conn, err = rd.Dial(network, netraddr)
	return conn, reuseErrShouldRetry(err), err // hey! it worked!
}

// reuseErrShouldRetry diagnoses whether to retry after a reuse error.
// if we failed to bind, we should retry. if bind worked and this is a
// real dial error (remote end didnt answer) then we should not retry.
func reuseErrShouldRetry(err error) bool {
	if err == nil {
		return false // hey, it worked! no need to retry.
	}

	// if it's a network timeout error, it's a legitimate failure.
	if nerr, ok := err.(net.Error); ok && nerr.Timeout() {
		return true
	}

	errno, ok := err.(syscall.Errno)
	if !ok { // not an errno? who knows what this is. retry.
		return true
	}

	switch errno {
	case syscall.EADDRINUSE, syscall.EADDRNOTAVAIL:
		return true // failure to bind. retry.
	case syscall.ECONNREFUSED:
		return false // real dial error
	default:
		return true // optimistically default to retry.
	}
}

func pickLocalAddr(laddrs []ma.Multiaddr, raddr ma.Multiaddr) (laddr ma.Multiaddr) {
	if len(laddrs) < 1 {
		return nil
	}

	// make sure that we ONLY use local addrs that match the remote addr.
	laddrs = manet.AddrMatch(raddr, laddrs)
	if len(laddrs) < 1 {
		return nil
	}

	// make sure that we ONLY use local addrs that CAN dial the remote addr.
	// filter out all the local addrs that aren't capable
	raddrIPLayer := ma.Split(raddr)[0]
	raddrIsLoopback := manet.IsIPLoopback(raddrIPLayer)
	raddrIsLinkLocal := manet.IsIP6LinkLocal(raddrIPLayer)
	laddrs = addrutil.FilterAddrs(laddrs, func(a ma.Multiaddr) bool {
		laddrIPLayer := ma.Split(a)[0]
		laddrIsLoopback := manet.IsIPLoopback(laddrIPLayer)
		laddrIsLinkLocal := manet.IsIP6LinkLocal(laddrIPLayer)
		if laddrIsLoopback { // our loopback addrs can only dial loopbacks.
			return raddrIsLoopback
		}
		if laddrIsLinkLocal {
			return raddrIsLinkLocal // out linklocal addrs can only dial link locals.
		}
		return true
	})

	// TODO pick with a good heuristic
	// we use a random one for now to prevent bad addresses from making nodes unreachable
	// with a random selection, multiple tries may work.
	return laddrs[rand.Intn(len(laddrs))]
}

// MultiaddrProtocolsMatch returns whether two multiaddrs match in protocol stacks.
func MultiaddrProtocolsMatch(a, b ma.Multiaddr) bool {
	ap := a.Protocols()
	bp := b.Protocols()

	if len(ap) != len(bp) {
		return false
	}

	for i, api := range ap {
		if api.Code != bp[i].Code {
			return false
		}
	}

	return true
}

// MultiaddrNetMatch returns the first Multiaddr found to match  network.
func MultiaddrNetMatch(tgt ma.Multiaddr, srcs []ma.Multiaddr) ma.Multiaddr {
	for _, a := range srcs {
		if MultiaddrProtocolsMatch(tgt, a) {
			return a
		}
	}
	return nil
}
<commit_msg>p2p/net/conn: reuse timeout is real error<commit_after>package conn

import (
	"fmt"
	"math/rand"
	"net"
	"strings"
	"syscall"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
	manet "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr-net"
	reuseport "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-reuseport"

	addrutil "github.com/jbenet/go-ipfs/p2p/net/swarm/addr"
	peer "github.com/jbenet/go-ipfs/p2p/peer"
	debugerror "github.com/jbenet/go-ipfs/util/debugerror"
)

// String returns the string rep of d.
func (d *Dialer) String() string {
	return fmt.Sprintf("<Dialer %s %s ...>", d.LocalPeer, d.LocalAddrs[0])
}

// Dial connects to a peer over a particular address
// Ensures raddr is part of peer.Addresses()
// Example: d.DialAddr(ctx, peer.Addresses()[0], peer)
func (d *Dialer) Dial(ctx context.Context, raddr ma.Multiaddr, remote peer.ID) (Conn, error) {

	maconn, err := d.rawConnDial(ctx, raddr, remote)
	if err != nil {
		return nil, err
	}

	var connOut Conn
	var errOut error
	done := make(chan struct{})

	// do it async to ensure we respect don contexteone
	go func() {
		defer func() { done <- struct{}{} }()

		c, err := newSingleConn(ctx, d.LocalPeer, remote, maconn)
		if err != nil {
			errOut = err
			return
		}

		if d.PrivateKey == nil {
			log.Warning("dialer %s dialing INSECURELY %s at %s!", d, remote, raddr)
			connOut = c
			return
		}
		c2, err := newSecureConn(ctx, d.PrivateKey, c)
		if err != nil {
			errOut = err
			c.Close()
			return
		}

		connOut = c2
	}()

	select {
	case <-ctx.Done():
		maconn.Close()
		return nil, ctx.Err()
	case <-done:
		// whew, finished.
	}

	return connOut, errOut
}

// rawConnDial dials the underlying net.Conn + manet.Conns
func (d *Dialer) rawConnDial(ctx context.Context, raddr ma.Multiaddr, remote peer.ID) (manet.Conn, error) {

	// before doing anything, check we're going to be able to dial.
	// we may not support the given address.
	if _, _, err := manet.DialArgs(raddr); err != nil {
		return nil, err
	}

	if strings.HasPrefix(raddr.String(), "/ip4/0.0.0.0") {
		return nil, debugerror.Errorf("Attempted to connect to zero address: %s", raddr)
	}

	// get local addr to use.
	laddr := pickLocalAddr(d.LocalAddrs, raddr)
	log.Debugf("%s dialing %s -- %s --> %s", d.LocalPeer, remote, laddr, raddr)

	// make a copy of the manet.Dialer, we may need to change its timeout.
	madialer := d.Dialer

	if laddr != nil && reuseport.Available() {
		// we're perhaps going to dial twice. half the timeout, so we can afford to.
		// otherwise our context would expire right after the first dial.
		madialer.Dialer.Timeout = (madialer.Dialer.Timeout / 2)

		// dial using reuseport.Dialer, because we're probably reusing addrs.
		// this is optimistic, as the reuseDial may fail to bind the port.
		if nconn, retry, reuseErr := reuseDial(madialer.Dialer, laddr, raddr); reuseErr == nil {
			// if it worked, wrap the raw net.Conn with our manet.Conn
			log.Debugf("%s reuse worked! %s %s %s", d.LocalPeer, laddr, nconn.RemoteAddr(), nconn)
			return manet.WrapNetConn(nconn)
		} else if !retry {
			// reuseDial is sure this is a legitimate dial failure, not a reuseport failure.
			return nil, reuseErr
		} else {
			// this is a failure to reuse port. log it.
			log.Debugf("%s port reuse failed: %s --> %s -- %s", d.LocalPeer, laddr, raddr, reuseErr)
		}
	}

	return madialer.Dial(raddr)
}

func reuseDial(dialer net.Dialer, laddr, raddr ma.Multiaddr) (conn net.Conn, retry bool, err error) {
	if laddr == nil {
		// if we're given no local address no sense in using reuseport to dial, dial out as usual.
		return nil, true, reuseport.ErrReuseFailed
	}

	// give reuse.Dialer the manet.Dialer's Dialer.
	// (wow, Dialer should've so been an interface...)
	rd := reuseport.Dialer{dialer}

	// get the local net.Addr manually
	rd.D.LocalAddr, err = manet.ToNetAddr(laddr)
	if err != nil {
		return nil, true, err // something wrong with laddr. retry without.
	}

	// get the raddr dial args for rd.dial
	network, netraddr, err := manet.DialArgs(raddr)
	if err != nil {
		return nil, true, err // something wrong with laddr. retry without.
	}

	// rd.Dial gets us a net.Conn with SO_REUSEPORT and SO_REUSEADDR set.
	conn, err = rd.Dial(network, netraddr)
	return conn, reuseErrShouldRetry(err), err // hey! it worked!
}

// reuseErrShouldRetry diagnoses whether to retry after a reuse error.
// if we failed to bind, we should retry. if bind worked and this is a
// real dial error (remote end didnt answer) then we should not retry.
func reuseErrShouldRetry(err error) bool {
	if err == nil {
		return false // hey, it worked! no need to retry.
	}

	// if it's a network timeout error, it's a legitimate failure.
	if nerr, ok := err.(net.Error); ok && nerr.Timeout() {
		return false
	}

	errno, ok := err.(syscall.Errno)
	if !ok { // not an errno? who knows what this is. retry.
		return true
	}

	switch errno {
	case syscall.EADDRINUSE, syscall.EADDRNOTAVAIL:
		return true // failure to bind. retry.
	case syscall.ECONNREFUSED:
		return false // real dial error
	default:
		return true // optimistically default to retry.
	}
}

func pickLocalAddr(laddrs []ma.Multiaddr, raddr ma.Multiaddr) (laddr ma.Multiaddr) {
	if len(laddrs) < 1 {
		return nil
	}

	// make sure that we ONLY use local addrs that match the remote addr.
	laddrs = manet.AddrMatch(raddr, laddrs)
	if len(laddrs) < 1 {
		return nil
	}

	// make sure that we ONLY use local addrs that CAN dial the remote addr.
	// filter out all the local addrs that aren't capable
	raddrIPLayer := ma.Split(raddr)[0]
	raddrIsLoopback := manet.IsIPLoopback(raddrIPLayer)
	raddrIsLinkLocal := manet.IsIP6LinkLocal(raddrIPLayer)
	laddrs = addrutil.FilterAddrs(laddrs, func(a ma.Multiaddr) bool {
		laddrIPLayer := ma.Split(a)[0]
		laddrIsLoopback := manet.IsIPLoopback(laddrIPLayer)
		laddrIsLinkLocal := manet.IsIP6LinkLocal(laddrIPLayer)
		if laddrIsLoopback { // our loopback addrs can only dial loopbacks.
			return raddrIsLoopback
		}
		if laddrIsLinkLocal {
			return raddrIsLinkLocal // out linklocal addrs can only dial link locals.
		}
		return true
	})

	// TODO pick with a good heuristic
	// we use a random one for now to prevent bad addresses from making nodes unreachable
	// with a random selection, multiple tries may work.
	return laddrs[rand.Intn(len(laddrs))]
}

// MultiaddrProtocolsMatch returns whether two multiaddrs match in protocol stacks.
func MultiaddrProtocolsMatch(a, b ma.Multiaddr) bool {
	ap := a.Protocols()
	bp := b.Protocols()

	if len(ap) != len(bp) {
		return false
	}

	for i, api := range ap {
		if api.Code != bp[i].Code {
			return false
		}
	}

	return true
}

// MultiaddrNetMatch returns the first Multiaddr found to match  network.
func MultiaddrNetMatch(tgt ma.Multiaddr, srcs []ma.Multiaddr) ma.Multiaddr {
	for _, a := range srcs {
		if MultiaddrProtocolsMatch(tgt, a) {
			return a
		}
	}
	return nil
}
<|endoftext|>