<commit_before>package main

import (
	"context"
	"flag"
	"fmt"
	"math"
	"math/rand"
	"net/http"
	"os"
	"sync/atomic"
	"testing"

	"github.com/bmatsuo/lmdb-go/lmdb"
	"github.com/dgraph-io/badger"
	"github.com/dgraph-io/badger-bench/store"
	"github.com/dgraph-io/badger/table"
	"github.com/dgraph-io/badger/y"
)

var (
	ctx           = context.Background()
	numKeys       = flag.Float64("keys_mil", 10.0, "How many million keys to write.")
	flagDir       = flag.String("dir", "bench-tmp", "Where data is temporarily stored.")
	flagValueSize = flag.Int("valsz", 128, "Size of each value.")
)

const Mi int = 1000000
const Mf float64 = 1000000

func getBadger() (*badger.KV, error) {
	opt := badger.DefaultOptions
	opt.MapTablesTo = table.LoadToRAM
	opt.Dir = *flagDir + "/badger"
	opt.ValueDir = opt.Dir
	fmt.Println(opt.Dir)
	opt.DoNotCompact = true
	opt.ValueGCThreshold = 0.0
	return badger.NewKV(&opt)
}

func getRocks() *store.Store {
	rdb, err := store.NewReadOnlyStore(*flagDir + "/rocks")
	y.Check(err)
	return rdb
}

func getLmdb() *lmdb.Env {
	lmdbEnv, err := lmdb.NewEnv()
	y.Check(err)
	err = lmdbEnv.SetMaxReaders(math.MaxInt64)
	y.Check(err)
	err = lmdbEnv.SetMaxDBs(1)
	y.Check(err)
	err = lmdbEnv.SetMapSize(1 << 38) // ~273Gb
	y.Check(err)

	err = lmdbEnv.Open(*flagDir+"/lmdb", lmdb.Readonly, 0777)
	y.Check(err)
	return lmdbEnv
}

func newKey() []byte {
	k := rand.Int() % int(*numKeys*Mf)
	key := fmt.Sprintf("vsz=%05d-k=%010d", *flagValueSize, k) // 22 bytes.
	return []byte(key)
}

func print(count int) {
	if count%100000 == 0 {
		fmt.Printf(".")
	} else if count%Mi == 0 {
		fmt.Printf("-")
	}
}

type hitCounter struct {
	found    uint64
	notFound uint64
	errored  uint64
}

func (h *hitCounter) Reset() {
	h.found, h.notFound, h.errored = 0, 0, 0
}

func (h *hitCounter) Print(storeName string, b *testing.B) {
	b.Logf("%s: %d keys had valid values.", storeName, h.found)
	b.Logf("%s: %d keys had no values", storeName, h.notFound)
	b.Logf("%s: %d keys had errors", storeName, h.errored)
	b.Logf("%s: %d total keys looked at", storeName, h.found+h.notFound+h.errored)
	b.Logf("%s: hit rate : %.2f", storeName, float64(h.found)/float64(h.found+h.notFound+h.errored))
}

// A generic read benchmark that runs the doGet func multiple times, counts
// the results and prints them out.
func runRandomReadBenchmark(storeName string, b *testing.B, doGet func(*hitCounter)) {
	counter := &hitCounter{}
	b.Run("read-random"+storeName, func(b *testing.B) {
		counter.Reset()
		b.RunParallel(func(pb *testing.PB) {
			c := &hitCounter{}
			for pb.Next() {
				doGet(c)
			}
			atomic.AddUint64(&counter.found, c.found)
			atomic.AddUint64(&counter.notFound, c.notFound)
			atomic.AddUint64(&counter.errored, c.errored)
		})
	})
	counter.Print(storeName, b)
}

func BenchmarkReadRandomBadger(b *testing.B) {
	bdb, err := getBadger()
	y.Check(err)
	defer bdb.Close()

	runRandomReadBenchmark("badger", b, func(c *hitCounter) {
		var item badger.KVItem
		key := newKey()
		err := bdb.Get(key, &item)
		if err != nil {
			c.errored++
		} else if item.Value() != nil {
			c.found++
		} else {
			c.notFound++
		}
	})
}

func BenchmarkReadRandomRocks(b *testing.B) {
	rdb := getRocks()
	defer rdb.Close()

	runRandomReadBenchmark("rocksdb", b, func(c *hitCounter) {
		key := newKey()
		// FIXME lookup API and increment accordingly
		if _, err := rdb.Get(key); err == nil {
			c.found++
		}
	})
}

func BenchmarkReadRandomLmdb(b *testing.B) {
	lmdbEnv := getLmdb()
	defer lmdbEnv.Close()

	var lmdbDBI lmdb.DBI
	// Acquire handle
	err := lmdbEnv.View(func(txn *lmdb.Txn) error {
		var err error
		lmdbDBI, err = txn.OpenDBI("bench", 0)
		return err
	})
	y.Check(err)
	defer lmdbEnv.CloseDBI(lmdbDBI)

	runRandomReadBenchmark("lmdb", b, func(c *hitCounter) {
		key := newKey()
		err = lmdbEnv.View(func(txn *lmdb.Txn) error {
			txn.RawRead = true
			_, err := txn.Get(lmdbDBI, key)
			if lmdb.IsNotFound(err) {
				c.notFound++
				return nil

			} else if err != nil {
				return err
			}
			c.found++
			return nil
		})
		if err != nil {
			c.errored++
		}
	})
}

func safecopy(dst []byte, src []byte) []byte {
	if cap(dst) < len(src) {
		dst = make([]byte, len(src))
	}
	dst = dst[0:len(src)]
	copy(dst, src)
	return dst
}

func BenchmarkIterateRocks(b *testing.B) {
	rdb := getRocks()
	k := make([]byte, 1024)
	v := make([]byte, Mi)
	b.ResetTimer()

	b.Run("rocksdb-iterate", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			itr := rdb.NewIterator()
			var count int
			for itr.SeekToFirst(); itr.Valid(); itr.Next() {
				{
					// do some processing.
					k = safecopy(k, itr.Key().Data())
					v = safecopy(v, itr.Value().Data())
				}
				count++
				print(count)
				if count > 2*Mi {
					break
				}
			}
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func BenchmarkIterateLmdb(b *testing.B) {
	lmdbEnv := getLmdb()

	var lmdbDBI lmdb.DBI
	// Acquire handle
	err := lmdbEnv.View(func(txn *lmdb.Txn) error {
		var err error
		lmdbDBI, err = txn.OpenDBI("bench", 0)
		return err
	})
	y.Check(err)
	defer lmdbEnv.CloseDBI(lmdbDBI)

	k := make([]byte, 1024)
	v := make([]byte, Mi)
	b.ResetTimer()

	b.Run("lmdb-iterate", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			var count int
			err = lmdbEnv.View(func(txn *lmdb.Txn) error {
				txn.RawRead = true
				cur, err := txn.OpenCursor(lmdbDBI)
				if err != nil {
					return err
				}
				defer cur.Close()

				for {
					k1, v1, err := cur.Get(nil, nil, lmdb.Next)
					if lmdb.IsNotFound(err) {
						return nil
					}
					if err != nil {
						return err
					}

					//fmt.Printf("%s %s\n", k, v)

					y.AssertTruef(len(v1) == *flagValueSize, "Assertion failed. value size is %d, expected %d", len(v1), *flagValueSize)

					// do some processing.
					k = safecopy(k, k1)
					v = safecopy(v, v1)

					count++
					print(count)
					if count > 2*Mi {
						break
					}
				}
				return nil
			})
			y.Check(err)
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func BenchmarkIterateBadgerOnlyKeys(b *testing.B) {
	bdb, err := getBadger()
	y.Check(err)
	k := make([]byte, 1024)
	b.ResetTimer()

	b.Run("badger-iterate-onlykeys", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			var count int
			// 100 = size, 0 = num workers, false = fwd direction.
			opt := badger.IteratorOptions{}
			opt.PrefetchSize = 10000
			itr := bdb.NewIterator(opt)
			for itr.Rewind(); itr.Valid(); itr.Next() {
				item := itr.Item()
				{
					// do some processing.
					k = safecopy(k, item.Key())
				}
				count++
				print(count)
				if count > 2*Mi {
					break
				}
			}
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func BenchmarkIterateBadgerWithValues(b *testing.B) {
	bdb, err := getBadger()
	y.Check(err)
	k := make([]byte, 1024)
	v := make([]byte, Mi)
	b.ResetTimer()

	b.Run("badger-iterate-withvals", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			var count int
			opt := badger.IteratorOptions{}
			opt.PrefetchSize = 10000
			opt.FetchValues = true
			itr := bdb.NewIterator(opt)
			for itr.Rewind(); itr.Valid(); itr.Next() {
				item := itr.Item()
				vsz := len(item.Value())
				y.AssertTruef(vsz == *flagValueSize, "Assertion failed. value size is %d, expected %d", vsz, *flagValueSize)
				{
					// do some processing.
					k = safecopy(k, item.Key())
					v = safecopy(v, item.Value())
				}
				count++
				print(count)
				if count >= 2*Mi {
					break
				}
			}
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func TestMain(m *testing.M) {
	flag.Parse()
	// call flag.Parse() here if TestMain uses flags
	go http.ListenAndServe(":8080", nil)
	os.Exit(m.Run())
}
<commit_msg>More cleanup<commit_after>package main

import (
	"context"
	"flag"
	"fmt"
	"math"
	"math/rand"
	"net/http"
	"os"
	"sync/atomic"
	"testing"

	"github.com/bmatsuo/lmdb-go/lmdb"
	"github.com/dgraph-io/badger"
	"github.com/dgraph-io/badger-bench/store"
	"github.com/dgraph-io/badger/table"
	"github.com/dgraph-io/badger/y"
)

var (
	ctx           = context.Background()
	numKeys       = flag.Float64("keys_mil", 10.0, "How many million keys to write.")
	flagDir       = flag.String("dir", "bench-tmp", "Where data is temporarily stored.")
	flagValueSize = flag.Int("valsz", 128, "Size of each value.")
)

const Mi int = 1000000
const Mf float64 = 1000000

func getBadger() (*badger.KV, error) {
	opt := badger.DefaultOptions
	opt.MapTablesTo = table.LoadToRAM
	opt.Dir = *flagDir + "/badger"
	opt.ValueDir = opt.Dir
	fmt.Println(opt.Dir)
	opt.DoNotCompact = true
	opt.ValueGCThreshold = 0.0
	return badger.NewKV(&opt)
}

func getRocks() *store.Store {
	rdb, err := store.NewReadOnlyStore(*flagDir + "/rocks")
	y.Check(err)
	return rdb
}

func getLmdb() *lmdb.Env {
	lmdbEnv, err := lmdb.NewEnv()
	y.Check(err)
	err = lmdbEnv.SetMaxReaders(math.MaxInt64)
	y.Check(err)
	err = lmdbEnv.SetMaxDBs(1)
	y.Check(err)
	err = lmdbEnv.SetMapSize(1 << 38) // ~273Gb
	y.Check(err)

	err = lmdbEnv.Open(*flagDir+"/lmdb", lmdb.Readonly, 0777)
	y.Check(err)
	return lmdbEnv
}

func newKey() []byte {
	k := rand.Int() % int(*numKeys*Mf)
	key := fmt.Sprintf("vsz=%05d-k=%010d", *flagValueSize, k) // 22 bytes.
	return []byte(key)
}

func print(count int) {
	if count%100000 == 0 {
		fmt.Printf(".")
	} else if count%Mi == 0 {
		fmt.Printf("-")
	}
}

type hitCounter struct {
	found    uint64
	notFound uint64
	errored  uint64
}

func (h *hitCounter) Reset() {
	h.found, h.notFound, h.errored = 0, 0, 0
}

func (h *hitCounter) Update(c *hitCounter) {
	atomic.AddUint64(&h.found, c.found)
	atomic.AddUint64(&h.notFound, c.notFound)
	atomic.AddUint64(&h.errored, c.errored)
}

func (h *hitCounter) Print(storeName string, b *testing.B) {
	b.Logf("%s: %d keys had valid values.", storeName, h.found)
	b.Logf("%s: %d keys had no values", storeName, h.notFound)
	b.Logf("%s: %d keys had errors", storeName, h.errored)
	b.Logf("%s: %d total keys looked at", storeName, h.found+h.notFound+h.errored)
	b.Logf("%s: hit rate : %.2f", storeName, float64(h.found)/float64(h.found+h.notFound+h.errored))
}

// A generic read benchmark that runs the doBench func for a specific key value store,
// aggregates the hit counts and prints them out.
func runRandomReadBenchmark(storeName string, b *testing.B, doBench func(*hitCounter, *testing.PB)) {
	counter := &hitCounter{}
	b.Run("read-random"+storeName, func(b *testing.B) {
		counter.Reset()
		b.RunParallel(func(pb *testing.PB) {
			c := &hitCounter{}
			doBench(c, pb)
			counter.Update(c)
		})
	})
	counter.Print(storeName, b)
}

func BenchmarkReadRandomBadger(b *testing.B) {
	bdb, err := getBadger()
	y.Check(err)
	defer bdb.Close()

	runRandomReadBenchmark("badger", b, func(c *hitCounter, pb *testing.PB) {
		var item badger.KVItem
		for pb.Next() {
			key := newKey()
			err := bdb.Get(key, &item)
			if err != nil {
				c.errored++
			} else if item.Value() != nil {
				c.found++
			} else {
				c.notFound++
			}
		}
	})
}

func BenchmarkReadRandomRocks(b *testing.B) {
	rdb := getRocks()
	defer rdb.Close()

	runRandomReadBenchmark("rocksdb", b, func(c *hitCounter, pb *testing.PB) {
		for pb.Next() {
			key := newKey()
			// FIXME lookup API and increment accordingly
			if _, err := rdb.Get(key); err == nil {
				c.found++
			}
		}
	})
}

func BenchmarkReadRandomLmdb(b *testing.B) {
	lmdbEnv := getLmdb()
	defer lmdbEnv.Close()

	var lmdbDBI lmdb.DBI
	// Acquire handle
	err := lmdbEnv.View(func(txn *lmdb.Txn) error {
		var err error
		lmdbDBI, err = txn.OpenDBI("bench", 0)
		return err
	})
	y.Check(err)
	defer lmdbEnv.CloseDBI(lmdbDBI)

	runRandomReadBenchmark("lmdb", b, func(c *hitCounter, pb *testing.PB) {
		for pb.Next() {
			key := newKey()
			err = lmdbEnv.View(func(txn *lmdb.Txn) error {
				txn.RawRead = true
				_, err := txn.Get(lmdbDBI, key)
				if lmdb.IsNotFound(err) {
					c.notFound++
					return nil

				} else if err != nil {
					return err
				}
				c.found++
				return nil
			})
			if err != nil {
				c.errored++
			}
		}
	})
}

func safecopy(dst []byte, src []byte) []byte {
	if cap(dst) < len(src) {
		dst = make([]byte, len(src))
	}
	dst = dst[0:len(src)]
	copy(dst, src)
	return dst
}

func BenchmarkIterateRocks(b *testing.B) {
	rdb := getRocks()
	k := make([]byte, 1024)
	v := make([]byte, Mi)
	b.ResetTimer()

	b.Run("rocksdb-iterate", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			itr := rdb.NewIterator()
			var count int
			for itr.SeekToFirst(); itr.Valid(); itr.Next() {
				{
					// do some processing.
					k = safecopy(k, itr.Key().Data())
					v = safecopy(v, itr.Value().Data())
				}
				count++
				print(count)
				if count > 2*Mi {
					break
				}
			}
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func BenchmarkIterateLmdb(b *testing.B) {
	lmdbEnv := getLmdb()

	var lmdbDBI lmdb.DBI
	// Acquire handle
	err := lmdbEnv.View(func(txn *lmdb.Txn) error {
		var err error
		lmdbDBI, err = txn.OpenDBI("bench", 0)
		return err
	})
	y.Check(err)
	defer lmdbEnv.CloseDBI(lmdbDBI)

	k := make([]byte, 1024)
	v := make([]byte, Mi)
	b.ResetTimer()

	b.Run("lmdb-iterate", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			var count int
			err = lmdbEnv.View(func(txn *lmdb.Txn) error {
				txn.RawRead = true
				cur, err := txn.OpenCursor(lmdbDBI)
				if err != nil {
					return err
				}
				defer cur.Close()

				for {
					k1, v1, err := cur.Get(nil, nil, lmdb.Next)
					if lmdb.IsNotFound(err) {
						return nil
					}
					if err != nil {
						return err
					}

					//fmt.Printf("%s %s\n", k, v)

					y.AssertTruef(len(v1) == *flagValueSize, "Assertion failed. value size is %d, expected %d", len(v1), *flagValueSize)

					// do some processing.
					k = safecopy(k, k1)
					v = safecopy(v, v1)

					count++
					print(count)
					if count > 2*Mi {
						break
					}
				}
				return nil
			})
			y.Check(err)
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func BenchmarkIterateBadgerOnlyKeys(b *testing.B) {
	bdb, err := getBadger()
	y.Check(err)
	k := make([]byte, 1024)
	b.ResetTimer()

	b.Run("badger-iterate-onlykeys", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			var count int
			// 100 = size, 0 = num workers, false = fwd direction.
			opt := badger.IteratorOptions{}
			opt.PrefetchSize = 10000
			itr := bdb.NewIterator(opt)
			for itr.Rewind(); itr.Valid(); itr.Next() {
				item := itr.Item()
				{
					// do some processing.
					k = safecopy(k, item.Key())
				}
				count++
				print(count)
				if count > 2*Mi {
					break
				}
			}
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func BenchmarkIterateBadgerWithValues(b *testing.B) {
	bdb, err := getBadger()
	y.Check(err)
	k := make([]byte, 1024)
	v := make([]byte, Mi)
	b.ResetTimer()

	b.Run("badger-iterate-withvals", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			var count int
			opt := badger.IteratorOptions{}
			opt.PrefetchSize = 10000
			opt.FetchValues = true
			itr := bdb.NewIterator(opt)
			for itr.Rewind(); itr.Valid(); itr.Next() {
				item := itr.Item()
				vsz := len(item.Value())
				y.AssertTruef(vsz == *flagValueSize, "Assertion failed. value size is %d, expected %d", vsz, *flagValueSize)
				{
					// do some processing.
					k = safecopy(k, item.Key())
					v = safecopy(v, item.Value())
				}
				count++
				print(count)
				if count >= 2*Mi {
					break
				}
			}
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func TestMain(m *testing.M) {
	flag.Parse()
	// call flag.Parse() here if TestMain uses flags
	go http.ListenAndServe(":8080", nil)
	os.Exit(m.Run())
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"fmt"
	"math"
	"math/rand"
	"net/http"
	"os"
	"sync/atomic"
	"testing"

	"github.com/bmatsuo/lmdb-go/lmdb"
	"github.com/dgraph-io/badger"
	"github.com/dgraph-io/badger-bench/store"
	"github.com/dgraph-io/badger/table"
	"github.com/dgraph-io/badger/y"
)

var (
	ctx           = context.Background()
	numKeys       = flag.Float64("keys_mil", 10.0, "How many million keys to write.")
	flagDir       = flag.String("dir", "bench-tmp", "Where data is temporarily stored.")
	flagValueSize = flag.Int("valsz", 128, "Size of each value.")
)

const Mi int = 1000000
const Mf float64 = 1000000

func getBadger() (*badger.KV, error) {
	opt := badger.DefaultOptions
	opt.MapTablesTo = table.LoadToRAM
	opt.Dir = *flagDir + "/badger"
	opt.ValueDir = opt.Dir
	fmt.Println(opt.Dir)
	opt.DoNotCompact = true
	opt.ValueGCThreshold = 0.0
	return badger.NewKV(&opt)
}

func getRocks() *store.Store {
	rdb, err := store.NewReadOnlyStore(*flagDir + "/rocks")
	y.Check(err)
	return rdb
}

func getLmdb() *lmdb.Env {
	lmdbEnv, err := lmdb.NewEnv()
	y.Check(err)
	err = lmdbEnv.SetMaxReaders(math.MaxInt64)
	y.Check(err)
	err = lmdbEnv.SetMaxDBs(1)
	y.Check(err)
	err = lmdbEnv.SetMapSize(1 << 38) // ~273Gb
	y.Check(err)

	err = lmdbEnv.Open(*flagDir+"/lmdb", lmdb.Readonly, 0777)
	y.Check(err)
	return lmdbEnv
}

func newKey() []byte {
	k := rand.Int() % int(*numKeys*Mf)
	key := fmt.Sprintf("vsz=%05d-k=%010d", *flagValueSize, k) // 22 bytes.
	return []byte(key)
}

func print(count int) {
	if count%100000 == 0 {
		fmt.Printf(".")
	} else if count%Mi == 0 {
		fmt.Printf("-")
	}
}

func BenchmarkReadRandomBadger(b *testing.B) {
	bdb, err := getBadger()
	y.Check(err)
	defer bdb.Close()

	var totalFound uint64
	var totalErr uint64
	var totalNotFound uint64
	b.Run("read-random-badger", func(b *testing.B) {
		totalFound, totalNotFound, totalErr = 0, 0, 0
		b.RunParallel(func(pb *testing.PB) {
			var found, errored, notFound uint64
			var val badger.KVItem
			for pb.Next() {
				key := newKey()
				err := bdb.Get(key, &val)
				if err != nil {
					errored++
				} else if val.Value() != nil {
					found++
				} else {
					notFound++
				}
			}
			atomic.AddUint64(&totalFound, found)
			atomic.AddUint64(&totalErr, errored)
			atomic.AddUint64(&totalNotFound, notFound)
		})
	})
	b.Logf("badger %d keys had valid values.", totalFound)
	b.Logf("badger %d keys had no values", totalNotFound)
	b.Logf("badger %d keys had errors", totalErr)
	b.Logf("badger %d total keys looked at", totalFound+totalNotFound+totalErr)
	b.Logf("badger hit rate : %.2f", float64(totalFound)/float64(totalFound+totalNotFound+totalErr))
}

func BenchmarkReadRandomRocks(b *testing.B) {
	rdb := getRocks()
	defer rdb.Close()

	var totalCount uint64
	b.Run("read-random-rocks", func(b *testing.B) {
		b.RunParallel(func(pb *testing.PB) {
			var count uint64
			for pb.Next() {
				key := newKey()
				if _, err := rdb.Get(key); err == nil {
					count++
				}
			}
			atomic.AddUint64(&totalCount, count)
		})
	})
	b.Logf("rocks %d keys had valid values.", totalCount)
}

func BenchmarkReadRandomLmdb(b *testing.B) {
	lmdbEnv := getLmdb()
	defer lmdbEnv.Close()

	var lmdbDBI lmdb.DBI
	// Acquire handle
	err := lmdbEnv.View(func(txn *lmdb.Txn) error {
		var err error
		lmdbDBI, err = txn.OpenDBI("bench", 0)
		return err
	})
	y.Check(err)
	defer lmdbEnv.CloseDBI(lmdbDBI)

	var totalFound uint64
	var totalErr uint64
	var totalNotFound uint64
	b.Run("read-random-lmdb", func(b *testing.B) {
		totalFound, totalNotFound, totalErr = 0, 0, 0
		b.RunParallel(func(pb *testing.PB) {
			var found, errored, notFound uint64

			for pb.Next() {
				key := newKey()
				err = lmdbEnv.View(func(txn *lmdb.Txn) error {
					_, err := txn.Get(lmdbDBI, key)
					if lmdb.IsNotFound(err) {
						notFound++
						return nil

					} else if err != nil {
						return err
					}
					found++
					return nil
				})
				if err != nil {
					errored++
				}
			}
			atomic.AddUint64(&totalFound, found)
			atomic.AddUint64(&totalErr, errored)
			atomic.AddUint64(&totalNotFound, notFound)
		})
	})
	b.Logf("lmdb %d keys had valid values.", totalFound)
	b.Logf("lmdb %d keys had no values", totalNotFound)
	b.Logf("lmdb %d keys had errors", totalErr)
	b.Logf("lmdb %d total keys looked at", totalFound+totalNotFound+totalErr)
	b.Logf("lmdb hit rate : %.2f", float64(totalFound)/float64(totalFound+totalNotFound+totalErr))
}

func safecopy(dst []byte, src []byte) []byte {
	if cap(dst) < len(src) {
		dst = make([]byte, len(src))
	}
	dst = dst[0:len(src)]
	copy(dst, src)
	return dst
}

func BenchmarkIterateRocks(b *testing.B) {
	rdb := getRocks()
	k := make([]byte, 1024)
	v := make([]byte, Mi)
	b.ResetTimer()

	b.Run("rocksdb-iterate", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			itr := rdb.NewIterator()
			var count int
			for itr.SeekToFirst(); itr.Valid(); itr.Next() {
				{
					// do some processing.
					k = safecopy(k, itr.Key().Data())
					v = safecopy(v, itr.Value().Data())
				}
				count++
				print(count)
				if count > 2*Mi {
					break
				}
			}
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func BenchmarkIterateLmdb(b *testing.B) {
	lmdbEnv := getLmdb()

	var lmdbDBI lmdb.DBI
	// Acquire handle
	err := lmdbEnv.View(func(txn *lmdb.Txn) error {
		var err error
		lmdbDBI, err = txn.OpenDBI("bench", 0)
		return err
	})
	y.Check(err)
	defer lmdbEnv.CloseDBI(lmdbDBI)

	k := make([]byte, 1024)
	v := make([]byte, Mi)
	b.ResetTimer()

	b.Run("lmdb-iterate", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			var count int
			err = lmdbEnv.View(func(txn *lmdb.Txn) error {
				cur, err := txn.OpenCursor(lmdbDBI)
				if err != nil {
					return err
				}
				defer cur.Close()

				for {
					k1, v1, err := cur.Get(nil, nil, lmdb.Next)
					if lmdb.IsNotFound(err) {
						return nil
					}
					if err != nil {
						return err
					}

					//fmt.Printf("%s %s\n", k, v)

					// do some processing.
					k = safecopy(k, k1)
					v = safecopy(v, v1)

					count++
					print(count)
					if count > 2*Mi {
						break
					}
				}
				return nil
			})
			y.Check(err)
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func BenchmarkIterateBadgerOnlyKeys(b *testing.B) {
	bdb, err := getBadger()
	y.Check(err)
	k := make([]byte, 1024)
	b.ResetTimer()

	b.Run("badger-iterate-onlykeys", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			var count int
			// 100 = size, 0 = num workers, false = fwd direction.
			opt := badger.IteratorOptions{}
			opt.PrefetchSize = 10000
			itr := bdb.NewIterator(opt)
			for itr.Rewind(); itr.Valid(); itr.Next() {
				item := itr.Item()
				{
					// do some processing.
					k = safecopy(k, item.Key())
				}
				count++
				print(count)
				if count > 2*Mi {
					break
				}
			}
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func BenchmarkIterateBadgerWithValues(b *testing.B) {
	bdb, err := getBadger()
	y.Check(err)
	k := make([]byte, 1024)
	v := make([]byte, Mi)
	b.ResetTimer()

	b.Run("badger-iterate-withvals", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			var count int
			opt := badger.IteratorOptions{}
			opt.PrefetchSize = 10000
			opt.FetchValues = true
			itr := bdb.NewIterator(opt)
			for itr.Rewind(); itr.Valid(); itr.Next() {
				item := itr.Item()
				{
					// do some processing.
					k = safecopy(k, item.Key())
					v = safecopy(v, item.Value())
				}
				count++
				print(count)
				if count >= 2*Mi {
					break
				}
			}
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func TestMain(m *testing.M) {
	flag.Parse()
	// call flag.Parse() here if TestMain uses flags
	go http.ListenAndServe(":8080", nil)
	os.Exit(m.Run())
}
<commit_msg>Rename val to item<commit_after>package main

import (
	"context"
	"flag"
	"fmt"
	"math"
	"math/rand"
	"net/http"
	"os"
	"sync/atomic"
	"testing"

	"github.com/bmatsuo/lmdb-go/lmdb"
	"github.com/dgraph-io/badger"
	"github.com/dgraph-io/badger-bench/store"
	"github.com/dgraph-io/badger/table"
	"github.com/dgraph-io/badger/y"
)

var (
	ctx           = context.Background()
	numKeys       = flag.Float64("keys_mil", 10.0, "How many million keys to write.")
	flagDir       = flag.String("dir", "bench-tmp", "Where data is temporarily stored.")
	flagValueSize = flag.Int("valsz", 128, "Size of each value.")
)

const Mi int = 1000000
const Mf float64 = 1000000

func getBadger() (*badger.KV, error) {
	opt := badger.DefaultOptions
	opt.MapTablesTo = table.LoadToRAM
	opt.Dir = *flagDir + "/badger"
	opt.ValueDir = opt.Dir
	fmt.Println(opt.Dir)
	opt.DoNotCompact = true
	opt.ValueGCThreshold = 0.0
	return badger.NewKV(&opt)
}

func getRocks() *store.Store {
	rdb, err := store.NewReadOnlyStore(*flagDir + "/rocks")
	y.Check(err)
	return rdb
}

func getLmdb() *lmdb.Env {
	lmdbEnv, err := lmdb.NewEnv()
	y.Check(err)
	err = lmdbEnv.SetMaxReaders(math.MaxInt64)
	y.Check(err)
	err = lmdbEnv.SetMaxDBs(1)
	y.Check(err)
	err = lmdbEnv.SetMapSize(1 << 38) // ~273Gb
	y.Check(err)

	err = lmdbEnv.Open(*flagDir+"/lmdb", lmdb.Readonly, 0777)
	y.Check(err)
	return lmdbEnv
}

func newKey() []byte {
	k := rand.Int() % int(*numKeys*Mf)
	key := fmt.Sprintf("vsz=%05d-k=%010d", *flagValueSize, k) // 22 bytes.
	return []byte(key)
}

func print(count int) {
	if count%100000 == 0 {
		fmt.Printf(".")
	} else if count%Mi == 0 {
		fmt.Printf("-")
	}
}

func BenchmarkReadRandomBadger(b *testing.B) {
	bdb, err := getBadger()
	y.Check(err)
	defer bdb.Close()

	var totalFound uint64
	var totalErr uint64
	var totalNotFound uint64
	b.Run("read-random-badger", func(b *testing.B) {
		totalFound, totalNotFound, totalErr = 0, 0, 0
		b.RunParallel(func(pb *testing.PB) {
			var found, errored, notFound uint64
			var item badger.KVItem
			for pb.Next() {
				key := newKey()
				err := bdb.Get(key, &item)
				if err != nil {
					errored++
				} else if item.Value() != nil {
					found++
				} else {
					notFound++
				}
			}
			atomic.AddUint64(&totalFound, found)
			atomic.AddUint64(&totalErr, errored)
			atomic.AddUint64(&totalNotFound, notFound)
		})
	})
	b.Logf("badger %d keys had valid values.", totalFound)
	b.Logf("badger %d keys had no values", totalNotFound)
	b.Logf("badger %d keys had errors", totalErr)
	b.Logf("badger %d total keys looked at", totalFound+totalNotFound+totalErr)
	b.Logf("badger hit rate : %.2f", float64(totalFound)/float64(totalFound+totalNotFound+totalErr))
}

func BenchmarkReadRandomRocks(b *testing.B) {
	rdb := getRocks()
	defer rdb.Close()

	var totalCount uint64
	b.Run("read-random-rocks", func(b *testing.B) {
		b.RunParallel(func(pb *testing.PB) {
			var count uint64
			for pb.Next() {
				key := newKey()
				if _, err := rdb.Get(key); err == nil {
					count++
				}
			}
			atomic.AddUint64(&totalCount, count)
		})
	})
	b.Logf("rocks %d keys had valid values.", totalCount)
}

func BenchmarkReadRandomLmdb(b *testing.B) {
	lmdbEnv := getLmdb()
	defer lmdbEnv.Close()

	var lmdbDBI lmdb.DBI
	// Acquire handle
	err := lmdbEnv.View(func(txn *lmdb.Txn) error {
		var err error
		lmdbDBI, err = txn.OpenDBI("bench", 0)
		return err
	})
	y.Check(err)
	defer lmdbEnv.CloseDBI(lmdbDBI)

	var totalFound uint64
	var totalErr uint64
	var totalNotFound uint64
	b.Run("read-random-lmdb", func(b *testing.B) {
		totalFound, totalNotFound, totalErr = 0, 0, 0
		b.RunParallel(func(pb *testing.PB) {
			var found, errored, notFound uint64

			for pb.Next() {
				key := newKey()
				err = lmdbEnv.View(func(txn *lmdb.Txn) error {
					_, err := txn.Get(lmdbDBI, key)
					if lmdb.IsNotFound(err) {
						notFound++
						return nil

					} else if err != nil {
						return err
					}
					found++
					return nil
				})
				if err != nil {
					errored++
				}
			}
			atomic.AddUint64(&totalFound, found)
			atomic.AddUint64(&totalErr, errored)
			atomic.AddUint64(&totalNotFound, notFound)
		})
	})
	b.Logf("lmdb %d keys had valid values.", totalFound)
	b.Logf("lmdb %d keys had no values", totalNotFound)
	b.Logf("lmdb %d keys had errors", totalErr)
	b.Logf("lmdb %d total keys looked at", totalFound+totalNotFound+totalErr)
	b.Logf("lmdb hit rate : %.2f", float64(totalFound)/float64(totalFound+totalNotFound+totalErr))
}

func safecopy(dst []byte, src []byte) []byte {
	if cap(dst) < len(src) {
		dst = make([]byte, len(src))
	}
	dst = dst[0:len(src)]
	copy(dst, src)
	return dst
}

func BenchmarkIterateRocks(b *testing.B) {
	rdb := getRocks()
	k := make([]byte, 1024)
	v := make([]byte, Mi)
	b.ResetTimer()

	b.Run("rocksdb-iterate", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			itr := rdb.NewIterator()
			var count int
			for itr.SeekToFirst(); itr.Valid(); itr.Next() {
				{
					// do some processing.
					k = safecopy(k, itr.Key().Data())
					v = safecopy(v, itr.Value().Data())
				}
				count++
				print(count)
				if count > 2*Mi {
					break
				}
			}
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func BenchmarkIterateLmdb(b *testing.B) {
	lmdbEnv := getLmdb()

	var lmdbDBI lmdb.DBI
	// Acquire handle
	err := lmdbEnv.View(func(txn *lmdb.Txn) error {
		var err error
		lmdbDBI, err = txn.OpenDBI("bench", 0)
		return err
	})
	y.Check(err)
	defer lmdbEnv.CloseDBI(lmdbDBI)

	k := make([]byte, 1024)
	v := make([]byte, Mi)
	b.ResetTimer()

	b.Run("lmdb-iterate", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			var count int
			err = lmdbEnv.View(func(txn *lmdb.Txn) error {
				cur, err := txn.OpenCursor(lmdbDBI)
				if err != nil {
					return err
				}
				defer cur.Close()

				for {
					k1, v1, err := cur.Get(nil, nil, lmdb.Next)
					if lmdb.IsNotFound(err) {
						return nil
					}
					if err != nil {
						return err
					}

					//fmt.Printf("%s %s\n", k, v)

					// do some processing.
					k = safecopy(k, k1)
					v = safecopy(v, v1)

					count++
					print(count)
					if count > 2*Mi {
						break
					}
				}
				return nil
			})
			y.Check(err)
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func BenchmarkIterateBadgerOnlyKeys(b *testing.B) {
	bdb, err := getBadger()
	y.Check(err)
	k := make([]byte, 1024)
	b.ResetTimer()

	b.Run("badger-iterate-onlykeys", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			var count int
			// 100 = size, 0 = num workers, false = fwd direction.
			opt := badger.IteratorOptions{}
			opt.PrefetchSize = 10000
			itr := bdb.NewIterator(opt)
			for itr.Rewind(); itr.Valid(); itr.Next() {
				item := itr.Item()
				{
					// do some processing.
					k = safecopy(k, item.Key())
				}
				count++
				print(count)
				if count > 2*Mi {
					break
				}
			}
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func BenchmarkIterateBadgerWithValues(b *testing.B) {
	bdb, err := getBadger()
	y.Check(err)
	k := make([]byte, 1024)
	v := make([]byte, Mi)
	b.ResetTimer()

	b.Run("badger-iterate-withvals", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			var count int
			opt := badger.IteratorOptions{}
			opt.PrefetchSize = 10000
			opt.FetchValues = true
			itr := bdb.NewIterator(opt)
			for itr.Rewind(); itr.Valid(); itr.Next() {
				item := itr.Item()
				{
					// do some processing.
					k = safecopy(k, item.Key())
					v = safecopy(v, item.Value())
				}
				count++
				print(count)
				if count >= 2*Mi {
					break
				}
			}
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func TestMain(m *testing.M) {
	flag.Parse()
	// call flag.Parse() here if TestMain uses flags
	go http.ListenAndServe(":8080", nil)
	os.Exit(m.Run())
}
<|endoftext|>


<commit_before>package backend

import (
	"sync"
	"errors"
	"bytes"
	"fmt"
	"crypto/sha1"
	"bufio"

	"github.com/tsileo/datadatabase/encrypt"

	"code.google.com/p/go.crypto/nacl/secretbox"
)

var headerSize = 59

type EncryptBackend struct {
	dest BlobHandler
	// index map the plain text hash to encrypted hash
	index map[string]string

	key *[32]byte

	sync.Mutex
}

func NewEncryptBackend(keyPath string, dest BlobHandler) *EncryptBackend {
	if err := encrypt.LoadKey(keyPath); err != nil {
		panic(err)
	}
	b := &EncryptBackend{dest: dest, index:make(map[string]string), key:&encrypt.Key}
	hashes := make(chan string)
	errs := make(chan error)
	go func() {
		errs <- b.dest.Enumerate(hashes)

	}()
	for hash := range hashes {
		scanner := b.scanner(hash)
		plainHash, err := scanHash(scanner)
		if err != nil {
			panic(err)
			//return errors.New(fmt.Sprintf("Error reading plain hash from %v, %v", hash, err))
		}
		b.index[plainHash] = hash
	}
	if err := <-errs; err != nil {
		panic(err)
		//return err
	}
	return b
}

func (b *EncryptBackend) Put(hash string, data []byte) (err error) {
	// #datadb/secretbox\n
	// data hash\n
	// data
	var nonce [24]byte
	//out := make([]byte, len(data) + secretbox.Overhead + 24 + headerSize)
	if err := encrypt.GenerateNonce(&nonce); err != nil {
		return err
	}
	var out bytes.Buffer
	out.WriteString("#datadb/secretbox\n")
	out.WriteString(fmt.Sprintf("%v\n", hash))
	encData := make([]byte, len(data) + secretbox.Overhead)
	secretbox.Seal(encData[0:0], data, &nonce, b.key)
	out.Write(nonce[:])
	out.Write(encData)
	encSha1 := sha1.New()
	encSha1.Write(out.Bytes())
	encHash := fmt.Sprintf("%x", encSha1.Sum(nil))
	b.dest.Put(encHash ,out.Bytes())
	b.Lock()
	b.index[hash] = encHash
	defer b.Unlock()
	return
}

func (b *EncryptBackend) Exists(hash string) bool {
	b.Lock()
	defer b.Unlock()
	_, exists := b.index[hash]
	return exists
}

func (b *EncryptBackend) scanner(hash string) *bufio.Scanner {
	enc, err := b.dest.Get(hash)
	if err != nil {
		return nil
	}
	buf := bytes.NewBuffer(enc)
	return bufio.NewScanner(buf)
}

func scanHash(scanner *bufio.Scanner) (hash string, err error) {
	if !scanner.Scan() {
		return "", errors.New("No line to read")
	}
	if scanner.Text() != "#datadb/secretbox" {
		return "", errors.New("bad header")
	}
	if !scanner.Scan() {
		return "", errors.New("ref not found")
	}
	return scanner.Text(), nil
}

func (b *EncryptBackend) Get(hash string) (data []byte, err error) {
	ref, _ := b.index[hash]
	enc, err := b.dest.Get(ref)
	if err != nil {
		return data, err
	}
	box := enc[headerSize:]
	var nonce [24]byte
	encData := make([]byte, len(box) - 24)
	copy(nonce[:], box[:24])
	copy(encData[:], box[24:])
	out := make([]byte, len(box) - 24)
	out, success := secretbox.Open(nil, encData, &nonce, b.key)
	if !success {
		return data, fmt.Errorf("failed to decrypt blob %v/%v", hash, ref)
	}
	return out, nil
}

func (b *EncryptBackend) Enumerate(blobs chan<- string) error {
	defer close(blobs)
	for plainHash, _ := range b.index {
		blobs <- plainHash
	}
	return nil
}
<commit_msg>EncryptBackend cleanup<commit_after>package backend

import (
	"sync"
	"errors"
	"bytes"
	"fmt"
	"crypto/sha1"
	"bufio"

	"github.com/tsileo/datadatabase/encrypt"

	"code.google.com/p/go.crypto/nacl/secretbox"
	"code.google.com/p/snappy-go/snappy"
)

var headerSize = 59

type EncryptBackend struct {
	dest BlobHandler
	// index map the plain text hash to encrypted hash
	index map[string]string

	// holds the encryption key
	key *[32]byte

	sync.Mutex
}

// NewEncryptBackend return a backend that encrypt/decrypt blobs on the fly,
// blobs are compressed with snappy before encryption with nacl/secretbox.
// At startup it scan encrypted blobs to discover the plain hash (the hash of the plain-text/unencrypted data).
// Blobs are stored in the following format:
//
// #datadb/secretbox\n
// [plain hash]\n
// [encrypted data]
//
func NewEncryptBackend(keyPath string, dest BlobHandler) *EncryptBackend {
	if err := encrypt.LoadKey(keyPath); err != nil {
		panic(err)
	}
	b := &EncryptBackend{dest: dest, index:make(map[string]string), key:&encrypt.Key}
	// Scan the blobs to discover the plain text blob hashes and build the in-memory index
	hashes := make(chan string)
	errs := make(chan error)
	go func() {
		errs <- b.dest.Enumerate(hashes)

	}()
	for hash := range hashes {
		scanner := b.scanner(hash)
		plainHash, err := scanHash(scanner)
		if err != nil {
			panic(err)
			//return errors.New(fmt.Sprintf("Error reading plain hash from %v, %v", hash, err))
		}
		b.index[plainHash] = hash
	}
	if err := <-errs; err != nil {
		panic(err)
		//return err
	}
	return b
}

func (b *EncryptBackend) Put(hash string, rawData []byte) (err error) {
	// #datadb/secretbox\n
	// data hash\n
	// data
	var nonce [24]byte
	//out := make([]byte, len(data) + secretbox.Overhead + 24 + headerSize)
	if err := encrypt.GenerateNonce(&nonce); err != nil {
		return err
	}
	// First we compress the data with snappy
	data, err := snappy.Encode(nil, rawData)
	if err != nil {
		return
	}
	var out bytes.Buffer
	out.WriteString("#datadb/secretbox\n")
	out.WriteString(fmt.Sprintf("%v\n", hash))
	encData := make([]byte, len(data) + secretbox.Overhead)
	secretbox.Seal(encData[0:0], data, &nonce, b.key)
	out.Write(nonce[:])
	out.Write(encData)
	encSha1 := sha1.New()
	encSha1.Write(out.Bytes())
	encHash := fmt.Sprintf("%x", encSha1.Sum(nil))
	b.dest.Put(encHash ,out.Bytes())
	b.Lock()
	b.index[hash] = encHash
	defer b.Unlock()
	return
}

func (b *EncryptBackend) Exists(hash string) bool {
	b.Lock()
	defer b.Unlock()
	_, exists := b.index[hash]
	return exists
}

func (b *EncryptBackend) scanner(hash string) *bufio.Scanner {
	enc, err := b.dest.Get(hash)
	if err != nil {
		return nil
	}
	buf := bytes.NewBuffer(enc)
	return bufio.NewScanner(buf)
}

func scanHash(scanner *bufio.Scanner) (hash string, err error) {
	if !scanner.Scan() {
		return "", errors.New("No line to read")
	}
	if scanner.Text() != "#datadb/secretbox" {
		return "", errors.New("bad header")
	}
	if !scanner.Scan() {
		return "", errors.New("ref not found")
	}
	return scanner.Text(), nil
}

func (b *EncryptBackend) Get(hash string) (data []byte, err error) {
	ref, _ := b.index[hash]
	enc, err := b.dest.Get(ref)
	if err != nil {
		return data, err
	}
	box := enc[headerSize:]
	var nonce [24]byte
	encData := make([]byte, len(box) - 24)
	copy(nonce[:], box[:24])
	copy(encData[:], box[24:])
	out := make([]byte, len(box) - 24)
	out, success := secretbox.Open(nil, encData, &nonce, b.key)
	if !success {
		return data, fmt.Errorf("failed to decrypt blob %v/%v", hash, ref)
	}
	// Decode snappy data
	data, err = snappy.Decode(nil, out)
	if err != nil {
		return data, fmt.Errorf("failed to decode blob %v/%v", hash, ref)
	}
	return
}

func (b *EncryptBackend) Enumerate(blobs chan<- string) error {
	defer close(blobs)
	for plainHash, _ := range b.index {
		blobs <- plainHash
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backup

import (
	"fmt"
	"github.com/jacobsa/aws/sdb"
	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/crypto"
	"time"
)

// A record in the backup registry describing a successful backup job.
type CompletedJob struct {
	// The name of the backup job.
	Name string

	// The time at which the backup was started.
	StartTime time.Time

	// The score representing the contents of the backup.
	Score blob.Score
}

type Registry interface {
	// Record that the named backup job has completed.
	RecordBackup(j CompletedJob) (err error)

	// Return a list of the most recent completed backups.
	ListRecentBackups() (jobs []CompletedJob, err error)
}

// Create a registry that stores data in the supplied SimpleDB domain.
//
// Before doing so, check to see whether this domain has been used as a
// registry before. If not, write an encrypted marker with the supplied
// crypter. If it has been used before, make sure that it was used with a
// crypter compatible with the supplied one. This prevents accidentally writing
// data with the wrong key, as if the user entered the wrong password.
//
// The crypter must be set up such that it is guaranteed to return an error if
// it is used to decrypt ciphertext encrypted with a different key. In that
// case, this function will return an *IncompatibleCrypterError.
func NewRegistry(
	crypter crypto.Crypter,
	domain sdb.Domain) (r Registry, err error) {
	err = fmt.Errorf("TODO")
	return
}

type registry struct {
	crypter crypto.Crypter
	domain sdb.Domain
}

func (r *registry) RecordBackup(j CompletedJob) (err error) {
	err = fmt.Errorf("TODO")
	return
}

func (r *registry) ListRecentBackups() (jobs []CompletedJob, err error) {
	err = fmt.Errorf("TODO")
	return
}

type IncompatibleCrypterError struct {
}

func (e *IncompatibleCrypterError) Error() string {
	return "TODO"
}
<commit_msg>Implemented NewRegistry.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backup

import (
	"fmt"
	"github.com/jacobsa/aws/sdb"
	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/crypto"
	"time"
)

const (
	// The item name we use for "domain is already in use" markers.
	markerItemName = "comeback_marker"

	// The attribute name we use for storing crypto-compatibility data.
	cryptoMarkerAttributeName = "encrypted_data"
)

// A record in the backup registry describing a successful backup job.
type CompletedJob struct {
	// The name of the backup job.
	Name string

	// The time at which the backup was started.
	StartTime time.Time

	// The score representing the contents of the backup.
	Score blob.Score
}

type Registry interface {
	// Record that the named backup job has completed.
	RecordBackup(j CompletedJob) (err error)

	// Return a list of the most recent completed backups.
	ListRecentBackups() (jobs []CompletedJob, err error)
}

// Create a registry that stores data in the supplied SimpleDB domain.
//
// Before doing so, check to see whether this domain has been used as a
// registry before. If not, write an encrypted marker with the supplied
// crypter. If it has been used before, make sure that it was used with a
// crypter compatible with the supplied one. This prevents accidentally writing
// data with the wrong key, as if the user entered the wrong password.
//
// The crypter must be set up such that it is guaranteed to return an error if
// it is used to decrypt ciphertext encrypted with a different key. In that
// case, this function will return an *IncompatibleCrypterError.
func NewRegistry(
	crypter crypto.Crypter,
	domain sdb.Domain) (r Registry, err error) {
	// Set up a tentative result.
	r = &registry{crypter, domain}

	// Ask for the data that will tell us whether the crypter is compatible with
	// the previous one used in this domain, if any.
	attrs, err := domain.GetAttributes(
		markerItemName,
		false,  // No need to ask for a consistent read
		[]string{cryptoMarkerAttributeName},
	)

	if err != nil {
		err = fmt.Errorf("GetAttributes: %v", err)
		return
	}

	// If we got back an attribute and its data is decryptable, we're done.
	if len(attrs) > 0 {
		ciphertext := []byte(attrs[0].Value)
		if _, err = crypter.Decrypt(ciphertext); err != nil {
			err = fmt.Errorf("Decrypt: %v", err)
			return
		}

		return
	}

	// Otherwise, we want to claim this domain. Encrypt some random data, then
	// write it out. Make sure to use a precondition to defeat the race condition
	// where another machine is doing the same simultaneously.
	plaintext := getRandBytes()
	ciphertext, err := crypter.Encrypt(plaintext)
	if err != nil {
		err = fmt.Errorf("Encrypt: %v", err)
		return
	}

	err = domain.PutAttributes(
		markerItemName,
		[]sdb.PutUpdate{
			sdb.PutUpdate{Name: cryptoMarkerAttributeName, Value: string(ciphertext)},
		},
		&sdb.Precondition{Name: cryptoMarkerAttributeName, Value: nil},
	)

	if err != nil {
		err = fmt.Errorf("PutAttributes: %v", err)
		return
	}

	// All is good.
	return
}

func getRandBytes() []byte

type registry struct {
	crypter crypto.Crypter
	domain sdb.Domain
}

func (r *registry) RecordBackup(j CompletedJob) (err error) {
	err = fmt.Errorf("TODO")
	return
}

func (r *registry) ListRecentBackups() (jobs []CompletedJob, err error) {
	err = fmt.Errorf("TODO")
	return
}

type IncompatibleCrypterError struct {
}

func (e *IncompatibleCrypterError) Error() string {
	return "TODO"
}
<|endoftext|>


<commit_before>package gomol

import . "gopkg.in/check.v1"

func (s *GomolSuite) BenchmarkBasicLogInsertion(c *C) {
	b := NewBase()
	b.SetAttr("attr1", 1234)

	l := newDefaultMemLogger()
	b.AddLogger(l)
	b.InitLoggers()

	for idx := 0; idx < c.N; idx++ {
		b.log(
			LevelDebug,
			NewAttrs().
				SetAttr("attr2", 4321).
				SetAttr("attr3", "val3").
				SetAttr("attr4", 1234),
			"test %v",
			1234)
	}
	b.ShutdownLoggers()
}

func (s *GomolSuite) BenchmarkBaseDbgm(c *C) {
	b := NewBase()
	b.SetAttr("attr1", 1234)

	l := newDefaultMemLogger()
	b.AddLogger(l)
	b.InitLoggers()

	for idx := 0; idx < c.N; idx++ {
		b.Dbgm(
			NewAttrs().
				SetAttr("attr2", 4321).
				SetAttr("attr3", "val3").
				SetAttr("attr4", 1234),
			"test %v",
			1234)
	}
	b.ShutdownLoggers()
}

func (s *GomolSuite) BenchmarkLogInsertionWithFilename(c *C) {
	base := NewBase()
	base.config.FilenameAttr = "filename"
	base.InitLoggers()
	for i := 0; i < c.N; i++ {
		base.log(LevelDebug,
			NewAttrs().
				SetAttr("attr1", "val1").
				SetAttr("attr2", "val2").
				SetAttr("attr3", 3).
				SetAttr("attr4", 4),
			"msg %v %v", "string", 1234)
	}
	base.ShutdownLoggers()
}
func (s *GomolSuite) BenchmarkLogInsertionWithLineNumber(c *C) {
	base := NewBase()
	base.config.LineNumberAttr = "line"
	base.InitLoggers()
	for i := 0; i < c.N; i++ {
		base.log(LevelDebug,
			NewAttrs().
				SetAttr("attr1", "val1").
				SetAttr("attr2", "val2").
				SetAttr("attr3", 3).
				SetAttr("attr4", 4),
			"msg %v %v", "string", 1234)
	}
	base.ShutdownLoggers()
}
func (s *GomolSuite) BenchmarkLogInsertionWithFilenameAndLineNumber(c *C) {
	base := NewBase()
	base.config.FilenameAttr = "filename"
	base.config.LineNumberAttr = "line"
	base.InitLoggers()
	for i := 0; i < c.N; i++ {
		base.log(LevelDebug,
			NewAttrs().
				SetAttr("attr1", "val1").
				SetAttr("attr2", "val2").
				SetAttr("attr3", 3).
				SetAttr("attr4", 4),
			"msg %v %v", "string", 1234)
	}
	base.ShutdownLoggers()
}

func (s *GomolSuite) BenchmarkIsGomolCaller(c *C) {
	for i := 0; i < c.N; i++ {
		isGomolCaller("/home/gomoltest/some/sub/dir/that/is/long/filename.go")
	}
}

func (s *GomolSuite) BenchmarkGetCallerInfo(c *C) {
	for i := 0; i < c.N; i++ {
		getCallerInfo()
	}
}
<commit_msg>Add a couple more benchmarks<commit_after>package gomol

import . "gopkg.in/check.v1"

func (s *GomolSuite) BenchmarkBasicLogInsertion(c *C) {
	b := NewBase()
	b.SetAttr("attr1", 1234)

	l := newDefaultMemLogger()
	b.AddLogger(l)
	b.InitLoggers()

	for idx := 0; idx < c.N; idx++ {
		b.log(
			LevelDebug,
			NewAttrs().
				SetAttr("attr2", 4321).
				SetAttr("attr3", "val3").
				SetAttr("attr4", 1234),
			"test %v",
			1234)
	}
	b.ShutdownLoggers()
}

func (s *GomolSuite) BenchmarkBaseDbgm(c *C) {
	b := NewBase()
	b.SetAttr("attr1", 1234)

	l := newDefaultMemLogger()
	b.AddLogger(l)
	b.InitLoggers()

	for idx := 0; idx < c.N; idx++ {
		b.Dbgm(
			NewAttrs().
				SetAttr("attr2", 4321).
				SetAttr("attr3", "val3").
				SetAttr("attr4", 1234),
			"test %v",
			1234)
	}
	b.ShutdownLoggers()
}

func (s *GomolSuite) BenchmarkBaseStaticStringNoLogger(c *C) {
	b := NewBase()

	b.InitLoggers()

	c.ResetTimer()
	for idx := 0; idx < c.N; idx++ {
		b.Info("Hi I'm a log!")
	}
	b.ShutdownLoggers()
}

func (s *GomolSuite) BenchmarkBaseStaticStringMemLogger(c *C) {
	b := NewBase()

	l := newDefaultMemLogger()
	b.AddLogger(l)
	b.InitLoggers()

	c.ResetTimer()
	for idx := 0; idx < c.N; idx++ {
		b.Info("Hi I'm a log!")
	}
	b.ShutdownLoggers()
}

func (s *GomolSuite) BenchmarkLogInsertionWithFilename(c *C) {
	base := NewBase()
	base.config.FilenameAttr = "filename"
	base.InitLoggers()
	for i := 0; i < c.N; i++ {
		base.log(LevelDebug,
			NewAttrs().
				SetAttr("attr1", "val1").
				SetAttr("attr2", "val2").
				SetAttr("attr3", 3).
				SetAttr("attr4", 4),
			"msg %v %v", "string", 1234)
	}
	base.ShutdownLoggers()
}
func (s *GomolSuite) BenchmarkLogInsertionWithLineNumber(c *C) {
	base := NewBase()
	base.config.LineNumberAttr = "line"
	base.InitLoggers()
	for i := 0; i < c.N; i++ {
		base.log(LevelDebug,
			NewAttrs().
				SetAttr("attr1", "val1").
				SetAttr("attr2", "val2").
				SetAttr("attr3", 3).
				SetAttr("attr4", 4),
			"msg %v %v", "string", 1234)
	}
	base.ShutdownLoggers()
}
func (s *GomolSuite) BenchmarkLogInsertionWithFilenameAndLineNumber(c *C) {
	base := NewBase()
	base.config.FilenameAttr = "filename"
	base.config.LineNumberAttr = "line"
	base.InitLoggers()
	for i := 0; i < c.N; i++ {
		base.log(LevelDebug,
			NewAttrs().
				SetAttr("attr1", "val1").
				SetAttr("attr2", "val2").
				SetAttr("attr3", 3).
				SetAttr("attr4", 4),
			"msg %v %v", "string", 1234)
	}
	base.ShutdownLoggers()
}

func (s *GomolSuite) BenchmarkIsGomolCaller(c *C) {
	for i := 0; i < c.N; i++ {
		isGomolCaller("/home/gomoltest/some/sub/dir/that/is/long/filename.go")
	}
}

func (s *GomolSuite) BenchmarkGetCallerInfo(c *C) {
	for i := 0; i < c.N; i++ {
		getCallerInfo()
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 ECS Team, Inc. - All Rights Reserved
// https://github.com/ECSTeam/cloudfoundry-top-plugin
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ui

import (
	"errors"
	"fmt"
	"regexp"

	"github.com/ecsteam/cloudfoundry-top-plugin/ui/masterUIInterface"
	"github.com/jroimartin/gocui"
)

type HelpTextTipsWidget struct {
	masterUI        masterUIInterface.MasterUIInterface
	name            string
	height          int
	formatTextRegex *regexp.Regexp
	helpTextTips    string
}

func NewHelpTextTipsWidget(masterUI masterUIInterface.MasterUIInterface, name string, height int) *HelpTextTipsWidget {
	w := &HelpTextTipsWidget{masterUI: masterUI, name: name, height: height}
	w.formatTextRegex = regexp.MustCompile(`\*\*([^\*]*)*\*\*`)
	return w
}

func (w *HelpTextTipsWidget) Name() string {
	return w.name
}

func (w *HelpTextTipsWidget) Layout(g *gocui.Gui) error {
	maxX, maxY := g.Size()
	v, err := g.SetView(w.name, 0, maxY-w.height, maxX-1, maxY)
	if err != nil {
		if err != gocui.ErrUnknownView {
			return errors.New(w.name + " layout error:" + err.Error())
		}
		v.Frame = false
		return w.showHelpTextTips(g, v)
	}
	return nil
}

func (w *HelpTextTipsWidget) SetHelpTextTips(g *gocui.Gui, helpTextTips string) error {

	w.helpTextTips = w.formatText(helpTextTips)
	v, err := g.View(w.name)
	if err != nil {
		return err
	}
	return w.showHelpTextTips(g, v)
}

func (w *HelpTextTipsWidget) showHelpTextTips(g *gocui.Gui, v *gocui.View) error {
	v.Clear()
	fmt.Fprint(v, w.helpTextTips)
	return nil
}

func (w *HelpTextTipsWidget) formatText(text string) string {
	return w.formatTextRegex.ReplaceAllString(text, "\033[37;1m${1}\033[0m")
}
<commit_msg>Fix issue with terminal window with zero dimention causing panic because helpTextTipsView layout error:invalid dimensions<commit_after>// Copyright (c) 2016 ECS Team, Inc. - All Rights Reserved
// https://github.com/ECSTeam/cloudfoundry-top-plugin
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ui

import (
	"errors"
	"fmt"
	"regexp"

	"github.com/ecsteam/cloudfoundry-top-plugin/ui/masterUIInterface"
	"github.com/jroimartin/gocui"
)

type HelpTextTipsWidget struct {
	masterUI        masterUIInterface.MasterUIInterface
	name            string
	height          int
	formatTextRegex *regexp.Regexp
	helpTextTips    string
}

func NewHelpTextTipsWidget(masterUI masterUIInterface.MasterUIInterface, name string, height int) *HelpTextTipsWidget {
	w := &HelpTextTipsWidget{masterUI: masterUI, name: name, height: height}
	w.formatTextRegex = regexp.MustCompile(`\*\*([^\*]*)*\*\*`)
	return w
}

func (w *HelpTextTipsWidget) Name() string {
	return w.name
}

func (w *HelpTextTipsWidget) Layout(g *gocui.Gui) error {
	maxX, maxY := g.Size()
	top := maxY - w.height
	if top < 0 {
		top = 0
	}
	right := maxX - 1
	if right < 1 {
		right = 1
	}

	v, err := g.SetView(w.name, 0, top, right, maxY)
	if err != nil {
		if err != gocui.ErrUnknownView {
			return errors.New(w.name + " layout error:" + err.Error())
		}
		v.Frame = false
		return w.showHelpTextTips(g, v)
	}
	return nil
}

func (w *HelpTextTipsWidget) SetHelpTextTips(g *gocui.Gui, helpTextTips string) error {

	w.helpTextTips = w.formatText(helpTextTips)
	v, err := g.View(w.name)
	if err != nil {
		return err
	}
	return w.showHelpTextTips(g, v)
}

func (w *HelpTextTipsWidget) showHelpTextTips(g *gocui.Gui, v *gocui.View) error {
	v.Clear()
	fmt.Fprint(v, w.helpTextTips)
	return nil
}

func (w *HelpTextTipsWidget) formatText(text string) string {
	return w.formatTextRegex.ReplaceAllString(text, "\033[37;1m${1}\033[0m")
}
<|endoftext|>


<commit_before>package jiralert

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/andygrunwald/go-jira"
	"github.com/free/jiralert/alertmanager"
	log "github.com/golang/glog"
	"github.com/trivago/tgo/tcontainer"
)

// Receiver wraps a JIRA client corresponding to a specific Alertmanager receiver, with its configuration and templates.
type Receiver struct {
	conf   *ReceiverConfig
	tmpl   *Template
	client *jira.Client
}

// NewReceiver creates a Receiver using the provided configuration and template.
func NewReceiver(c *ReceiverConfig, t *Template) (*Receiver, error) {
	client, err := jira.NewClient(http.DefaultClient, c.APIURL)
	if err != nil {
		return nil, err
	}
	client.Authentication.SetBasicAuth(c.User, string(c.Password))

	return &Receiver{conf: c, tmpl: t, client: client}, nil
}

// Notify implements the Notifier interface.
func (r *Receiver) Notify(data *alertmanager.Data) (bool, error) {
	project := r.tmpl.Execute(r.conf.Project, data)
	// check errors from r.tmpl.Execute()
	if r.tmpl.err != nil {
		return false, r.tmpl.err
	}
	// Looks like an ALERT metric name, with spaces removed.
	issueLabel := toIssueLabel(data.GroupLabels)

	issue, retry, err := r.search(project, issueLabel)
	if err != nil {
		return retry, err
	}

	if issue != nil {
		// The set of JIRA status categories is fixed, this is a safe check to make.
		if issue.Fields.Status.StatusCategory.Key != "done" {
			// Issue is in a "to do" or "in progress" state, all done here.
			log.V(1).Infof("Issue %s for %s is unresolved, nothing to do", issue.Key, issueLabel)
			return false, nil
		}
		if r.conf.WontFixResolution != "" && issue.Fields.Resolution.Name == r.conf.WontFixResolution {
			// Issue is resolved as "Won't Fix" or equivalent, log a message just in case.
			log.Infof("Issue %s for %s is resolved as %q, not reopening", issue.Key, issueLabel, issue.Fields.Resolution.Name)
			return false, nil
		}
		log.Infof("Issue %s for %s was resolved, reopening", issue.Key, issueLabel)
		return r.reopen(issue.Key)
	}

	log.Infof("No issue matching %s found, creating new issue", issueLabel)
	issue = &jira.Issue{
		Fields: &jira.IssueFields{
			Project:     jira.Project{Key: project},
			Type:        jira.IssueType{Name: r.tmpl.Execute(r.conf.IssueType, data)},
			Description: r.tmpl.Execute(r.conf.Description, data),
			Summary:     r.tmpl.Execute(r.conf.Summary, data),
			Labels: []string{
				issueLabel,
			},
			Unknowns: tcontainer.NewMarshalMap(),
		},
	}
	if r.conf.Priority != "" {
		issue.Fields.Priority = &jira.Priority{Name: r.tmpl.Execute(r.conf.Priority, data)}
	}

	// Add Components
	issue.Fields.Components = make([]*jira.Component, 0, len(r.conf.Components))
	for _, component := range r.conf.Components {
		issue.Fields.Components = append(issue.Fields.Components, &jira.Component{Name: component})
	}

	// Add Labels
	if r.conf.CopyPromLabels {
		for k, v := range data.GroupLabels {
			issue.Fields.Labels = append(issue.Fields.Labels, fmt.Sprintf("%s=\"%s\"", k, v))
		}
	}

	for key, value := range r.conf.Fields {
		issue.Fields.Unknowns[key] = r.tmpl.Execute(fmt.Sprint(value), data)
	}
	// check errors from r.tmpl.Execute()
	if r.tmpl.err != nil {
		return false, r.tmpl.err
	}
	retry, err = r.create(issue)
	if err != nil {
		log.Infof("Issue created: key=%s ID=%s", issue.Key, issue.ID)
	}
	return retry, err
}

// toIssueLabel returns the group labels in the form of an ALERT metric name, with all spaces removed.
func toIssueLabel(groupLabels alertmanager.KV) string {
	buf := bytes.NewBufferString("ALERT{")
	for _, p := range groupLabels.SortedPairs() {
		buf.WriteString(p.Name)
		buf.WriteString(fmt.Sprintf("=%q,", p.Value))
	}
	buf.Truncate(buf.Len() - 1)
	buf.WriteString("}")
	return strings.Replace(buf.String(), " ", "", -1)
}

func (r *Receiver) search(project, issueLabel string) (*jira.Issue, bool, error) {
	query := fmt.Sprintf("project=%s and labels=%q order by key", project, issueLabel)
	options := &jira.SearchOptions{
		Fields:     []string{"summary", "status", "resolution"},
		MaxResults: 50,
	}
	log.V(1).Infof("search: query=%v options=%+v", query, options)
	issues, resp, err := r.client.Issue.Search(query, options)
	if err != nil {
		retry, err := handleJiraError(resp, err)
		return nil, retry, err
	}
	if len(issues) > 0 {
		if len(issues) > 1 {
			// Swallow it, but log an error.
			log.Errorf("More than one issue matched %s, will only update first: %+v", query, issues)
		}
		log.V(1).Infof("  found: %+v", issues[0])
		return &issues[0], false, nil
	}
	log.V(1).Infof("  no results")
	return nil, false, nil
}

func (r *Receiver) reopen(issueKey string) (bool, error) {
	transitions, resp, err := r.client.Issue.GetTransitions(issueKey)
	if err != nil {
		return handleJiraError(resp, err)
	}
	for _, t := range transitions {
		if t.Name == r.conf.ReopenState {
			log.V(1).Infof("reopen: issueKey=%v transitionID=%v", issueKey, t.ID)
			resp, err = r.client.Issue.DoTransition(issueKey, t.ID)
			if err != nil {
				return handleJiraError(resp, err)
			}
			log.V(1).Infof("  done")
			return false, nil
		}
	}
	return false, fmt.Errorf("JIRA state %q does not exist or no transition possible for %s", r.conf.ReopenState, issueKey)
}

func (r *Receiver) create(issue *jira.Issue) (bool, error) {
	log.V(1).Infof("create: issue=%v", *issue)
	issue, resp, err := r.client.Issue.Create(issue)
	if err != nil {
		return handleJiraError(resp, err)
	}

	log.V(1).Infof("  done: key=%s ID=%s", issue.Key, issue.ID)
	return false, nil
}

func handleJiraError(resp *jira.Response, err error) (bool, error) {
	log.V(1).Infof("handleJiraError: err=%s, req=%s", err, resp.Request.URL)
	if resp != nil && resp.StatusCode/100 != 2 {
		retry := resp.StatusCode == 500 || resp.StatusCode == 503
		body, _ := ioutil.ReadAll(resp.Body)
		// go-jira error message is not particularly helpful, replace it
		return retry, fmt.Errorf("JIRA request %s returned status %s, body %q", resp.Request.URL, resp.Status, string(body))
	}
	return false, err
}
<commit_msg>Improving format string<commit_after>package jiralert

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/andygrunwald/go-jira"
	"github.com/free/jiralert/alertmanager"
	log "github.com/golang/glog"
	"github.com/trivago/tgo/tcontainer"
)

// Receiver wraps a JIRA client corresponding to a specific Alertmanager receiver, with its configuration and templates.
type Receiver struct {
	conf   *ReceiverConfig
	tmpl   *Template
	client *jira.Client
}

// NewReceiver creates a Receiver using the provided configuration and template.
func NewReceiver(c *ReceiverConfig, t *Template) (*Receiver, error) {
	client, err := jira.NewClient(http.DefaultClient, c.APIURL)
	if err != nil {
		return nil, err
	}
	client.Authentication.SetBasicAuth(c.User, string(c.Password))

	return &Receiver{conf: c, tmpl: t, client: client}, nil
}

// Notify implements the Notifier interface.
func (r *Receiver) Notify(data *alertmanager.Data) (bool, error) {
	project := r.tmpl.Execute(r.conf.Project, data)
	// check errors from r.tmpl.Execute()
	if r.tmpl.err != nil {
		return false, r.tmpl.err
	}
	// Looks like an ALERT metric name, with spaces removed.
	issueLabel := toIssueLabel(data.GroupLabels)

	issue, retry, err := r.search(project, issueLabel)
	if err != nil {
		return retry, err
	}

	if issue != nil {
		// The set of JIRA status categories is fixed, this is a safe check to make.
		if issue.Fields.Status.StatusCategory.Key != "done" {
			// Issue is in a "to do" or "in progress" state, all done here.
			log.V(1).Infof("Issue %s for %s is unresolved, nothing to do", issue.Key, issueLabel)
			return false, nil
		}
		if r.conf.WontFixResolution != "" && issue.Fields.Resolution.Name == r.conf.WontFixResolution {
			// Issue is resolved as "Won't Fix" or equivalent, log a message just in case.
			log.Infof("Issue %s for %s is resolved as %q, not reopening", issue.Key, issueLabel, issue.Fields.Resolution.Name)
			return false, nil
		}
		log.Infof("Issue %s for %s was resolved, reopening", issue.Key, issueLabel)
		return r.reopen(issue.Key)
	}

	log.Infof("No issue matching %s found, creating new issue", issueLabel)
	issue = &jira.Issue{
		Fields: &jira.IssueFields{
			Project:     jira.Project{Key: project},
			Type:        jira.IssueType{Name: r.tmpl.Execute(r.conf.IssueType, data)},
			Description: r.tmpl.Execute(r.conf.Description, data),
			Summary:     r.tmpl.Execute(r.conf.Summary, data),
			Labels: []string{
				issueLabel,
			},
			Unknowns: tcontainer.NewMarshalMap(),
		},
	}
	if r.conf.Priority != "" {
		issue.Fields.Priority = &jira.Priority{Name: r.tmpl.Execute(r.conf.Priority, data)}
	}

	// Add Components
	issue.Fields.Components = make([]*jira.Component, 0, len(r.conf.Components))
	for _, component := range r.conf.Components {
		issue.Fields.Components = append(issue.Fields.Components, &jira.Component{Name: component})
	}

	// Add Labels
	if r.conf.CopyPromLabels {
		for k, v := range data.GroupLabels {
			issue.Fields.Labels = append(issue.Fields.Labels, fmt.Sprintf("%s=\"%q\"", k, v))
		}
	}

	for key, value := range r.conf.Fields {
		issue.Fields.Unknowns[key] = r.tmpl.Execute(fmt.Sprint(value), data)
	}
	// check errors from r.tmpl.Execute()
	if r.tmpl.err != nil {
		return false, r.tmpl.err
	}
	retry, err = r.create(issue)
	if err != nil {
		log.Infof("Issue created: key=%s ID=%s", issue.Key, issue.ID)
	}
	return retry, err
}

// toIssueLabel returns the group labels in the form of an ALERT metric name, with all spaces removed.
func toIssueLabel(groupLabels alertmanager.KV) string {
	buf := bytes.NewBufferString("ALERT{")
	for _, p := range groupLabels.SortedPairs() {
		buf.WriteString(p.Name)
		buf.WriteString(fmt.Sprintf("=%q,", p.Value))
	}
	buf.Truncate(buf.Len() - 1)
	buf.WriteString("}")
	return strings.Replace(buf.String(), " ", "", -1)
}

func (r *Receiver) search(project, issueLabel string) (*jira.Issue, bool, error) {
	query := fmt.Sprintf("project=%s and labels=%q order by key", project, issueLabel)
	options := &jira.SearchOptions{
		Fields:     []string{"summary", "status", "resolution"},
		MaxResults: 50,
	}
	log.V(1).Infof("search: query=%v options=%+v", query, options)
	issues, resp, err := r.client.Issue.Search(query, options)
	if err != nil {
		retry, err := handleJiraError(resp, err)
		return nil, retry, err
	}
	if len(issues) > 0 {
		if len(issues) > 1 {
			// Swallow it, but log an error.
			log.Errorf("More than one issue matched %s, will only update first: %+v", query, issues)
		}
		log.V(1).Infof("  found: %+v", issues[0])
		return &issues[0], false, nil
	}
	log.V(1).Infof("  no results")
	return nil, false, nil
}

func (r *Receiver) reopen(issueKey string) (bool, error) {
	transitions, resp, err := r.client.Issue.GetTransitions(issueKey)
	if err != nil {
		return handleJiraError(resp, err)
	}
	for _, t := range transitions {
		if t.Name == r.conf.ReopenState {
			log.V(1).Infof("reopen: issueKey=%v transitionID=%v", issueKey, t.ID)
			resp, err = r.client.Issue.DoTransition(issueKey, t.ID)
			if err != nil {
				return handleJiraError(resp, err)
			}
			log.V(1).Infof("  done")
			return false, nil
		}
	}
	return false, fmt.Errorf("JIRA state %q does not exist or no transition possible for %s", r.conf.ReopenState, issueKey)
}

func (r *Receiver) create(issue *jira.Issue) (bool, error) {
	log.V(1).Infof("create: issue=%v", *issue)
	issue, resp, err := r.client.Issue.Create(issue)
	if err != nil {
		return handleJiraError(resp, err)
	}

	log.V(1).Infof("  done: key=%s ID=%s", issue.Key, issue.ID)
	return false, nil
}

func handleJiraError(resp *jira.Response, err error) (bool, error) {
	log.V(1).Infof("handleJiraError: err=%s, req=%s", err, resp.Request.URL)
	if resp != nil && resp.StatusCode/100 != 2 {
		retry := resp.StatusCode == 500 || resp.StatusCode == 503
		body, _ := ioutil.ReadAll(resp.Body)
		// go-jira error message is not particularly helpful, replace it
		return retry, fmt.Errorf("JIRA request %s returned status %s, body %q", resp.Request.URL, resp.Status, string(body))
	}
	return false, err
}
<|endoftext|>


<commit_before>package channel

import (
	"encoding/hex"
	"errors"
	"fmt"
	"log"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/codedust/go-tox"
	"github.com/xamino/tox-dynboot"
)

/*
Channel is a wrapper of the gotox wrapper that creates and manages the underlying Tox
instance.

TODO all callbacks will block, need to avoid that especially when user interaction is required
*/
type Channel struct {
	tox                *gotox.Tox
	callbacks          Callbacks
	wg                 sync.WaitGroup
	stop               chan bool
	transfers          map[uint32]*os.File
	transfersFilesizes map[uint32]uint64
}

/*
Create and starts a new tox channel that continously runs in the background
until this object is destroyed.
*/
func Create(name string, toxdata []byte, callbacks Callbacks) (*Channel, error) {
	if name == "" {
		return nil, errors.New("CreateChannel called with no name!")
	}
	var init bool
	var channel = &Channel{}
	var options *gotox.Options
	var err error

	// prepare for file transfers
	channel.transfers = make(map[uint32]*os.File)
	channel.transfersFilesizes = make(map[uint32]uint64)

	// this decides whether we are initiating a new connection or using an existing one
	if toxdata == nil {
		options = &gotox.Options{
			true, true,
			gotox.TOX_PROXY_TYPE_NONE, "127.0.0.1", 5555, 0, 0, 0,
			gotox.TOX_SAVEDATA_TYPE_NONE, nil}
		init = true
	} else {
		options = &gotox.Options{
			true, true,
			gotox.TOX_PROXY_TYPE_NONE, "127.0.0.1", 5555, 0, 0, 0,
			gotox.TOX_SAVEDATA_TYPE_TOX_SAVE, toxdata}
		init = false
	}
	channel.tox, err = gotox.New(options)
	if err != nil {
		return nil, err
	}
	if init {
		channel.tox.SelfSetName(name)
		channel.tox.SelfSetStatusMessage("Tin Peer")
	}
	err = channel.tox.SelfSetStatus(gotox.TOX_USERSTATUS_NONE)
	// Register our callbacks
	channel.tox.CallbackFriendRequest(channel.onFriendRequest)
	channel.tox.CallbackFriendMessage(channel.onFriendMessage)
	channel.tox.CallbackFileRecvControl(channel.onFileRecvControl)
	channel.tox.CallbackFileRecv(channel.onFileRecv)
	channel.tox.CallbackFileRecvChunk(channel.onFileRecvChunk)
	channel.tox.CallbackFileChunkRequest(channel.onFileChunkRequest)
	// some things must only be done if first start
	if init {
		// Bootstrap
		toxNode, err := toxdynboot.FetchFirstAlive(200 * time.Millisecond)
		if err != nil {
			return nil, err
		}
		err = channel.tox.Bootstrap(toxNode.IPv4, toxNode.Port, toxNode.PublicKey)
		if err != nil {
			return nil, err
		}
	}
	// register callbacks
	channel.callbacks = callbacks
	// now to run it:
	channel.wg.Add(1)
	channel.stop = make(chan bool, 1)
	go channel.run()
	return channel, nil
}

// --- public methods here ---

/*
Close shuts down the channel.
*/
func (channel *Channel) Close() {
	// send stop signal
	channel.stop <- false
	// wait for it to close
	channel.wg.Wait()
	// kill tox
	channel.tox.Kill()
}

/*
Address of the Tox instance.
*/
func (channel *Channel) Address() (string, error) {
	address, err := channel.tox.SelfGetAddress()
	if err != nil {
		return "", err
	}
	return strings.ToUpper(hex.EncodeToString(address)), nil
}

/*
ToxData returns the underlying current representation of the tox data. Can be
used to store a Tox instance to disk.
*/
func (channel *Channel) ToxData() ([]byte, error) {
	return channel.tox.GetSavedata()
}

/*
Send a message to the given peer address.
*/
func (channel *Channel) Send(address, message string) error {
	if ok, _ := channel.IsOnline(address); !ok {
		return errOffline
	}
	// find friend id to send to
	key, err := hex.DecodeString(address)
	if err != nil {
		return err
	}
	id, err := channel.tox.FriendByPublicKey(key)
	if err != nil {
		return err
	}
	// returns message ID but we currently don't use it
	_, err = channel.tox.FriendSendMessage(id, gotox.TOX_MESSAGE_TYPE_NORMAL, message)
	return err
}

/*
SendFile sends a file to the given address. NOTE: Will block until done!
*/
func (channel *Channel) SendFile(address string, path string, identification string) error {
	if ok, _ := channel.IsOnline(address); !ok {
		return errOffline
	}
	// find friend id to send to
	key, err := hex.DecodeString(address)
	if err != nil {
		return err
	}
	id, err := channel.tox.FriendByPublicKey(key)
	if err != nil {
		return err
	}
	// get file
	file, err := os.Open(path)
	if err != nil {
		return err
	}
	// do NOT close file! must be done elsewhere since we may need it later
	// get file size
	stat, err := os.Lstat(path)
	if err != nil {
		file.Close()
		return err
	}
	size := uint64(stat.Size())
	// prepare send (file will be transmitted via filechunk)
	fileNumber, err := channel.tox.FileSend(id, gotox.TOX_FILE_KIND_DATA, size, nil, identification)
	if err != nil {
		file.Close()
		return err
	}
	channel.transfers[fileNumber] = file
	channel.transfersFilesizes[fileNumber] = size
	return nil
}

/*
AcceptConnection accepts the given address as a connection partner.
*/
func (channel *Channel) AcceptConnection(address string) error {
	publicKey, err := hex.DecodeString(address)
	if err != nil {
		return err
	}
	// ignore friendnumber
	_, err = channel.tox.FriendAddNorequest(publicKey)
	return err
}

/*
RequestConnection sends a friend request to the given address with the sending
peer information as the message for bootstrapping.
*/
func (channel *Channel) RequestConnection(address, message string) error {
	publicKey, err := hex.DecodeString(address)
	if err != nil {
		return err
	}
	// send non blocking friend request
	_, err = channel.tox.FriendAdd(publicKey, message)
	return err
}

/*
IsOnline checks whether the given address is currently reachable.
*/
func (channel *Channel) IsOnline(address string) (bool, error) {
	publicKey, err := hex.DecodeString(address)
	if err != nil {
		return false, err
	}
	num, err := channel.tox.FriendByPublicKey(publicKey)
	if err != nil {
		return false, err
	}
	status, err := channel.tox.FriendGetConnectionStatus(num)
	if err != nil {
		return false, err
	}
	return status != gotox.TOX_CONNECTION_NONE, nil
}

/*
NameOf the key associated to the given address.
*/
func (channel *Channel) NameOf(address string) (string, error) {
	publicKey, err := hex.DecodeString(address)
	if err != nil {
		return "", err
	}
	num, err := channel.tox.FriendByPublicKey(publicKey)
	if err != nil {
		return "", err
	}
	name, err := channel.tox.FriendGetName(num)
	if err != nil {
		return "", err
	}
	return name, nil
}

// --- private methods here ---

/*
run is the background go routine method that keeps the Tox instance iterating
until Close() is called.
*/
func (channel *Channel) run() {
	for {
		temp, _ := channel.tox.IterationInterval()
		intervall := time.Duration(temp) * time.Millisecond
		select {
		case <-channel.stop:
			channel.wg.Done()
			return
		case <-time.Tick(intervall):
			err := channel.tox.Iterate()
			if err != nil {
				// TODO what do we do here? Can we cleanly close the channel and
				// catch the error further up?
				log.Println(err.Error())
			}
		} // select
	} // for
}

/*
addressOf given friend number.
*/
func (channel *Channel) addressOf(friendnumber uint32) (string, error) {
	publicKey, err := channel.tox.FriendGetPublickey(friendnumber)
	if err != nil {
		return "", errLostAddress
	}
	return hex.EncodeToString(publicKey), nil
}

/*
onFriendRequest calls the appropriate callback, wrapping it sanely for our purposes.
*/
func (channel *Channel) onFriendRequest(_ *gotox.Tox, publicKey []byte, message string) {
	if channel.callbacks != nil {
		channel.callbacks.OnNewConnection(hex.EncodeToString(publicKey), message)
	} else {
		log.Println("Error: callbacks are nil!")
	}
}

/*
onFriendMessage calls the appropriate callback, wrapping it sanely for our purposes.
*/
func (channel *Channel) onFriendMessage(_ *gotox.Tox, friendnumber uint32, messagetype gotox.ToxMessageType, message string) {
	/*TODO make sensible*/
	if messagetype == gotox.TOX_MESSAGE_TYPE_NORMAL {
		if channel.callbacks != nil {
			address, err := channel.addressOf(friendnumber)
			if err != nil {
				log.Println(err.Error())
				address = illegalAddress
			}
			channel.callbacks.OnMessage(address, message)
		} else {
			log.Println("Error: callbacks are nil!")
		}
	}
}

/*
TODO implement and comment
*/
func (channel *Channel) onFileRecvControl(_ *gotox.Tox, friendnumber uint32, filenumber uint32, fileControl gotox.ToxFileControl) {
	// we only explicitely need to handle cancel because we then have to remove resources
	if fileControl == gotox.TOX_FILE_CONTROL_CANCEL {
		log.Println("Transfer was canceled!")
		// free resources
		channel.transfers[filenumber].Close()
		delete(channel.transfers, filenumber)
		delete(channel.transfersFilesizes, filenumber)
	}
}

/*
TODO implement and comment
*/
func (channel *Channel) onFileRecv(_ *gotox.Tox, friendnumber uint32, filenumber uint32, kind gotox.ToxFileKind, filesize uint64, filename string) {
	address, err := channel.addressOf(friendnumber)
	if err != nil {
		log.Println(err.Error())
		address = illegalAddress
	}
	// use callback to check whether to accept from Tinzenite
	accept, path := channel.callbacks.OnAllowFile(address, filename)
	if !accept {
		return
	}
	// accept file send request if we come to here
	channel.tox.FileControl(friendnumber, true, filenumber, gotox.TOX_FILE_CONTROL_RESUME, nil)
	// create file at correct location
	/*TODO how are pause & resume handled?*/
	f, _ := os.Create(path)
	// Append f to the map[uint8]*os.File
	channel.transfers[filenumber] = f
	channel.transfersFilesizes[filenumber] = filesize
}

/*
onFileRecvChunk is called when a chunk of a file is received. Writes the data to
the correct file.
*/
func (channel *Channel) onFileRecvChunk(_ *gotox.Tox, friendnumber uint32, filenumber uint32, position uint64, data []byte) {
	// Write data to the hopefully valid *File handle
	if f, exists := channel.transfers[filenumber]; exists {
		f.WriteAt(data, (int64)(position))
	} else {
		log.Println("File doesn't seem to exist!")
		return
	}
	// this means the file has been completey received
	if position == channel.transfersFilesizes[filenumber] {
		// ensure file is written
		f := channel.transfers[filenumber]
		err := f.Sync()
		if err != nil {
			log.Println("Disk error: " + err.Error())
			return
		}
		pathelements := strings.Split(f.Name(), "/")
		f.Close()
		// free resources
		delete(channel.transfers, filenumber)
		delete(channel.transfersFilesizes, filenumber)
		// callback with file name / identification
		address, _ := channel.addressOf(friendnumber)
		name := pathelements[len(pathelements)-1]
		path := strings.Join(pathelements, "/")
		channel.callbacks.OnFileReceived(address, path, name)
	}
}

/*
onFileChunkRequest is called when a chunk must be sent.
*/
func (channel *Channel) onFileChunkRequest(_ *gotox.Tox, friendNumber uint32, fileNumber uint32, position uint64, length uint64) {
	size, ok := channel.transfersFilesizes[fileNumber]
	// sanity check
	if !ok {
		log.Println("Failed to read from channel.transfers!")
		return
	}
	// recalculate length if near end of file
	if length+position > size {
		length = size - position
	}
	// get bytes to send
	data := make([]byte, length)
	_, err := channel.transfers[fileNumber].ReadAt(data, int64(position))
	if err != nil {
		fmt.Println("Error reading file: " + err.Error())
		return
	}
	// send
	err = channel.tox.FileSendChunk(friendNumber, fileNumber, position, data)
	if err != nil {
		log.Println("File send error: " + err.Error())
	}
}
<commit_msg>some fixes<commit_after>package channel

import (
	"encoding/hex"
	"errors"
	"fmt"
	"log"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/codedust/go-tox"
	"github.com/xamino/tox-dynboot"
)

/*
Channel is a wrapper of the gotox wrapper that creates and manages the underlying Tox
instance.

TODO all callbacks will block, need to avoid that especially when user interaction is required
*/
type Channel struct {
	tox                *gotox.Tox
	callbacks          Callbacks
	wg                 sync.WaitGroup
	stop               chan bool
	transfers          map[uint32]*os.File
	transfersFilesizes map[uint32]uint64
}

/*
Create and starts a new tox channel that continously runs in the background
until this object is destroyed.
*/
func Create(name string, toxdata []byte, callbacks Callbacks) (*Channel, error) {
	if name == "" {
		return nil, errors.New("CreateChannel called with no name!")
	}
	var init bool
	var channel = &Channel{}
	var options *gotox.Options
	var err error

	// prepare for file transfers
	channel.transfers = make(map[uint32]*os.File)
	channel.transfersFilesizes = make(map[uint32]uint64)

	// this decides whether we are initiating a new connection or using an existing one
	if toxdata == nil {
		options = &gotox.Options{
			true, true,
			gotox.TOX_PROXY_TYPE_NONE, "127.0.0.1", 5555, 0, 0, 0,
			gotox.TOX_SAVEDATA_TYPE_NONE, nil}
		init = true
	} else {
		options = &gotox.Options{
			true, true,
			gotox.TOX_PROXY_TYPE_NONE, "127.0.0.1", 5555, 0, 0, 0,
			gotox.TOX_SAVEDATA_TYPE_TOX_SAVE, toxdata}
		init = false
	}
	channel.tox, err = gotox.New(options)
	if err != nil {
		return nil, err
	}
	if init {
		channel.tox.SelfSetName(name)
		channel.tox.SelfSetStatusMessage("Tin Peer")
	}
	err = channel.tox.SelfSetStatus(gotox.TOX_USERSTATUS_NONE)
	// Register our callbacks
	channel.tox.CallbackFriendRequest(channel.onFriendRequest)
	channel.tox.CallbackFriendMessage(channel.onFriendMessage)
	channel.tox.CallbackFileRecvControl(channel.onFileRecvControl)
	channel.tox.CallbackFileRecv(channel.onFileRecv)
	channel.tox.CallbackFileRecvChunk(channel.onFileRecvChunk)
	channel.tox.CallbackFileChunkRequest(channel.onFileChunkRequest)
	// some things must only be done if first start
	if init {
		// Bootstrap
		toxNode, err := toxdynboot.FetchFirstAlive(200 * time.Millisecond)
		if err != nil {
			return nil, err
		}
		err = channel.tox.Bootstrap(toxNode.IPv4, toxNode.Port, toxNode.PublicKey)
		if err != nil {
			return nil, err
		}
	}
	// register callbacks
	channel.callbacks = callbacks
	// now to run it:
	channel.wg.Add(1)
	channel.stop = make(chan bool, 1)
	go channel.run()
	return channel, nil
}

// --- public methods here ---

/*
Close shuts down the channel.
*/
func (channel *Channel) Close() {
	// send stop signal
	channel.stop <- false
	// wait for it to close
	channel.wg.Wait()
	// kill tox
	channel.tox.Kill()
	// clean all file transfers
	for _, file := range channel.transfers {
		file.Close()
	}
}

/*
Address of the Tox instance.
*/
func (channel *Channel) Address() (string, error) {
	address, err := channel.tox.SelfGetAddress()
	if err != nil {
		return "", err
	}
	return strings.ToUpper(hex.EncodeToString(address)), nil
}

/*
ToxData returns the underlying current representation of the tox data. Can be
used to store a Tox instance to disk.
*/
func (channel *Channel) ToxData() ([]byte, error) {
	return channel.tox.GetSavedata()
}

/*
Send a message to the given peer address.
*/
func (channel *Channel) Send(address, message string) error {
	if ok, _ := channel.IsOnline(address); !ok {
		return errOffline
	}
	// find friend id to send to
	key, err := hex.DecodeString(address)
	if err != nil {
		return err
	}
	id, err := channel.tox.FriendByPublicKey(key)
	if err != nil {
		return err
	}
	// returns message ID but we currently don't use it
	_, err = channel.tox.FriendSendMessage(id, gotox.TOX_MESSAGE_TYPE_NORMAL, message)
	return err
}

/*
SendFile sends a file to the given address. NOTE: Will block until done!
*/
func (channel *Channel) SendFile(address string, path string, identification string) error {
	if ok, _ := channel.IsOnline(address); !ok {
		return errOffline
	}
	// find friend id to send to
	key, err := hex.DecodeString(address)
	if err != nil {
		return err
	}
	id, err := channel.tox.FriendByPublicKey(key)
	if err != nil {
		return err
	}
	// get file
	file, err := os.Open(path)
	if err != nil {
		return err
	}
	// do NOT close file! must be done elsewhere since we may need it later
	// get file size
	stat, err := os.Lstat(path)
	if err != nil {
		file.Close()
		return err
	}
	size := uint64(stat.Size())
	// prepare send (file will be transmitted via filechunk)
	fileNumber, err := channel.tox.FileSend(id, gotox.TOX_FILE_KIND_DATA, size, nil, identification)
	if err != nil {
		file.Close()
		return err
	}
	channel.transfers[fileNumber] = file
	channel.transfersFilesizes[fileNumber] = size
	return nil
}

/*
AcceptConnection accepts the given address as a connection partner.
*/
func (channel *Channel) AcceptConnection(address string) error {
	publicKey, err := hex.DecodeString(address)
	if err != nil {
		return err
	}
	// ignore friendnumber
	_, err = channel.tox.FriendAddNorequest(publicKey)
	return err
}

/*
RequestConnection sends a friend request to the given address with the sending
peer information as the message for bootstrapping.
*/
func (channel *Channel) RequestConnection(address, message string) error {
	publicKey, err := hex.DecodeString(address)
	if err != nil {
		return err
	}
	// send non blocking friend request
	_, err = channel.tox.FriendAdd(publicKey, message)
	return err
}

/*
IsOnline checks whether the given address is currently reachable.
*/
func (channel *Channel) IsOnline(address string) (bool, error) {
	publicKey, err := hex.DecodeString(address)
	if err != nil {
		return false, err
	}
	num, err := channel.tox.FriendByPublicKey(publicKey)
	if err != nil {
		return false, err
	}
	status, err := channel.tox.FriendGetConnectionStatus(num)
	if err != nil {
		return false, err
	}
	return status != gotox.TOX_CONNECTION_NONE, nil
}

/*
NameOf the key associated to the given address.
*/
func (channel *Channel) NameOf(address string) (string, error) {
	publicKey, err := hex.DecodeString(address)
	if err != nil {
		return "", err
	}
	num, err := channel.tox.FriendByPublicKey(publicKey)
	if err != nil {
		return "", err
	}
	name, err := channel.tox.FriendGetName(num)
	if err != nil {
		return "", err
	}
	return name, nil
}

// --- private methods here ---

/*
run is the background go routine method that keeps the Tox instance iterating
until Close() is called.
*/
func (channel *Channel) run() {
	for {
		temp, _ := channel.tox.IterationInterval()
		intervall := time.Duration(temp) * time.Millisecond
		select {
		case <-channel.stop:
			channel.wg.Done()
			return
		case <-time.Tick(intervall):
			err := channel.tox.Iterate()
			if err != nil {
				// TODO what do we do here? Can we cleanly close the channel and
				// catch the error further up?
				log.Println(err.Error())
			}
		} // select
	} // for
}

/*
addressOf given friend number.
*/
func (channel *Channel) addressOf(friendnumber uint32) (string, error) {
	publicKey, err := channel.tox.FriendGetPublickey(friendnumber)
	if err != nil {
		return "", errLostAddress
	}
	return hex.EncodeToString(publicKey), nil
}

/*
onFriendRequest calls the appropriate callback, wrapping it sanely for our purposes.
*/
func (channel *Channel) onFriendRequest(_ *gotox.Tox, publicKey []byte, message string) {
	if channel.callbacks != nil {
		channel.callbacks.OnNewConnection(hex.EncodeToString(publicKey), message)
	} else {
		log.Println("Error: callbacks are nil!")
	}
}

/*
onFriendMessage calls the appropriate callback, wrapping it sanely for our purposes.
*/
func (channel *Channel) onFriendMessage(_ *gotox.Tox, friendnumber uint32, messagetype gotox.ToxMessageType, message string) {
	/*TODO make sensible*/
	if messagetype == gotox.TOX_MESSAGE_TYPE_NORMAL {
		if channel.callbacks != nil {
			address, err := channel.addressOf(friendnumber)
			if err != nil {
				log.Println(err.Error())
				address = illegalAddress
			}
			channel.callbacks.OnMessage(address, message)
		} else {
			log.Println("Error: callbacks are nil!")
		}
	}
}

/*
TODO implement and comment
*/
func (channel *Channel) onFileRecvControl(_ *gotox.Tox, friendnumber uint32, filenumber uint32, fileControl gotox.ToxFileControl) {
	// we only explicitely need to handle cancel because we then have to remove resources
	if fileControl == gotox.TOX_FILE_CONTROL_CANCEL {
		log.Println("Transfer was canceled!")
		// free resources
		channel.transfers[filenumber].Close()
		delete(channel.transfers, filenumber)
		delete(channel.transfersFilesizes, filenumber)
	}
}

/*
TODO implement and comment
*/
func (channel *Channel) onFileRecv(_ *gotox.Tox, friendnumber uint32, filenumber uint32, kind gotox.ToxFileKind, filesize uint64, filename string) {
	address, err := channel.addressOf(friendnumber)
	if err != nil {
		log.Println(err.Error())
		address = illegalAddress
	}
	// use callback to check whether to accept from Tinzenite
	accept, path := channel.callbacks.OnAllowFile(address, filename)
	if !accept {
		return
	}
	// accept file send request if we come to here
	channel.tox.FileControl(friendnumber, true, filenumber, gotox.TOX_FILE_CONTROL_RESUME, nil)
	// create file at correct location
	/*TODO how are pause & resume handled?*/
	f, _ := os.Create(path)
	// Append f to the map[uint8]*os.File
	channel.transfers[filenumber] = f
	channel.transfersFilesizes[filenumber] = filesize
}

/*
onFileRecvChunk is called when a chunk of a file is received. Writes the data to
the correct file.
*/
func (channel *Channel) onFileRecvChunk(_ *gotox.Tox, friendnumber uint32, filenumber uint32, position uint64, data []byte) {
	// Write data to the hopefully valid *File handle
	if f, exists := channel.transfers[filenumber]; exists {
		f.WriteAt(data, (int64)(position))
	} else {
		log.Println("File doesn't seem to exist!")
		return
	}
	// this means the file has been completey received
	if position == channel.transfersFilesizes[filenumber] {
		// ensure file is written
		f := channel.transfers[filenumber]
		err := f.Sync()
		if err != nil {
			log.Println("Disk error: " + err.Error())
			return
		}
		pathelements := strings.Split(f.Name(), "/")
		f.Close()
		// free resources
		delete(channel.transfers, filenumber)
		delete(channel.transfersFilesizes, filenumber)
		// callback with file name / identification
		address, _ := channel.addressOf(friendnumber)
		name := pathelements[len(pathelements)-1]
		path := "/" + strings.Join(pathelements, "/")
		channel.callbacks.OnFileReceived(address, path, name)
	}
}

/*
onFileChunkRequest is called when a chunk must be sent.
*/
func (channel *Channel) onFileChunkRequest(_ *gotox.Tox, friendNumber uint32, fileNumber uint32, position uint64, length uint64) {
	size, ok := channel.transfersFilesizes[fileNumber]
	// sanity check
	if !ok {
		log.Println("Failed to read from channel.transfers!")
		return
	}
	// recalculate length if near end of file
	if length+position > size {
		length = size - position
	}
	// get bytes to send
	data := make([]byte, length)
	_, err := channel.transfers[fileNumber].ReadAt(data, int64(position))
	if err != nil {
		fmt.Println("Error reading file: " + err.Error())
		return
	}
	// send
	err = channel.tox.FileSendChunk(friendNumber, fileNumber, position, data)
	if err != nil {
		log.Println("File send error: " + err.Error())
	}
}
<|endoftext|>