<commit_before>package kite

import (
	"fmt"
	"math/rand"
	"net/http"
	"os"
	"strconv"
	"sync"
	"testing"
	"time"

	"github.com/koding/kite/config"
	"github.com/koding/kite/dnode"
	_ "github.com/koding/kite/testutil"
)

func TestMultiple(t *testing.T) {
	testDuration := time.Second * 10

	// number of available mathworker kites to be called
	kiteNumber := 100

	// number of exp kites that will call mathwork kites
	clientNumber := 100

	// ports are starting from 6000 up to 6000 + kiteNumber
	port := 6000

	fmt.Printf("Creating %d mathworker kites\n", kiteNumber)

	var transport config.Transport
	if transportName := os.Getenv("KITE_TRANSPORT"); transportName != "" {
		tr, ok := config.Transports[transportName]
		if !ok {
			t.Fatalf("transport '%s' doesn't exists", transportName)
		}

		transport = tr
	}

	for i := 0; i < kiteNumber; i++ {
		m := New("mathworker"+strconv.Itoa(i), "0.1."+strconv.Itoa(i))
		m.Config.DisableAuthentication = true
		m.Config.Transport = transport

		m.HandleFunc("square", Square)

		go http.ListenAndServe("127.0.0.1:"+strconv.Itoa(port+i), m)
	}

	// Wait until it's started
	time.Sleep(time.Second * 2)

	fmt.Printf("Creating %d exp clients\n", clientNumber)
	clients := make([]*Client, clientNumber)
	for i := 0; i < clientNumber; i++ {
		cn := New("exp"+strconv.Itoa(i), "0.0.1")
		cn.Config.Transport = transport
		c := cn.NewClient("http://127.0.0.1:" + strconv.Itoa(port+i) + "/kite")
		if err := c.Dial(); err != nil {
			t.Fatal(err)
		}

		clients[i] = c
	}

	var wg sync.WaitGroup

	fmt.Printf("Calling mathworker kites with %d conccurent clients randomly\n", clientNumber)
	timeout := time.After(testDuration)

	// every one second
	for {
		select {
		case <-time.Tick(time.Second):
			for i := 0; i < clientNumber; i++ {
				wg.Add(1)

				go func(i int, t *testing.T) {
					defer wg.Done()
					time.Sleep(time.Millisecond * time.Duration(rand.Intn(500)))

					_, err := clients[i].TellWithTimeout("square", 4*time.Second, 2)
					if err != nil {
						t.Error(err)
					}
				}(i, t)
			}
		case <-timeout:
			fmt.Println("test stopped")
			t.SkipNow()
		}

	}

	wg.Wait()
}

// Call a single method with multiple clients. This test is implemented to be
// sure the method is calling back with in the same time and not timing out.
func TestConcurrency(t *testing.T) {
	// Create a mathworker kite
	mathKite := NewKite("mathworker", "0.0.1")
	mathKite.Config.DisableAuthentication = true
	mathKite.HandleFunc("ping", func(r *Request) (interface{}, error) {
		time.Sleep(time.Second)
		return "pong", nil
	})
	go http.ListenAndServe("127.0.0.1:3637", mathKite)

	// Wait until it's started
	time.Sleep(time.Second)

	// number of exp kites that will call mathworker kite
	clientNumber := 30

	fmt.Printf("Creating %d exp clients\n", clientNumber)
	clients := make([]*Client, clientNumber)
	for i := 0; i < clientNumber; i++ {
		c := NewKite("exp", "0.0.1").NewClient("http://127.0.0.1:3637/kite")
		if err := c.Dial(); err != nil {
			t.Fatal(err)
		}

		clients[i] = c
	}

	var wg sync.WaitGroup

	for i := range clients {
		wg.Add(1)
		go func(i int) {
			defer wg.Done()
			result, err := clients[i].TellWithTimeout("ping", 4*time.Second)
			if err != nil {
				t.Fatal(err)
			}

			if result.MustString() != "pong" {
				t.Errorf("Got %s want: pong", result.MustString())
			}
		}(i)
	}

	wg.Wait()
}

// Test 2 way communication between kites.
func TestKite(t *testing.T) {
	// Create a mathworker kite
	mathKite := NewKite("mathworker", "0.0.1")
	mathKite.Config.DisableAuthentication = true
	mathKite.HandleFunc("square", Square)
	mathKite.HandleFunc("squareCB", SquareCB)
	mathKite.HandleFunc("sleep", Sleep)
	go http.ListenAndServe("127.0.0.1:3636", mathKite)

	// Wait until it's started
	time.Sleep(time.Second)

	// Create exp2 kite
	exp2Kite := NewKite("exp2", "0.0.1")
	fooChan := make(chan string)
	exp2Kite.HandleFunc("foo", func(r *Request) (interface{}, error) {
		s := r.Args.One().MustString()
		t.Logf("Message received: %s\n", s)
		fooChan <- s
		return nil, nil
	})

	// exp2 connects to mathworker
	remote := exp2Kite.NewClient("http://127.0.0.1:3636/kite")
	err := remote.Dial()
	if err != nil {
		t.Fatal(err)
	}

	result, err := remote.TellWithTimeout("square", 4*time.Second, 2)
	if err != nil {
		t.Fatal(err)
	}

	number := result.MustFloat64()

	t.Logf("rpc result: %f\n", number)

	if number != 4 {
		t.Fatalf("Invalid result: %f", number)
	}

	select {
	case s := <-fooChan:
		if s != "bar" {
			t.Fatalf("Invalid message: %s", s)
		}
	case <-time.After(100 * time.Millisecond):
		t.Fatal("Did not get the message")
	}

	resultChan := make(chan float64, 1)
	resultCallback := func(args *dnode.Partial) {
		n := args.One().MustFloat64()
		resultChan <- n
	}

	result, err = remote.TellWithTimeout("squareCB", 4*time.Second, 3, dnode.Callback(resultCallback))
	if err != nil {
		t.Fatal(err)
	}

	select {
	case n := <-resultChan:
		if n != 9.0 {
			t.Fatalf("Unexpected result: %f", n)
		}
	case <-time.After(100 * time.Millisecond):
		t.Fatal("Did not get the message")
	}

	result, err = remote.TellWithTimeout("sleep", time.Second)
	if err == nil {
		t.Fatal("Did get message in 1 seconds, however the sleep method takes 2 seconds to response")
	}

	result, err = remote.Tell("sleep")
	if err != nil {
		t.Fatal(err)
	}

	if !result.MustBool() {
		t.Fatal("sleep result must be true")
	}

}

// Sleeps for 2 seconds and returns true
func Sleep(r *Request) (interface{}, error) {
	time.Sleep(time.Second * 2)
	return true, nil
}

// Returns the result. Also tests reverse call.
func Square(r *Request) (interface{}, error) {
	a := r.Args.One().MustFloat64()
	result := a * a

	r.LocalKite.Log.Info("Kite call, sending result '%f' back\n", result)

	// Reverse method call
	r.Client.Go("foo", "bar")

	return result, nil
}

// Calls the callback with the result. For testing requests with Callback.
func SquareCB(r *Request) (interface{}, error) {
	args := r.Args.MustSliceOfLength(2)
	a := args[0].MustFloat64()
	cb := args[1].MustFunction()

	result := a * a

	r.LocalKite.Log.Info("Kite call, sending result '%f' back\n", result)

	// Send the result.
	err := cb.Call(result)
	if err != nil {
		return nil, err
	}

	return nil, nil
}

func NewKite(name, version string) *Kite {
	k := New(name, version)
	k.Config.Transport = config.XHRPolling
	return k
}
<commit_msg>kite_test: make it private and obvious<commit_after>package kite

import (
	"fmt"
	"math/rand"
	"net/http"
	"os"
	"strconv"
	"sync"
	"testing"
	"time"

	"github.com/koding/kite/config"
	"github.com/koding/kite/dnode"
	_ "github.com/koding/kite/testutil"
)

func TestMultiple(t *testing.T) {
	testDuration := time.Second * 10

	// number of available mathworker kites to be called
	kiteNumber := 100

	// number of exp kites that will call mathwork kites
	clientNumber := 100

	// ports are starting from 6000 up to 6000 + kiteNumber
	port := 6000

	fmt.Printf("Creating %d mathworker kites\n", kiteNumber)

	var transport config.Transport
	if transportName := os.Getenv("KITE_TRANSPORT"); transportName != "" {
		tr, ok := config.Transports[transportName]
		if !ok {
			t.Fatalf("transport '%s' doesn't exists", transportName)
		}

		transport = tr
	}

	for i := 0; i < kiteNumber; i++ {
		m := New("mathworker"+strconv.Itoa(i), "0.1."+strconv.Itoa(i))
		m.Config.DisableAuthentication = true
		m.Config.Transport = transport

		m.HandleFunc("square", Square)

		go http.ListenAndServe("127.0.0.1:"+strconv.Itoa(port+i), m)
	}

	// Wait until it's started
	time.Sleep(time.Second * 2)

	fmt.Printf("Creating %d exp clients\n", clientNumber)
	clients := make([]*Client, clientNumber)
	for i := 0; i < clientNumber; i++ {
		cn := New("exp"+strconv.Itoa(i), "0.0.1")
		cn.Config.Transport = transport
		c := cn.NewClient("http://127.0.0.1:" + strconv.Itoa(port+i) + "/kite")
		if err := c.Dial(); err != nil {
			t.Fatal(err)
		}

		clients[i] = c
	}

	var wg sync.WaitGroup

	fmt.Printf("Calling mathworker kites with %d conccurent clients randomly\n", clientNumber)
	timeout := time.After(testDuration)

	// every one second
	for {
		select {
		case <-time.Tick(time.Second):
			for i := 0; i < clientNumber; i++ {
				wg.Add(1)

				go func(i int, t *testing.T) {
					defer wg.Done()
					time.Sleep(time.Millisecond * time.Duration(rand.Intn(500)))

					_, err := clients[i].TellWithTimeout("square", 4*time.Second, 2)
					if err != nil {
						t.Error(err)
					}
				}(i, t)
			}
		case <-timeout:
			fmt.Println("test stopped")
			t.SkipNow()
		}

	}

	wg.Wait()
}

// Call a single method with multiple clients. This test is implemented to be
// sure the method is calling back with in the same time and not timing out.
func TestConcurrency(t *testing.T) {
	// Create a mathworker kite
	mathKite := newXhrKite("mathworker", "0.0.1")
	mathKite.Config.DisableAuthentication = true
	mathKite.HandleFunc("ping", func(r *Request) (interface{}, error) {
		time.Sleep(time.Second)
		return "pong", nil
	})
	go http.ListenAndServe("127.0.0.1:3637", mathKite)

	// Wait until it's started
	time.Sleep(time.Second)

	// number of exp kites that will call mathworker kite
	clientNumber := 30

	fmt.Printf("Creating %d exp clients\n", clientNumber)
	clients := make([]*Client, clientNumber)
	for i := 0; i < clientNumber; i++ {
		c := newXhrKite("exp", "0.0.1").NewClient("http://127.0.0.1:3637/kite")
		if err := c.Dial(); err != nil {
			t.Fatal(err)
		}

		clients[i] = c
	}

	var wg sync.WaitGroup

	for i := range clients {
		wg.Add(1)
		go func(i int) {
			defer wg.Done()
			result, err := clients[i].TellWithTimeout("ping", 4*time.Second)
			if err != nil {
				t.Fatal(err)
			}

			if result.MustString() != "pong" {
				t.Errorf("Got %s want: pong", result.MustString())
			}
		}(i)
	}

	wg.Wait()
}

// Test 2 way communication between kites.
func TestKite(t *testing.T) {
	// Create a mathworker kite
	mathKite := newXhrKite("mathworker", "0.0.1")
	mathKite.Config.DisableAuthentication = true
	mathKite.HandleFunc("square", Square)
	mathKite.HandleFunc("squareCB", SquareCB)
	mathKite.HandleFunc("sleep", Sleep)
	go http.ListenAndServe("127.0.0.1:3636", mathKite)

	// Wait until it's started
	time.Sleep(time.Second)

	// Create exp2 kite
	exp2Kite := newXhrKite("exp2", "0.0.1")
	fooChan := make(chan string)
	exp2Kite.HandleFunc("foo", func(r *Request) (interface{}, error) {
		s := r.Args.One().MustString()
		t.Logf("Message received: %s\n", s)
		fooChan <- s
		return nil, nil
	})

	// exp2 connects to mathworker
	remote := exp2Kite.NewClient("http://127.0.0.1:3636/kite")
	err := remote.Dial()
	if err != nil {
		t.Fatal(err)
	}

	result, err := remote.TellWithTimeout("square", 4*time.Second, 2)
	if err != nil {
		t.Fatal(err)
	}

	number := result.MustFloat64()

	t.Logf("rpc result: %f\n", number)

	if number != 4 {
		t.Fatalf("Invalid result: %f", number)
	}

	select {
	case s := <-fooChan:
		if s != "bar" {
			t.Fatalf("Invalid message: %s", s)
		}
	case <-time.After(100 * time.Millisecond):
		t.Fatal("Did not get the message")
	}

	resultChan := make(chan float64, 1)
	resultCallback := func(args *dnode.Partial) {
		n := args.One().MustFloat64()
		resultChan <- n
	}

	result, err = remote.TellWithTimeout("squareCB", 4*time.Second, 3, dnode.Callback(resultCallback))
	if err != nil {
		t.Fatal(err)
	}

	select {
	case n := <-resultChan:
		if n != 9.0 {
			t.Fatalf("Unexpected result: %f", n)
		}
	case <-time.After(100 * time.Millisecond):
		t.Fatal("Did not get the message")
	}

	result, err = remote.TellWithTimeout("sleep", time.Second)
	if err == nil {
		t.Fatal("Did get message in 1 seconds, however the sleep method takes 2 seconds to response")
	}

	result, err = remote.Tell("sleep")
	if err != nil {
		t.Fatal(err)
	}

	if !result.MustBool() {
		t.Fatal("sleep result must be true")
	}

}

// Sleeps for 2 seconds and returns true
func Sleep(r *Request) (interface{}, error) {
	time.Sleep(time.Second * 2)
	return true, nil
}

// Returns the result. Also tests reverse call.
func Square(r *Request) (interface{}, error) {
	a := r.Args.One().MustFloat64()
	result := a * a

	r.LocalKite.Log.Info("Kite call, sending result '%f' back\n", result)

	// Reverse method call
	r.Client.Go("foo", "bar")

	return result, nil
}

// Calls the callback with the result. For testing requests with Callback.
func SquareCB(r *Request) (interface{}, error) {
	args := r.Args.MustSliceOfLength(2)
	a := args[0].MustFloat64()
	cb := args[1].MustFunction()

	result := a * a

	r.LocalKite.Log.Info("Kite call, sending result '%f' back\n", result)

	// Send the result.
	err := cb.Call(result)
	if err != nil {
		return nil, err
	}

	return nil, nil
}

func newXhrKite(name, version string) *Kite {
	k := New(name, version)
	k.Config.Transport = config.XHRPolling
	return k
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/sha1"
	"errors"
	"fmt"
	"hash"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/zeebo/bencode"

	"github.com/rakoo/rakoshare/pkg/sharesession"
)

var (
	errNewFile    = errors.New("Got new file")
	errInvalidDir = errors.New("Invalid watched dir")
)

type state int

const (
	IDEM = iota
	CHANGED
)

type Watcher struct {
	session    *sharesession.Session
	watchedDir string
	lock       sync.Mutex

	PingNewTorrent chan string
}

func NewWatcher(session *sharesession.Session, watchedDir string) (w *Watcher, err error) {
	w = &Watcher{
		session:        session,
		watchedDir:     watchedDir,
		PingNewTorrent: make(chan string),
	}

	go w.watch()

	// Initialization, only if there is something in the dir
	if _, err := os.Stat(watchedDir); err != nil {
		return nil, err
	}

	st, err := os.Stat(watchedDir)
	if err != nil {
		return nil, err
	}
	if !st.IsDir() {
		return nil, errInvalidDir
	}

	go func() {
		w.PingNewTorrent <- session.GetCurrentInfohash()
	}()

	return
}

func (w *Watcher) watch() {
	var previousState, currentState state
	currentState = IDEM

	compareTime := w.session.GetLastModTime()

	// All paths in previous scan, sorted alphabetically
	previousScanPaths := []string{}

	// All paths in current torrent, sorted alphabetically
	torrentPaths := []string{}

	for _ = range time.Tick(10 * time.Second) {
		w.lock.Lock()

		currentTorrent := w.session.GetCurrentTorrent()
		if len(currentTorrent) != 0 {
			torrentPaths = []string{}
			m, err := NewMetaInfoFromContent([]byte(currentTorrent))
			if err == nil {
				for _, f := range m.Info.Files {
					torrentPaths = append(torrentPaths, filepath.Join(f.Path...))
				}
			}
		}

		scanPaths := []string{}

		err := torrentWalk(w.watchedDir, func(path string, info os.FileInfo, perr error) (err error) {
			if info.ModTime().After(compareTime) {
				fmt.Printf("[newer] %s\n", path)
				return errNewFile
			}
			scanPaths = append(scanPaths, path)
			return
		})

		w.lock.Unlock()

		// Check if the folder has changed:
		// - if any number of file was added or removed
		// - if any number of file path were changed
		hasChanged := err == errNewFile
		if len(scanPaths) != len(previousScanPaths) {
			hasChanged = true
		} else {
			for i := range scanPaths {
				if previousScanPaths[i] != scanPaths[i] {
					hasChanged = true
					break
				}
			}
		}

		if hasChanged {
			currentState = CHANGED
		} else if err == nil {
			currentState = IDEM
		} else {
			log.Println("Error while walking dir:", err)
		}

		compareTime = time.Now()

		if currentState == IDEM && previousState == CHANGED {
			// Note that we may be in the CHANGED state for multiple
			// iterations, such as when changes take more than 10 seconds to
			// finish. When we go back to "idle" state, we kick in the
			// metadata creation.

			// Block until we completely manage it. We will take
			// care of other changes in the next run of the loop.
			ih, err := w.torrentify()
			if err != nil {
				log.Printf("Couldn't torrentify: ", err)
				continue
			}
			w.PingNewTorrent <- ih
		}

		previousScanPaths = scanPaths
		previousState = currentState
	}
}

func (w *Watcher) torrentify() (ih string, err error) {
	w.lock.Lock()
	defer w.lock.Unlock()

	meta, err := createMeta(w.watchedDir)
	if err != nil {
		log.Println(err)
		return
	}

	var buf bytes.Buffer
	err = bencode.NewEncoder(&buf).Encode(meta)
	if err != nil {
		return
	}
	w.session.SaveTorrent(buf.Bytes(), meta.InfoHash, time.Now().Format(time.RFC3339))

	return meta.InfoHash, err
}

func createMeta(dir string) (meta *MetaInfo, err error) {
	blockSize := int64(1 << 20) // 1MiB

	fileDicts := make([]*FileDict, 0)

	hasher := NewBlockHasher(blockSize)
	err = torrentWalk(dir, func(path string, info os.FileInfo, perr error) (err error) {
		if perr != nil {
			return perr
		}

		f, err := os.Open(path)
		if err != nil {
			return errors.New(fmt.Sprintf("Couldn't open %s for hashing: %s\n", path, err))
		}
		defer f.Close()

		_, err = io.Copy(hasher, f)
		if err != nil {
			log.Printf("Couldn't hash %s: %s\n", path, err)
			return err
		}

		relPath, err := filepath.Rel(dir, path)
		if err != nil {
			return
		}

		fileDict := &FileDict{
			Length: info.Size(),
			Path:   strings.Split(relPath, string(os.PathSeparator)),
		}
		fileDicts = append(fileDicts, fileDict)

		return
	})
	if err != nil {
		return
	}

	end := hasher.Close()
	if end != nil {
		return
	}

	meta = &MetaInfo{
		Info: &InfoDict{
			Pieces:      string(hasher.Pieces),
			PieceLength: blockSize,
			Private:     0,
			Name:        "rakoshare",
			Files:       fileDicts,
		},
	}

	hash := sha1.New()
	err = bencode.NewEncoder(hash).Encode(meta.Info)
	if err != nil {
		return
	}
	meta.InfoHash = string(hash.Sum(nil))

	return
}

type BlockHasher struct {
	sha1er    hash.Hash
	left      int64
	blockSize int64
	Pieces    []byte
}

func NewBlockHasher(blockSize int64) (h *BlockHasher) {
	return &BlockHasher{
		blockSize: blockSize,
		sha1er:    sha1.New(),
		left:      blockSize,
	}
}

// You shouldn't use this one
func (h *BlockHasher) Write(p []byte) (n int, err error) {
	n2, err := h.ReadFrom(bytes.NewReader(p))
	return int(n2), err
}

func (h *BlockHasher) ReadFrom(rd io.Reader) (n int64, err error) {
	var stop bool

	for {
		if h.left > 0 {
			thisN, err := io.CopyN(h.sha1er, rd, h.left)
			if err != nil {
				if err == io.EOF {
					stop = true
				} else {
					return n, err
				}
			}
			h.left -= thisN
			n += thisN
		}
		if h.left == 0 {
			h.Pieces = h.sha1er.Sum(h.Pieces)
			h.sha1er = sha1.New()
			h.left = h.blockSize
		}

		if stop {
			break
		}
	}

	return
}

func (h *BlockHasher) Close() (err error) {
	if h.left == h.blockSize {
		// We're at the end of a blockSize, we don't have any buffered data
		return
	}
	h.Pieces = h.sha1er.Sum(h.Pieces)
	return
}

func torrentWalk(root string, fn filepath.WalkFunc) (err error) {
	return filepath.Walk(root, func(path string, info os.FileInfo, perr error) (err error) {
		if info == nil || !info.Mode().IsRegular() {
			return
		}

		// Torrents can't have empty files
		if info.Size() == 0 {
			return
		}

		if strings.HasPrefix(filepath.Base(path), ".") {
			return
		}

		if filepath.Ext(path) == ".part" {
			return
		}

		return fn(path, info, perr)
	})
}
<commit_msg>Fix dirwatch mechanism<commit_after>package main

import (
	"bytes"
	"crypto/sha1"
	"errors"
	"fmt"
	"hash"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/zeebo/bencode"

	"github.com/rakoo/rakoshare/pkg/sharesession"
)

var (
	errNewFile    = errors.New("Got new file")
	errInvalidDir = errors.New("Invalid watched dir")
)

type state int

const (
	IDEM = iota
	CHANGED
)

type Watcher struct {
	session    *sharesession.Session
	watchedDir string
	lock       sync.Mutex

	PingNewTorrent chan string
}

func NewWatcher(session *sharesession.Session, watchedDir string) (w *Watcher, err error) {
	w = &Watcher{
		session:        session,
		watchedDir:     watchedDir,
		PingNewTorrent: make(chan string),
	}

	go w.watch()

	// Initialization, only if there is something in the dir
	if _, err := os.Stat(watchedDir); err != nil {
		return nil, err
	}

	st, err := os.Stat(watchedDir)
	if err != nil {
		return nil, err
	}
	if !st.IsDir() {
		return nil, errInvalidDir
	}

	go func() {
		w.PingNewTorrent <- session.GetCurrentInfohash()
	}()

	return
}

func (w *Watcher) watch() {
	var previousState, currentState state
	currentState = IDEM

	compareTime := w.session.GetLastModTime()

	// All paths in previous scan, sorted alphabetically
	previousScanPaths := []string{}

	w.lock.Lock()
	currentTorrent := w.session.GetCurrentTorrent()
	if len(currentTorrent) != 0 {
		m, err := NewMetaInfoFromContent([]byte(currentTorrent))
		if err == nil {
			for _, f := range m.Info.Files {
				previousScanPaths = append(previousScanPaths, filepath.Join(f.Path...))
			}
		}
	}
	w.lock.Unlock()

	for _ = range time.Tick(10 * time.Second) {
		w.lock.Lock()

		err := torrentWalk(w.watchedDir, func(path string, info os.FileInfo, perr error) (err error) {
			if perr != nil {
				return perr
			}

			if info.ModTime().After(compareTime) {
				fmt.Printf("[TORRENTWATCH] newer at %s\n", path)
				return errNewFile
			}
			return nil
		})

		w.lock.Unlock()

		if err == errNewFile {
			currentState = CHANGED
		} else if err == nil {
			currentState = IDEM
		} else {
			log.Println("Error while walking dir:", err)
		}

		compareTime = time.Now()

		if currentState == IDEM && previousState == CHANGED {
			// Note that we may be in the CHANGED state for multiple
			// iterations, such as when changes take more than 10 seconds to
			// finish. When we go back to "idle" state, we kick in the
			// metadata creation.

			// Block until we completely manage it. We will take
			// care of other changes in the next run of the loop.
			ih, err := w.torrentify()
			if err != nil {
				log.Printf("Couldn't torrentify: ", err)
				continue
			}
			w.PingNewTorrent <- ih
		}

		previousState = currentState
	}
}

func (w *Watcher) torrentify() (ih string, err error) {
	w.lock.Lock()
	defer w.lock.Unlock()

	meta, err := createMeta(w.watchedDir)
	if err != nil {
		log.Println(err)
		return
	}

	var buf bytes.Buffer
	err = bencode.NewEncoder(&buf).Encode(meta)
	if err != nil {
		return
	}
	w.session.SaveTorrent(buf.Bytes(), meta.InfoHash, time.Now().Format(time.RFC3339))

	return meta.InfoHash, err
}

func createMeta(dir string) (meta *MetaInfo, err error) {
	blockSize := int64(1 << 20) // 1MiB

	fileDicts := make([]*FileDict, 0)

	hasher := NewBlockHasher(blockSize)
	err = torrentWalk(dir, func(path string, info os.FileInfo, perr error) (err error) {
		if perr != nil {
			return perr
		}

		f, err := os.Open(path)
		if err != nil {
			return errors.New(fmt.Sprintf("Couldn't open %s for hashing: %s\n", path, err))
		}
		defer f.Close()

		_, err = io.Copy(hasher, f)
		if err != nil {
			log.Printf("Couldn't hash %s: %s\n", path, err)
			return err
		}

		relPath, err := filepath.Rel(dir, path)
		if err != nil {
			return
		}

		fileDict := &FileDict{
			Length: info.Size(),
			Path:   strings.Split(relPath, string(os.PathSeparator)),
		}
		fileDicts = append(fileDicts, fileDict)

		return
	})
	if err != nil {
		return
	}

	end := hasher.Close()
	if end != nil {
		return
	}

	meta = &MetaInfo{
		Info: &InfoDict{
			Pieces:      string(hasher.Pieces),
			PieceLength: blockSize,
			Private:     0,
			Name:        "rakoshare",
			Files:       fileDicts,
		},
	}

	hash := sha1.New()
	err = bencode.NewEncoder(hash).Encode(meta.Info)
	if err != nil {
		return
	}
	meta.InfoHash = string(hash.Sum(nil))

	return
}

type BlockHasher struct {
	sha1er    hash.Hash
	left      int64
	blockSize int64
	Pieces    []byte
}

func NewBlockHasher(blockSize int64) (h *BlockHasher) {
	return &BlockHasher{
		blockSize: blockSize,
		sha1er:    sha1.New(),
		left:      blockSize,
	}
}

// You shouldn't use this one
func (h *BlockHasher) Write(p []byte) (n int, err error) {
	n2, err := h.ReadFrom(bytes.NewReader(p))
	return int(n2), err
}

func (h *BlockHasher) ReadFrom(rd io.Reader) (n int64, err error) {
	var stop bool

	for {
		if h.left > 0 {
			thisN, err := io.CopyN(h.sha1er, rd, h.left)
			if err != nil {
				if err == io.EOF {
					stop = true
				} else {
					return n, err
				}
			}
			h.left -= thisN
			n += thisN
		}
		if h.left == 0 {
			h.Pieces = h.sha1er.Sum(h.Pieces)
			h.sha1er = sha1.New()
			h.left = h.blockSize
		}

		if stop {
			break
		}
	}

	return
}

func (h *BlockHasher) Close() (err error) {
	if h.left == h.blockSize {
		// We're at the end of a blockSize, we don't have any buffered data
		return
	}
	h.Pieces = h.sha1er.Sum(h.Pieces)
	return
}

func torrentWalk(root string, fn filepath.WalkFunc) (err error) {
	return filepath.Walk(root, func(path string, info os.FileInfo, perr error) (err error) {
		if info == nil || !info.Mode().IsRegular() {
			return
		}

		// Torrents can't have empty files
		if info.Size() == 0 {
			return
		}

		if strings.HasPrefix(filepath.Base(path), ".") {
			return
		}

		if filepath.Ext(path) == ".part" {
			return
		}

		return fn(path, info, perr)
	})
}
<|endoftext|>


<commit_before>// Package analytics provides the methods to run an analytics reporting system
// for API requests which may be useful to users for measuring access and
// possibly identifying bad actors abusing requests.
package analytics

import (
	"log"
	"net/http"
	"strings"
	"time"

	"runtime"

	"encoding/json"

	"github.com/boltdb/bolt"
)

type apiRequest struct {
	URL        string `json:"url"`
	Method     string `json:"http_method"`
	Origin     string `json:"origin"`
	Proto      string `json:"http_protocol"`
	RemoteAddr string `json:"ip_address"`
	Timestamp  int64  `json:"timestamp"`
	External   bool   `json:"external"`
}

var (
	store      *bolt.DB
	recordChan chan apiRequest
)

// Record queues an apiRequest for metrics
func Record(req *http.Request) {
	external := strings.Contains(req.URL.Path, "/external/")

	r := apiRequest{
		URL:        req.URL.String(),
		Method:     req.Method,
		Origin:     req.Header.Get("Origin"),
		Proto:      req.Proto,
		RemoteAddr: req.RemoteAddr,
		Timestamp:  time.Now().Unix() * 1000,
		External:   external,
	}

	// put r on buffered recordChan to take advantage of batch insertion in DB
	recordChan <- r
}

// Close exports the abillity to close our db file. Should be called with defer
// after call to Init() from the same place.
func Close() {
	err := store.Close()
	if err != nil {
		log.Println(err)
	}
}

// Init creates a db connection, should run an initial prune of old data, and
// sets up the queue/batching channel
func Init() {
	var err error
	store, err = bolt.Open("analytics.db", 0666, nil)
	if err != nil {
		log.Fatalln(err)
	}

	recordChan = make(chan apiRequest, 1024*64*runtime.NumCPU())

	go serve()

	if err != nil {
		log.Fatalln(err)
	}
}

func serve() {
	// make timer to notify select to batch request insert from recordChan
	// interval: 30 seconds
	apiRequestTimer := time.NewTicker(time.Second * 30)

	// make timer to notify select to remove old analytics
	// interval: 2 weeks
	// TODO: enable analytics backup service to cloud
	pruneDBTimer := time.NewTicker(time.Hour * 24 * 14)

	for {
		select {
		case <-apiRequestTimer.C:
			var reqs []apiRequest
			batchSize := len(recordChan)

			for i := 0; i < batchSize; i++ {
				reqs = append(reqs, <-recordChan)
			}

			err := batchInsert(reqs)
			if err != nil {
				log.Println(err)
			}

		case <-pruneDBTimer.C:

		default:
		}
	}
}

// Week returns the map containing decoded javascript needed to chart a week of data by day
func Week() (map[string]interface{}, error) {
	// set thresholds for today and the 6 days preceeding
	times := [7]time.Time{}
	dates := [7]string{}
	now := time.Now()
	today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)

	for i := range times {
		// subtract 24 * i hours to make days prior
		dur := time.Duration(24 * i * -1)
		day := today.Add(time.Hour * dur)

		// day threshold is [...n-1-i, n-1, n]
		times[len(times)-1-i] = day
		dates[len(times)-1-i] = day.Format("01/02")
	}

	// get api request analytics from db
	var requests = []apiRequest{}
	err := store.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("requests"))

		err := b.ForEach(func(k, v []byte) error {
			var r apiRequest
			err := json.Unmarshal(v, &r)
			if err != nil {
				log.Println("Error decoding json from analytics db:", err)
				return nil
			}

			requests = append(requests, r)

			return nil
		})
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return nil, err
	}

	ips := [7]map[string]struct{}{}
	total := [7]int{}
	unique := [7]int{}

CHECK_REQUEST:
	for i := range requests {
		ts := time.Unix(requests[i].Timestamp/1000, 0)

		for j := range times {
			// if on today, there will be no next iteration to set values for
			// day prior so all valid requests belong to today
			if j == len(times) {
				if ts.After(times[j-1]) || ts.Equal(times[j]) {
					// do all record keeping
					total[j]++

					if _, ok := ips[j][requests[i].RemoteAddr]; !ok {
						unique[j-1]++
						ips[j][requests[i].RemoteAddr] = struct{}{}
					}

					continue CHECK_REQUEST
				}
			}

			if ts.Equal(times[j]) {
				// increment total count for current time threshold (day)
				total[j]++

				// if no IP found for current threshold, increment unique and record IP
				if _, ok := ips[j][requests[i].RemoteAddr]; !ok {
					unique[j]++
					ips[j][requests[i].RemoteAddr] = struct{}{}
				}

				continue CHECK_REQUEST
			}

			if ts.Before(times[j]) {
				// check if older than earliest threshold
				if j == 0 {
					continue CHECK_REQUEST
				}

				// increment total count for previous time threshold (day)
				total[j-1]++

				// if no IP found for day prior, increment unique and record IP
				if _, ok := ips[j-1][requests[i].RemoteAddr]; !ok {
					unique[j-1]++
					ips[j-1][requests[i].RemoteAddr] = struct{}{}
				}
			}
		}
	}

	jsUnique, err := json.Marshal(unique)
	if err != nil {
		return nil, err
	}

	jsTotal, err := json.Marshal(total)
	if err != nil {
		return nil, err
	}

	return map[string]interface{}{
		"dates":  dates,
		"unique": string(jsUnique),
		"total":  string(jsTotal),
	}, nil
}
<commit_msg>initializing map at each index of ips<commit_after>// Package analytics provides the methods to run an analytics reporting system
// for API requests which may be useful to users for measuring access and
// possibly identifying bad actors abusing requests.
package analytics

import (
	"log"
	"net/http"
	"strings"
	"time"

	"runtime"

	"encoding/json"

	"github.com/boltdb/bolt"
)

type apiRequest struct {
	URL        string `json:"url"`
	Method     string `json:"http_method"`
	Origin     string `json:"origin"`
	Proto      string `json:"http_protocol"`
	RemoteAddr string `json:"ip_address"`
	Timestamp  int64  `json:"timestamp"`
	External   bool   `json:"external"`
}

var (
	store      *bolt.DB
	recordChan chan apiRequest
)

// Record queues an apiRequest for metrics
func Record(req *http.Request) {
	external := strings.Contains(req.URL.Path, "/external/")

	r := apiRequest{
		URL:        req.URL.String(),
		Method:     req.Method,
		Origin:     req.Header.Get("Origin"),
		Proto:      req.Proto,
		RemoteAddr: req.RemoteAddr,
		Timestamp:  time.Now().Unix() * 1000,
		External:   external,
	}

	// put r on buffered recordChan to take advantage of batch insertion in DB
	recordChan <- r
}

// Close exports the abillity to close our db file. Should be called with defer
// after call to Init() from the same place.
func Close() {
	err := store.Close()
	if err != nil {
		log.Println(err)
	}
}

// Init creates a db connection, should run an initial prune of old data, and
// sets up the queue/batching channel
func Init() {
	var err error
	store, err = bolt.Open("analytics.db", 0666, nil)
	if err != nil {
		log.Fatalln(err)
	}

	recordChan = make(chan apiRequest, 1024*64*runtime.NumCPU())

	go serve()

	if err != nil {
		log.Fatalln(err)
	}
}

func serve() {
	// make timer to notify select to batch request insert from recordChan
	// interval: 30 seconds
	apiRequestTimer := time.NewTicker(time.Second * 30)

	// make timer to notify select to remove old analytics
	// interval: 2 weeks
	// TODO: enable analytics backup service to cloud
	pruneDBTimer := time.NewTicker(time.Hour * 24 * 14)

	for {
		select {
		case <-apiRequestTimer.C:
			var reqs []apiRequest
			batchSize := len(recordChan)

			for i := 0; i < batchSize; i++ {
				reqs = append(reqs, <-recordChan)
			}

			err := batchInsert(reqs)
			if err != nil {
				log.Println(err)
			}

		case <-pruneDBTimer.C:

		default:
		}
	}
}

// Week returns the map containing decoded javascript needed to chart a week of data by day
func Week() (map[string]interface{}, error) {
	// set thresholds for today and the 6 days preceeding
	times := [7]time.Time{}
	dates := [7]string{}
	now := time.Now()
	today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)

	for i := range times {
		// subtract 24 * i hours to make days prior
		dur := time.Duration(24 * i * -1)
		day := today.Add(time.Hour * dur)

		// day threshold is [...n-1-i, n-1, n]
		times[len(times)-1-i] = day
		dates[len(times)-1-i] = day.Format("01/02")
	}

	// get api request analytics from db
	var requests = []apiRequest{}
	err := store.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("requests"))

		err := b.ForEach(func(k, v []byte) error {
			var r apiRequest
			err := json.Unmarshal(v, &r)
			if err != nil {
				log.Println("Error decoding json from analytics db:", err)
				return nil
			}

			requests = append(requests, r)

			return nil
		})
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return nil, err
	}

	ips := [7]map[string]struct{}{}
	for i := range ips {
		ips[i] = make(map[string]struct{})
	}

	total := [7]int{}
	unique := [7]int{}

CHECK_REQUEST:
	for i := range requests {
		ts := time.Unix(requests[i].Timestamp/1000, 0)

		for j := range times {
			// if on today, there will be no next iteration to set values for
			// day prior so all valid requests belong to today
			if j == len(times) {
				if ts.After(times[j-1]) || ts.Equal(times[j]) {
					// do all record keeping
					total[j]++

					if _, ok := ips[j][requests[i].RemoteAddr]; !ok {
						unique[j-1]++
						ips[j][requests[i].RemoteAddr] = struct{}{}
					}

					continue CHECK_REQUEST
				}
			}

			if ts.Equal(times[j]) {
				// increment total count for current time threshold (day)
				total[j]++

				// if no IP found for current threshold, increment unique and record IP
				if _, ok := ips[j][requests[i].RemoteAddr]; !ok {
					unique[j]++
					ips[j][requests[i].RemoteAddr] = struct{}{}
				}

				continue CHECK_REQUEST
			}

			if ts.Before(times[j]) {
				// check if older than earliest threshold
				if j == 0 {
					continue CHECK_REQUEST
				}

				// increment total count for previous time threshold (day)
				total[j-1]++

				// if no IP found for day prior, increment unique and record IP
				if _, ok := ips[j-1][requests[i].RemoteAddr]; !ok {
					unique[j-1]++
					ips[j-1][requests[i].RemoteAddr] = struct{}{}
				}
			}
		}
	}

	jsUnique, err := json.Marshal(unique)
	if err != nil {
		return nil, err
	}

	jsTotal, err := json.Marshal(total)
	if err != nil {
		return nil, err
	}

	return map[string]interface{}{
		"dates":  dates,
		"unique": string(jsUnique),
		"total":  string(jsTotal),
	}, nil
}
<|endoftext|>


<commit_before>package lbot

import (
	"crypto/hmac"
	"crypto/sha256"
	"encoding/json"
	// "errors"
	"io/ioutil"
	"net/http"
	// "strings"
	"time"
)

// The LINE BOT
type Bot struct {
	// Configuration of this BOT
	config *Config
}

// mid, message? ID
type mid string

type eventString string

type Location struct {
	Title   string `json:"title"`
	Address string `json:"Address"`
}

// When a user sends a message, the following data is sent to your server from the LINE platform.
type Message struct {

	// Identifier of the message.
	Id string `json:"id,omitempty"`
	// A numeric value indicating the type of message sent.
	ContentType int `json:"contentType,omitempty"`
	// MID of the user who sent the message.
	From string `json:"from,omitempty"`
	// Time and date request created. Displayed as the amount of time passed since 0:00:00 on January 1, 1970. The unit is given in milliseconds.
	CreatedTime       int `json:"createdTime,omitempty"`
	parsedCreatedTime *time.Time

	// Array of user who will receive the message.
	To []mid `json:"to,omitempty"`
	// Type of user who will receive the message. (1: To user )
	ToType int `json:"toType,omitempty"`
	// Detailed information about the message
	// ContentMetadata

	// Posted text to be delivered. Note: users can send a message which has max 10,000 characters.
	Text string `json:"text,omitempty"`

	// Location data. This property is defined if the text message sent contains location data.
	Location *Location `json:"location,omitempty"`
}

// The LINE platform sends operation requests to your BOT API server when users perform actions such as adding your official account as friend.
type Operation struct {

	// Revision number of operation
	Revision int `json:"revision,omitempty"`
	// Type of operation
	OpType int `json:"opType,omitempty"`
	// Array of MIDs
	Params []*string `json:"params,omitempty"`
}

type Content struct {
	Message
	Operation
}

type Result struct {
	// Fixed value "u2ddf2eb3c959e561f6c9fa2ea732e7eb8"
	From mid `json:from`
	// Fixed value "1341301815"
	FromChannel json.Number `json:"fromChannel"`
	// MID value granted by the BOT API server’s Channel
	To []mid `json:"to"`
	// Channel ID of the BOT API server
	ToChannel json.Number `json:"toChannel"`
	// Identifier used to show the type of data
	EventType eventString `json:eventType`
	// ID string to identify each event
	Id string `json:"id"`
	// Actual data relayed by the message
	Content Content `json:"content"`
}

type Request struct {

	// Array of target user. Max count: 150.
	To []mid `json:"to"`
	// 1383378250 Fixed value
	ToChannel int `json:"toChannel"`
	// "138311608800106203" Fixed value.
	EventType eventString `json:"eventType"`
	// Object that contains the message (varies according to message type).
	Content Content `json:"content"`
}

// Return object for Callback Request
type CallbackRequest struct {
	vaild  bool
	Result []Result `json:"result"`
}

type UserProfileResponse struct {
	// contacts
	Contacts []ProfileInfo `json:"contacts"`
	Count    int           `json:"count"`
	Total    int           `json:"total"`
	Start    int           `json:"start"`
	Display  int           `json:"display"`
}

type ProfileInfo struct {
	DisplayName   string `json:"displayName"`
	MID           mid    `json:"mid"`
	pictureUrl    string `json:"pictureUrl"`
	statusMessage string `json:"statusMessage"`
}

func ParseRequest(r *http.Request) (*CallbackRequest, error) {
	result := CallbackRequest{}
	data, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return nil, err
	}
	err = json.Unmarshal(data, &result)
	if err != nil {
		return nil, err
	}
	return &result, nil
}

func ParseProfileResponse(r *http.Response) (*UserProfileResponse, error) {
	result := UserProfileResponse{}
	data, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return nil, err
	}
	err = json.Unmarshal(data, &result)
	if err != nil {
		return nil, err
	}
	return &result, nil

}

// checkSignature reports whether messageMAC is a valid HMAC tag for message.
func CheckSignature(message, messageMAC, key []byte) bool {
	mac := hmac.New(sha256.New, key)
	mac.Write(message)
	expectedMAC := mac.Sum(nil)
	return hmac.Equal(messageMAC, expectedMAC)
}

// Set default fixed value for request
func (r *Request) SetDefaults() {
	r.ToChannel = DefaultToChannel
	r.EventType = EventSendMessage
}

func (r *Request) AddTargetUser(m mid) error {
	if len(r.To) >= 150 {
		return ErrUserExceed
	}
	r.To = append(r.To, m)
	return nil
}

func (r *Request) SetText(text string) error {
	r.Content.ToType = ToTypeUser
	r.Content.ContentType = TextMessage
	r.Content.Text = text
	return nil
}
<commit_msg>Add mid<commit_after>package lbot

import (
	"crypto/hmac"
	"crypto/sha256"
	"encoding/json"
	// "errors"
	"io/ioutil"
	"net/http"
	// "strings"
	"time"
)

// The LINE BOT
type Bot struct {
	// Configuration of this BOT
	config *Config
}

// mid, message? ID
type mid string

type eventString string

type Location struct {
	Title   string `json:"title"`
	Address string `json:"Address"`
}

// When a user sends a message, the following data is sent to your server from the LINE platform.
type Message struct {

	// Identifier of the message.
	Id string `json:"id,omitempty"`
	// A numeric value indicating the type of message sent.
	ContentType int `json:"contentType,omitempty"`
	// MID of the user who sent the message.
	From mid `json:"from,omitempty"`
	// Time and date request created. Displayed as the amount of time passed since 0:00:00 on January 1, 1970. The unit is given in milliseconds.
	CreatedTime       int `json:"createdTime,omitempty"`
	parsedCreatedTime *time.Time

	// Array of user who will receive the message.
	To []mid `json:"to,omitempty"`
	// Type of user who will receive the message. (1: To user )
	ToType int `json:"toType,omitempty"`
	// Detailed information about the message
	// ContentMetadata

	// Posted text to be delivered. Note: users can send a message which has max 10,000 characters.
	Text string `json:"text,omitempty"`

	// Location data. This property is defined if the text message sent contains location data.
	Location *Location `json:"location,omitempty"`
}

// The LINE platform sends operation requests to your BOT API server when users perform actions such as adding your official account as friend.
type Operation struct {

	// Revision number of operation
	Revision int `json:"revision,omitempty"`
	// Type of operation
	OpType int `json:"opType,omitempty"`
	// Array of MIDs
	Params []*string `json:"params,omitempty"`
}

type Content struct {
	Message
	Operation
}

type Result struct {
	// Fixed value "u2ddf2eb3c959e561f6c9fa2ea732e7eb8"
	From mid `json:from`
	// Fixed value "1341301815"
	FromChannel json.Number `json:"fromChannel"`
	// MID value granted by the BOT API server’s Channel
	To []mid `json:"to"`
	// Channel ID of the BOT API server
	ToChannel json.Number `json:"toChannel"`
	// Identifier used to show the type of data
	EventType eventString `json:eventType`
	// ID string to identify each event
	Id string `json:"id"`
	// Actual data relayed by the message
	Content Content `json:"content"`
}

type Request struct {

	// Array of target user. Max count: 150.
	To []mid `json:"to"`
	// 1383378250 Fixed value
	ToChannel int `json:"toChannel"`
	// "138311608800106203" Fixed value.
	EventType eventString `json:"eventType"`
	// Object that contains the message (varies according to message type).
	Content Content `json:"content"`
}

// Return object for Callback Request
type CallbackRequest struct {
	vaild  bool
	Result []Result `json:"result"`
}

type UserProfileResponse struct {
	// contacts
	Contacts []ProfileInfo `json:"contacts"`
	Count    int           `json:"count"`
	Total    int           `json:"total"`
	Start    int           `json:"start"`
	Display  int           `json:"display"`
}

type ProfileInfo struct {
	DisplayName   string `json:"displayName"`
	MID           mid    `json:"mid"`
	pictureUrl    string `json:"pictureUrl"`
	statusMessage string `json:"statusMessage"`
}

func ParseRequest(r *http.Request) (*CallbackRequest, error) {
	result := CallbackRequest{}
	data, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return nil, err
	}
	err = json.Unmarshal(data, &result)
	if err != nil {
		return nil, err
	}
	return &result, nil
}

func ParseProfileResponse(r *http.Response) (*UserProfileResponse, error) {
	result := UserProfileResponse{}
	data, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return nil, err
	}
	err = json.Unmarshal(data, &result)
	if err != nil {
		return nil, err
	}
	return &result, nil

}

// checkSignature reports whether messageMAC is a valid HMAC tag for message.
func CheckSignature(message, messageMAC, key []byte) bool {
	mac := hmac.New(sha256.New, key)
	mac.Write(message)
	expectedMAC := mac.Sum(nil)
	return hmac.Equal(messageMAC, expectedMAC)
}

// Set default fixed value for request
func (r *Request) SetDefaults() {
	r.ToChannel = DefaultToChannel
	r.EventType = EventSendMessage
}

func (r *Request) AddTargetUser(m mid) error {
	if len(r.To) >= 150 {
		return ErrUserExceed
	}
	r.To = append(r.To, m)
	return nil
}

func (r *Request) SetText(text string) error {
	r.Content.ToType = ToTypeUser
	r.Content.ContentType = TextMessage
	r.Content.Text = text
	return nil
}
<|endoftext|>


<commit_before>package ldap

import (
	//"bufio"
	//"errors"
	"fmt"
	//"io"
	//"os"
	"strings"
	"testing"
)

var simpleLDIF string = `
version: 1

# comment

dn: cn=bob,ou=people,o=example.com
cn: bob
# comment in entry
description: a multi-line
  attribute value
-

dn: cn=joe,ou=people,o=example.com
cn: joe
description:: VGhpcyB0ZXh0IHdhcyBvcmlnaW5hbGx5IGJhc2U2NCBlbmNvZGVkLg==

dn: cn=joe,ou=people,o=example.com
changetype: modify
replace: cn
cn: joe blogs
-
delete: cn=joeDelete,ou=people,o=example.com
-
add: sn
sn: clogs
-

dn: cn=joe2,ou=people,o=example.com
changetype: add
cn: joe blogs

dn: cn=joe2,ou=people,o=example.com
changetype: delete
`

func TestLDIFOpenAndRead(t *testing.T) {
	reader := strings.NewReader(simpleLDIF)
	lr, err := NewLDIFReader(reader)
	if err != nil {
		t.Errorf(err.Error())
	}

	// record 0
	fmt.Printf("Reading record 0\n")
	record, err := lr.ReadLDIFEntry()
	if err != nil {
		t.Errorf(err.Error())
	}
	if record.RecordType() != EntryRecord {
		t.Errorf("record 0: record.RecordType() mismatch")
	}
	entry := record.(*Entry)
	if entry.GetAttributeValues("description")[0] != "a multi-line attribute value" {
		t.Errorf("record 0: description mismatch")
	}
	fmt.Printf("0 (entry): DN: %s\n", entry.DN)

	// record 1
	fmt.Printf("Reading record 1\n")
	//LDIFDebug = true
	record, err = lr.ReadLDIFEntry()
	if err != nil {
		t.Errorf(err.Error())
	}
	if record.RecordType() != EntryRecord {
		t.Errorf("record 1: record.RecordType() mismatch")
	}
	entry = record.(*Entry)
	if entry.GetAttributeValues("description")[0] != "This text was originally base64 encoded." {
		t.Errorf("record 1: description mismatch")
	}
	fmt.Printf("1 (entry): DN: %s\n", entry.DN)
	//LDIFDebug = false

	// record 2
	fmt.Printf("Reading record 2\n")
	record, err = lr.ReadLDIFEntry()
	if err != nil {
		fmt.Println(err.Error())
	}
	if record.RecordType() != ModifyRecord {
		fmt.Errorf("record 2: record.RecordType() mismatch")
	}
	modRequest := record.(*ModifyRequest)
	fmt.Printf("2 (ModifyRequest): DN: %s\n", modRequest.DN)
	fmt.Println(modRequest)

	// record 3
	fmt.Printf("Reading record 3\n")
	record, err = lr.ReadLDIFEntry()
	if err != nil {
		fmt.Println(err.Error())
	}
	if record.RecordType() != AddRecord {
		t.Errorf("record 3: record.RecordType() mismatch")
	}
	addRequest := record.(*AddRequest)
	fmt.Printf("3 (addRequest): DN: %s\n", addRequest.Entry.DN)

	// record 4
	fmt.Printf("Reading record 4\n")
	record, err = lr.ReadLDIFEntry()
	if err != nil {
		fmt.Println(err.Error())
	}
	if record.RecordType() != DeleteRecord {
		t.Errorf("record 4: record.RecordType() mismatch")
	}
	deleteRequest := record.(*DeleteRequest)
	fmt.Printf("3 (deleteRequest): DN: %s\n", deleteRequest.DN)

	// nil record
	fmt.Printf("Reading record 5 (nil)\n")
	record, err = lr.ReadLDIFEntry()
	if err != nil {
		t.Errorf(err.Error())
	}
	if record != nil {
		t.Errorf("record nil: record was not nil!")
	}

	// reading 250K entries ~ 15sec on 4+ year old desktop.
	// ldif generated from OpenDJ install
	//file, nerr := os.Open("e:/temp/250k.ldif")
	//if nerr != nil {
	//	t.Errorf(nerr.Error())
	//	return
	//}
	//defer file.Close()

	//bufReader := bufio.NewReader(file)
	//lr, err = NewLDIFReader(bufReader)
	//if err != nil {
	//	t.Errorf(err.Error())
	//}
	//for {
	//	record, err = lr.ReadLDIFEntry()
	//	if err != nil {
	//		t.Errorf(err.Error())
	//	}
	//	if record == nil {
	//		break
	//	}
	//	entry := record.(*Entry)
	//	fmt.Println(entry.DN)
	//	fmt.Println(entry.GetAttributeValue("entryUUID"))
	//}
}
<commit_msg>Extra LDIF entries.<commit_after>package ldap

import (
	//"bufio"
	//"errors"
	"fmt"
	//"io"
	//"os"
	"strings"
	"testing"
)

var simpleLDIF string = `
version: 1

# comment

dn: cn=bob,ou=people,o=example.com
cn: bob
# comment in entry
description: a multi-line
  attribute value
-

dn: cn=joe,ou=people,o=example.com
cn: joe
description:: VGhpcyB0ZXh0IHdhcyBvcmlnaW5hbGx5IGJhc2U2NCBlbmNvZGVkLg==

dn: cn=joe,ou=people,o=example.com
changetype: modify
replace: cn
cn: joe blogs
-
add: sn
sn: clogs
-

dn: cn=joe2,ou=people,o=example.com
changetype: add
cn: joe blogs

dn: cn=joe2,ou=people,o=example.com
changetype: delete

dn: cn=joe3,ou=people,o=example.com
cn: joe3
description: space at end of sn
sn: space at end

dn: cn=joe4,ou=people,o=example.com
cn: joe4
description: space at start of sn
sn:  space at start

dn: cn=joe5,ou=people,o=example.com
cn: joe5
description: less than "<" at start of sn
sn: <blogs

dn: cn=joe6,ou=people,o=example.com
cn: joe6
description: utf8 sn Hello World?
sn: 世界

dn: cn=joe7,ou=people,o=example.com
cn: joe7
description: A longish attibute value that should end up being wrapped around if that is enabled.
sn: joe7

dn: cn=joe7,ou=people,o=example.com
cn: joe7
description: <A longish attibute value that should end up being wrapped around if that is enabled, plus base64'ed
sn: joe7
`

func TestLDIFOpenAndRead(t *testing.T) {
	fmt.Printf("TestLDIFOpenAndRead: starting...\n")
	reader := strings.NewReader(simpleLDIF)
	lr, err := NewLDIFReader(reader)
	if err != nil {
		t.Errorf(err.Error())
	}

	// record 0
	fmt.Printf("Reading record 0\n")
	record, err := lr.ReadLDIFEntry()
	if err != nil {
		t.Errorf(err.Error())
	}
	if record.RecordType() != EntryRecord {
		t.Errorf("record 0: record.RecordType() mismatch")
	}
	entry := record.(*Entry)
	if entry.GetAttributeValues("description")[0] != "a multi-line attribute value" {
		t.Errorf("record 0: description mismatch")
	}
	fmt.Printf("0 (entry): DN: %s\n", entry.DN)

	// record 1
	fmt.Printf("Reading record 1\n")
	//LDIFDebug = true
	record, err = lr.ReadLDIFEntry()
	if err != nil {
		t.Errorf(err.Error())
	}
	if record.RecordType() != EntryRecord {
		t.Errorf("record 1: record.RecordType() mismatch")
	}
	entry = record.(*Entry)
	if entry.GetAttributeValues("description")[0] != "This text was originally base64 encoded." {
		t.Errorf("record 1: description mismatch")
	}
	fmt.Printf("1 (entry): DN: %s\n", entry.DN)
	//LDIFDebug = false

	// record 2
	fmt.Printf("Reading record 2\n")
	record, err = lr.ReadLDIFEntry()
	if err != nil {
		fmt.Println(err.Error())
	}
	if record.RecordType() != ModifyRecord {
		fmt.Errorf("record 2: record.RecordType() mismatch")
	}
	modRequest := record.(*ModifyRequest)
	fmt.Printf("2 (ModifyRequest): DN: %s\n", modRequest.DN)
	fmt.Println(modRequest)

	// record 3
	fmt.Printf("Reading record 3\n")
	record, err = lr.ReadLDIFEntry()
	if err != nil {
		fmt.Println(err.Error())
	}
	if record.RecordType() != AddRecord {
		t.Errorf("record 3: record.RecordType() mismatch")
	}
	addRequest := record.(*AddRequest)
	fmt.Printf("3 (addRequest): DN: %s\n", addRequest.Entry.DN)

	// record 4
	fmt.Printf("Reading record 4\n")
	record, err = lr.ReadLDIFEntry()
	if err != nil {
		fmt.Println(err.Error())
	}
	if record.RecordType() != DeleteRecord {
		t.Errorf("record 4: record.RecordType() mismatch")
	}
	deleteRequest := record.(*DeleteRequest)
	fmt.Printf("3 (deleteRequest): DN: %s\n", deleteRequest.DN)

	// nil record
	fmt.Printf("Reading record 5 (nil)\n")
	record, err = lr.ReadLDIFEntry()
	if err != nil {
		t.Errorf(err.Error())
	}
	if record != nil {
		t.Errorf("record nil: record was not nil!")
	}

	// reading 250K entries ~ 15sec on 4+ year old desktop.
	// ldif generated from OpenDJ install
	//file, nerr := os.Open("e:/temp/250k.ldif")
	//if nerr != nil {
	//	t.Errorf(nerr.Error())
	//	return
	//}
	//defer file.Close()

	//bufReader := bufio.NewReader(file)
	//lr, err = NewLDIFReader(bufReader)
	//if err != nil {
	//	t.Errorf(err.Error())
	//}
	//for {
	//	record, err = lr.ReadLDIFEntry()
	//	if err != nil {
	//		t.Errorf(err.Error())
	//	}
	//	if record == nil {
	//		break
	//	}
	//	entry := record.(*Entry)
	//	fmt.Println(entry.DN)
	//	fmt.Println(entry.GetAttributeValue("entryUUID"))
	//}
}
<|endoftext|>


<commit_before>package hdrhistogram_test

import (
	"reflect"
	"testing"

	"github.com/codahale/hdrhistogram"
)

func TestHighSigFig(t *testing.T) {
	input := []int64{
		459876, 669187, 711612, 816326, 931423, 1033197, 1131895, 2477317,
		3964974, 12718782,
	}

	hist := hdrhistogram.New(459876, 12718782, 5)
	for _, sample := range input {
		hist.RecordValue(sample)
	}

	if v, want := hist.ValueAtQuantile(50), int64(1048575); v != want {
		t.Errorf("Median was %v, but expected %v", v, want)
	}
}

func TestValueAtQuantile(t *testing.T) {
	h := hdrhistogram.New(1, 10000000, 3)

	for i := 0; i < 1000000; i++ {
		if err := h.RecordValue(int64(i)); err != nil {
			t.Fatal(err)
		}
	}

	data := []struct {
		q float64
		v int64
	}{
		{q: 50, v: 500223},
		{q: 75, v: 750079},
		{q: 90, v: 900095},
		{q: 95, v: 950271},
		{q: 99, v: 990207},
		{q: 99.9, v: 999423},
		{q: 99.99, v: 999935},
	}

	for _, d := range data {
		if v := h.ValueAtQuantile(d.q); v != d.v {
			t.Errorf("P%v was %v, but expected %v", d.q, v, d.v)
		}
	}
}

func TestMean(t *testing.T) {
	h := hdrhistogram.New(1, 10000000, 3)

	for i := 0; i < 1000000; i++ {
		if err := h.RecordValue(int64(i)); err != nil {
			t.Fatal(err)
		}
	}

	if v, want := h.Mean(), 500000.013312; v != want {
		t.Errorf("Mean was %v, but expected %v", v, want)
	}
}

func TestStdDev(t *testing.T) {
	h := hdrhistogram.New(1, 10000000, 3)

	for i := 0; i < 1000000; i++ {
		if err := h.RecordValue(int64(i)); err != nil {
			t.Fatal(err)
		}
	}

	if v, want := h.StdDev(), 288675.1403682715; v != want {
		t.Errorf("StdDev was %v, but expected %v", v, want)
	}
}

func TestMax(t *testing.T) {
	h := hdrhistogram.New(1, 10000000, 3)

	for i := 0; i < 1000000; i++ {
		if err := h.RecordValue(int64(i)); err != nil {
			t.Fatal(err)
		}
	}

	if v, want := h.Max(), int64(999936); v != want {
		t.Errorf("Max was %v, but expected %v", v, want)
	}
}

func TestReset(t *testing.T) {
	h := hdrhistogram.New(1, 10000000, 3)

	for i := 0; i < 1000000; i++ {
		if err := h.RecordValue(int64(i)); err != nil {
			t.Fatal(err)
		}
	}

	h.Reset()

	if v, want := h.Max(), int64(0); v != want {
		t.Errorf("Max was %v, but expected %v", v, want)
	}
}

func TestMerge(t *testing.T) {
	h1 := hdrhistogram.New(1, 1000, 3)
	h2 := hdrhistogram.New(1, 1000, 3)

	for i := 0; i < 100; i++ {
		if err := h1.RecordValue(int64(i)); err != nil {
			t.Fatal(err)
		}
	}

	for i := 100; i < 200; i++ {
		if err := h2.RecordValue(int64(i)); err != nil {
			t.Fatal(err)
		}
	}

	h1.Merge(h2)

	if v, want := h1.ValueAtQuantile(50), int64(99); v != want {
		t.Errorf("Median was %v, but expected %v", v, want)
	}
}

func TestMin(t *testing.T) {
	h := hdrhistogram.New(1, 10000000, 3)

	for i := 0; i < 1000000; i++ {
		if err := h.RecordValue(int64(i)); err != nil {
			t.Fatal(err)
		}
	}

	if v, want := h.Min(), int64(0); v != want {
		t.Errorf("Min was %v, but expected %v", v, want)
	}
}

func TestByteSize(t *testing.T) {
	h := hdrhistogram.New(1, 100000, 3)

	if v, want := h.ByteSize(), 65604; v != want {
		t.Errorf("ByteSize was %v, but expected %d", v, want)
	}
}

func TestRecordCorrectedValue(t *testing.T) {
	h := hdrhistogram.New(1, 100000, 3)

	if err := h.RecordCorrectedValue(10, 100); err != nil {
		t.Fatal(err)
	}

	if v, want := h.ValueAtQuantile(75), int64(10); v != want {
		t.Errorf("Corrected value was %v, but expected %v", v, want)
	}
}

func TestRecordCorrectedValueStall(t *testing.T) {
	h := hdrhistogram.New(1, 100000, 3)

	if err := h.RecordCorrectedValue(1000, 100); err != nil {
		t.Fatal(err)
	}

	if v, want := h.ValueAtQuantile(75), int64(800); v != want {
		t.Errorf("Corrected value was %v, but expected %v", v, want)
	}
}

func TestCumulativeDistribution(t *testing.T) {
	h := hdrhistogram.New(1, 100000000, 3)

	for i := 0; i < 1000000; i++ {
		if err := h.RecordValue(int64(i)); err != nil {
			t.Fatal(err)
		}
	}

	actual := h.CumulativeDistribution()
	expected := []hdrhistogram.Bracket{
		hdrhistogram.Bracket{Quantile: 0, Count: 1, ValueAt: 0},
		hdrhistogram.Bracket{Quantile: 50, Count: 500224, ValueAt: 500223},
		hdrhistogram.Bracket{Quantile: 75, Count: 750080, ValueAt: 750079},
		hdrhistogram.Bracket{Quantile: 87.5, Count: 875008, ValueAt: 875007},
		hdrhistogram.Bracket{Quantile: 93.75, Count: 937984, ValueAt: 937983},
		hdrhistogram.Bracket{Quantile: 96.875, Count: 969216, ValueAt: 969215},
		hdrhistogram.Bracket{Quantile: 98.4375, Count: 984576, ValueAt: 984575},
		hdrhistogram.Bracket{Quantile: 99.21875, Count: 992256, ValueAt: 992255},
		hdrhistogram.Bracket{Quantile: 99.609375, Count: 996352, ValueAt: 996351},
		hdrhistogram.Bracket{Quantile: 99.8046875, Count: 998400, ValueAt: 998399},
		hdrhistogram.Bracket{Quantile: 99.90234375, Count: 999424, ValueAt: 999423},
		hdrhistogram.Bracket{Quantile: 99.951171875, Count: 999936, ValueAt: 999935},
		hdrhistogram.Bracket{Quantile: 99.9755859375, Count: 999936, ValueAt: 999935},
		hdrhistogram.Bracket{Quantile: 99.98779296875, Count: 999936, ValueAt: 999935},
		hdrhistogram.Bracket{Quantile: 99.993896484375, Count: 1000000, ValueAt: 1000447},
		hdrhistogram.Bracket{Quantile: 100, Count: 1000000, ValueAt: 1000447},
	}

	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("CF was %#v, but expected %#v", actual, expected)
	}
}

func BenchmarkHistogramRecordValue(b *testing.B) {
	h := hdrhistogram.New(1, 10000000, 3)
	for i := 0; i < 1000000; i++ {
		if err := h.RecordValue(int64(i)); err != nil {
			b.Fatal(err)
		}
	}
	b.ResetTimer()
	b.ReportAllocs()

	for i := 0; i < b.N; i++ {
		h.RecordValue(100)
	}
}

func BenchmarkNew(b *testing.B) {
	b.ReportAllocs()

	for i := 0; i < b.N; i++ {
		hdrhistogram.New(1, 120000, 3) // this could track 1ms-2min
	}
}

func TestUnitMagnitudeOverflow(t *testing.T) {
	h := hdrhistogram.New(0, 200, 4)
	if err := h.RecordValue(11); err != nil {
		t.Fatal(err)
	}
}

func TestSubBucketMaskOverflow(t *testing.T) {
	hist := hdrhistogram.New(2e7, 1e8, 5)
	for _, sample := range [...]int64{1e8, 2e7, 3e7} {
		hist.RecordValue(sample)
	}

	for q, want := range map[float64]int64{
		50:    33554431,
		83.33: 33554431,
		83.34: 100663295,
		99:    100663295,
	} {
		if got := hist.ValueAtQuantile(q); got != want {
			t.Errorf("got %d for %fth percentile. want: %d", got, q, want)
		}
	}
}
<commit_msg>Add unit tests around Import/Export and Equals.<commit_after>package hdrhistogram_test

import (
	"reflect"
	"testing"

	"github.com/codahale/hdrhistogram"
)

func TestHighSigFig(t *testing.T) {
	input := []int64{
		459876, 669187, 711612, 816326, 931423, 1033197, 1131895, 2477317,
		3964974, 12718782,
	}

	hist := hdrhistogram.New(459876, 12718782, 5)
	for _, sample := range input {
		hist.RecordValue(sample)
	}

	if v, want := hist.ValueAtQuantile(50), int64(1048575); v != want {
		t.Errorf("Median was %v, but expected %v", v, want)
	}
}

func TestValueAtQuantile(t *testing.T) {
	h := hdrhistogram.New(1, 10000000, 3)

	for i := 0; i < 1000000; i++ {
		if err := h.RecordValue(int64(i)); err != nil {
			t.Fatal(err)
		}
	}

	data := []struct {
		q float64
		v int64
	}{
		{q: 50, v: 500223},
		{q: 75, v: 750079},
		{q: 90, v: 900095},
		{q: 95, v: 950271},
		{q: 99, v: 990207},
		{q: 99.9, v: 999423},
		{q: 99.99, v: 999935},
	}

	for _, d := range data {
		if v := h.ValueAtQuantile(d.q); v != d.v {
			t.Errorf("P%v was %v, but expected %v", d.q, v, d.v)
		}
	}
}

func TestMean(t *testing.T) {
	h := hdrhistogram.New(1, 10000000, 3)

	for i := 0; i < 1000000; i++ {
		if err := h.RecordValue(int64(i)); err != nil {
			t.Fatal(err)
		}
	}

	if v, want := h.Mean(), 500000.013312; v != want {
		t.Errorf("Mean was %v, but expected %v", v, want)
	}
}

func TestStdDev(t *testing.T) {
	h := hdrhistogram.New(1, 10000000, 3)

	for i := 0; i < 1000000; i++ {
		if err := h.RecordValue(int64(i)); err != nil {
			t.Fatal(err)
		}
	}

	if v, want := h.StdDev(), 288675.1403682715; v != want {
		t.Errorf("StdDev was %v, but expected %v", v, want)
	}
}

func TestMax(t *testing.T) {
	h := hdrhistogram.New(1, 10000000, 3)

	for i := 0; i < 1000000; i++ {
		if err := h.RecordValue(int64(i)); err != nil {
			t.Fatal(err)
		}
	}

	if v, want := h.Max(), int64(999936); v != want {
		t.Errorf("Max was %v, but expected %v", v, want)
	}
}

func TestReset(t *testing.T) {
	h := hdrhistogram.New(1, 10000000, 3)

	for i := 0; i < 1000000; i++ {
		if err := h.RecordValue(int64(i)); err != nil {
			t.Fatal(err)
		}
	}

	h.Reset()

	if v, want := h.Max(), int64(0); v != want {
		t.Errorf("Max was %v, but expected %v", v, want)
	}
}

func TestMerge(t *testing.T) {
	h1 := hdrhistogram.New(1, 1000, 3)
	h2 := hdrhistogram.New(1, 1000, 3)

	for i := 0; i < 100; i++ {
		if err := h1.RecordValue(int64(i)); err != nil {
			t.Fatal(err)
		}
	}

	for i := 100; i < 200; i++ {
		if err := h2.RecordValue(int64(i)); err != nil {
			t.Fatal(err)
		}
	}

	h1.Merge(h2)

	if v, want := h1.ValueAtQuantile(50), int64(99); v != want {
		t.Errorf("Median was %v, but expected %v", v, want)
	}
}

func TestMin(t *testing.T) {
	h := hdrhistogram.New(1, 10000000, 3)

	for i := 0; i < 1000000; i++ {
		if err := h.RecordValue(int64(i)); err != nil {
			t.Fatal(err)
		}
	}

	if v, want := h.Min(), int64(0); v != want {
		t.Errorf("Min was %v, but expected %v", v, want)
	}
}

func TestByteSize(t *testing.T) {
	h := hdrhistogram.New(1, 100000, 3)

	if v, want := h.ByteSize(), 65604; v != want {
		t.Errorf("ByteSize was %v, but expected %d", v, want)
	}
}

func TestRecordCorrectedValue(t *testing.T) {
	h := hdrhistogram.New(1, 100000, 3)

	if err := h.RecordCorrectedValue(10, 100); err != nil {
		t.Fatal(err)
	}

	if v, want := h.ValueAtQuantile(75), int64(10); v != want {
		t.Errorf("Corrected value was %v, but expected %v", v, want)
	}
}

func TestRecordCorrectedValueStall(t *testing.T) {
	h := hdrhistogram.New(1, 100000, 3)

	if err := h.RecordCorrectedValue(1000, 100); err != nil {
		t.Fatal(err)
	}

	if v, want := h.ValueAtQuantile(75), int64(800); v != want {
		t.Errorf("Corrected value was %v, but expected %v", v, want)
	}
}

func TestCumulativeDistribution(t *testing.T) {
	h := hdrhistogram.New(1, 100000000, 3)

	for i := 0; i < 1000000; i++ {
		if err := h.RecordValue(int64(i)); err != nil {
			t.Fatal(err)
		}
	}

	actual := h.CumulativeDistribution()
	expected := []hdrhistogram.Bracket{
		hdrhistogram.Bracket{Quantile: 0, Count: 1, ValueAt: 0},
		hdrhistogram.Bracket{Quantile: 50, Count: 500224, ValueAt: 500223},
		hdrhistogram.Bracket{Quantile: 75, Count: 750080, ValueAt: 750079},
		hdrhistogram.Bracket{Quantile: 87.5, Count: 875008, ValueAt: 875007},
		hdrhistogram.Bracket{Quantile: 93.75, Count: 937984, ValueAt: 937983},
		hdrhistogram.Bracket{Quantile: 96.875, Count: 969216, ValueAt: 969215},
		hdrhistogram.Bracket{Quantile: 98.4375, Count: 984576, ValueAt: 984575},
		hdrhistogram.Bracket{Quantile: 99.21875, Count: 992256, ValueAt: 992255},
		hdrhistogram.Bracket{Quantile: 99.609375, Count: 996352, ValueAt: 996351},
		hdrhistogram.Bracket{Quantile: 99.8046875, Count: 998400, ValueAt: 998399},
		hdrhistogram.Bracket{Quantile: 99.90234375, Count: 999424, ValueAt: 999423},
		hdrhistogram.Bracket{Quantile: 99.951171875, Count: 999936, ValueAt: 999935},
		hdrhistogram.Bracket{Quantile: 99.9755859375, Count: 999936, ValueAt: 999935},
		hdrhistogram.Bracket{Quantile: 99.98779296875, Count: 999936, ValueAt: 999935},
		hdrhistogram.Bracket{Quantile: 99.993896484375, Count: 1000000, ValueAt: 1000447},
		hdrhistogram.Bracket{Quantile: 100, Count: 1000000, ValueAt: 1000447},
	}

	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("CF was %#v, but expected %#v", actual, expected)
	}
}

func BenchmarkHistogramRecordValue(b *testing.B) {
	h := hdrhistogram.New(1, 10000000, 3)
	for i := 0; i < 1000000; i++ {
		if err := h.RecordValue(int64(i)); err != nil {
			b.Fatal(err)
		}
	}
	b.ResetTimer()
	b.ReportAllocs()

	for i := 0; i < b.N; i++ {
		h.RecordValue(100)
	}
}

func BenchmarkNew(b *testing.B) {
	b.ReportAllocs()

	for i := 0; i < b.N; i++ {
		hdrhistogram.New(1, 120000, 3) // this could track 1ms-2min
	}
}

func TestUnitMagnitudeOverflow(t *testing.T) {
	h := hdrhistogram.New(0, 200, 4)
	if err := h.RecordValue(11); err != nil {
		t.Fatal(err)
	}
}

func TestSubBucketMaskOverflow(t *testing.T) {
	hist := hdrhistogram.New(2e7, 1e8, 5)
	for _, sample := range [...]int64{1e8, 2e7, 3e7} {
		hist.RecordValue(sample)
	}

	for q, want := range map[float64]int64{
		50:    33554431,
		83.33: 33554431,
		83.34: 100663295,
		99:    100663295,
	} {
		if got := hist.ValueAtQuantile(q); got != want {
			t.Errorf("got %d for %fth percentile. want: %d", got, q, want)
		}
	}
}

func TestExportImport(t *testing.T) {
	min := int64(1)
	max := int64(10000000)
	sigfigs := int64(3)
	h := hdrhistogram.New(min, max, sigfigs)
	for i := 0; i < 1000000; i++ {
		if err := h.RecordValue(int64(i)); err != nil {
			t.Fatal(err)
		}
	}

	s := h.Export()

	if v := s.LowestTrackableValue; v != min {
		t.Errorf("LowestTrackableValue was %v, but expected %v", v, min)
	}

	if v := s.HighestTrackableValue; v != max {
		t.Errorf("HighestTrackableValue was %v, but expected %v", v, max)
	}

	if v := s.SignificantFigures; v != sigfigs {
		t.Errorf("SignificantFigures was %v, but expected %v", v, sigfigs)
	}

	if imported := hdrhistogram.Import(s); !imported.Equals(h) {
		t.Error("Expected Histograms to be equivalent")
	}

}

func TestEquals(t *testing.T) {
	h1 := hdrhistogram.New(1, 10000000, 3)
	for i := 0; i < 1000000; i++ {
		if err := h1.RecordValue(int64(i)); err != nil {
			t.Fatal(err)
		}
	}

	h2 := hdrhistogram.New(1, 10000000, 3)
	for i := 0; i < 10000; i++ {
		if err := h1.RecordValue(int64(i)); err != nil {
			t.Fatal(err)
		}
	}

	if h1.Equals(h2) {
		t.Error("Expected Histograms to not be equivalent")
	}

	h1.Reset()
	h2.Reset()

	if !h1.Equals(h2) {
		t.Error("Expected Histograms to be equivalent")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/signal"
	"strings"
	"syscall"
)

func main() {
	if os.Args[0] == "/init" {
		fmt.Fprintf(os.Stdout, "started dockersh persistent container\n")
		// Wait for terminating signal
		sc := make(chan os.Signal, 2)
		signal.Notify(sc, syscall.SIGTERM, syscall.SIGINT)
		<-sc
		os.Exit(0)
	} else {
		os.Exit(realMain())
	}
}

func tmplConfigVar(template string, v *configInterpolation) string {
	shell := "/bin/bash"
	return strings.Replace(strings.Replace(strings.Replace(template, "%h", v.Home, -1), "%u", v.User, -1), "%s", shell, -1)
}

func realMain() int {
	_, err := nsenterdetect()
	if err != nil {
		return 1
	}
	if err != nil {
		fmt.Fprintf(os.Stderr, "could not load config: %v", err)
		return 1
	}
	/* Woo! We found nsenter, now to move onto more interesting things */
	username, homedir, uid, gid, err := getCurrentUser()
	config, err := loadAllConfig(username, homedir)
	configInterpolations := configInterpolation{homedir, username}
	realUsername := tmplConfigVar(config.ContainerUsername, &configInterpolations)
	realHomedirTo := tmplConfigVar(config.MountHomeTo, &configInterpolations)
	realHomedirFrom := tmplConfigVar(config.MountHomeFrom, &configInterpolations)
	realImageName := tmplConfigVar(config.ImageName, &configInterpolations)
	realShell := tmplConfigVar(config.Shell, &configInterpolations)
	containerName := fmt.Sprintf("%s_dockersh", realUsername)

	pid, err := dockerpid(containerName)
	if err != nil {
		pid, err = dockerstart(realUsername, realHomedirFrom, realHomedirTo, containerName, realImageName, config.DockerSocket, config.MountHome, config.MountTmp, config.MountDockerSocket, config.Entrypoint)
		if err != nil {
			fmt.Fprintf(os.Stderr, "could not start container: %s\n", err)
			return 1
		}
	}
	// FIXME - Should this be it's own setting not realHomedirTo
	err = nsenterexec(pid, uid, gid, realHomedirTo, realShell)
	if err != nil {
		fmt.Fprintf(os.Stderr, err)
		return 1
	}
	return 0
}
<commit_msg>Actually compile<commit_after>package main

import (
	"fmt"
	"os"
	"os/signal"
	"strings"
	"syscall"
)

func main() {
	if os.Args[0] == "/init" {
		fmt.Fprintf(os.Stdout, "started dockersh persistent container\n")
		// Wait for terminating signal
		sc := make(chan os.Signal, 2)
		signal.Notify(sc, syscall.SIGTERM, syscall.SIGINT)
		<-sc
		os.Exit(0)
	} else {
		os.Exit(realMain())
	}
}

func tmplConfigVar(template string, v *configInterpolation) string {
	shell := "/bin/bash"
	return strings.Replace(strings.Replace(strings.Replace(template, "%h", v.Home, -1), "%u", v.User, -1), "%s", shell, -1)
}

func realMain() int {
	_, err := nsenterdetect()
	if err != nil {
		return 1
	}
	if err != nil {
		fmt.Fprintf(os.Stderr, "could not load config: %v", err)
		return 1
	}
	/* Woo! We found nsenter, now to move onto more interesting things */
	username, homedir, uid, gid, err := getCurrentUser()
	config, err := loadAllConfig(username, homedir)
	configInterpolations := configInterpolation{homedir, username}
	realUsername := tmplConfigVar(config.ContainerUsername, &configInterpolations)
	realHomedirTo := tmplConfigVar(config.MountHomeTo, &configInterpolations)
	realHomedirFrom := tmplConfigVar(config.MountHomeFrom, &configInterpolations)
	realImageName := tmplConfigVar(config.ImageName, &configInterpolations)
	realShell := tmplConfigVar(config.Shell, &configInterpolations)
	containerName := fmt.Sprintf("%s_dockersh", realUsername)

	pid, err := dockerpid(containerName)
	if err != nil {
		pid, err = dockerstart(realUsername, realHomedirFrom, realHomedirTo, containerName, realImageName, config.DockerSocket, config.MountHome, config.MountTmp, config.MountDockerSocket, config.Entrypoint)
		if err != nil {
			fmt.Fprintf(os.Stderr, "could not start container: %s\n", err)
			return 1
		}
	}
	err = nsenterexec(pid, uid, gid, realHomedirTo, realShell)
	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR")
		return 1
	}
	return 0
}
<|endoftext|>


<commit_before>package openapi

import (
	"strconv"
	"strings"
)

// codebeat:disable[TOO_MANY_IVARS]

// Document represents a OpenAPI Specification document.
type Document struct {
	Version      string `yaml:"openapi"`
	Info         *Info
	Servers      []*Server
	Paths        Paths
	Components   *Components
	Security     []*SecurityRequirement
	Tags         []*Tag
	ExternalDocs *ExternalDocumentation `yaml:"externalDocs"`
}

// Validate the values of spec.
func (doc Document) Validate() error {
	if err := validateOASVersion(doc.Version); err != nil {
		return err
	}
	if err := doc.validateRequiredObjects(); err != nil {
		return err
	}
	var validaters []validater
	validaters = append(validaters, doc.Info) // doc.Info nil check has done
	for _, s := range doc.Servers {
		validaters = append(validaters, s)
	}
	validaters = append(validaters, doc.Paths) // doc.Paths nil check has done
	if doc.Components != nil {
		validaters = append(validaters, doc.Components)
	}
	for _, securityRequirement := range doc.Security {
		validaters = append(validaters, securityRequirement)
	}
	for _, t := range doc.Tags {
		validaters = append(validaters, t)
	}
	if doc.ExternalDocs != nil {
		validaters = append(validaters, doc.ExternalDocs)
	}
	return validateAll(validaters)
}

func validateOASVersion(version string) error {
	if version == "" {
		return ErrRequired{Target: "openapi"}
	}
	splited := strings.Split(version, ".")
	if len(splited) != 3 {
		return ErrFormatInvalid{Target: "openapi version", Format: "X.Y.Z"}
	}
	major, err := strconv.Atoi(splited[0])
	if err != nil {
		return ErrFormatInvalid{Target: "major part of openapi version"}
	}
	minor, err := strconv.Atoi(splited[1])
	if err != nil {
		return ErrFormatInvalid{Target: "minor part of openapi version"}
	}
	_, err = strconv.Atoi(splited[2])
	if err != nil {
		return ErrFormatInvalid{Target: "patch part of openapi version"}
	}
	if major == 3 && 0 <= minor {
		return nil
	}
	return UnsupportedVersionError
}

func (doc Document) validateRequiredObjects() error {
	if doc.Info == nil {
		return ErrRequired{Target: "info"}
	}
	if doc.Paths == nil {
		return ErrRequired{Target: "paths"}
	}
	return nil
}
<commit_msg>simplify<commit_after>package openapi

import (
	"strconv"
	"strings"
)

// codebeat:disable[TOO_MANY_IVARS]

// Document represents a OpenAPI Specification document.
type Document struct {
	Version      string `yaml:"openapi"`
	Info         *Info
	Servers      []*Server
	Paths        Paths
	Components   *Components
	Security     []*SecurityRequirement
	Tags         []*Tag
	ExternalDocs *ExternalDocumentation `yaml:"externalDocs"`
}

// Validate the values of spec.
func (doc Document) Validate() error {
	if err := validateOASVersion(doc.Version); err != nil {
		return err
	}
	var validaters []validater
	if doc.Info == nil {
		return ErrRequired{Target: "info"}
	}
	validaters = append(validaters, doc.Info)
	for _, s := range doc.Servers {
		validaters = append(validaters, s)
	}
	if doc.Paths == nil {
		return ErrRequired{Target: "paths"}
	}
	validaters = append(validaters, doc.Paths)
	if doc.Components != nil {
		validaters = append(validaters, doc.Components)
	}
	for _, securityRequirement := range doc.Security {
		validaters = append(validaters, securityRequirement)
	}
	for _, t := range doc.Tags {
		validaters = append(validaters, t)
	}
	if doc.ExternalDocs != nil {
		validaters = append(validaters, doc.ExternalDocs)
	}
	return validateAll(validaters)
}

func validateOASVersion(version string) error {
	if version == "" {
		return ErrRequired{Target: "openapi"}
	}
	splited := strings.Split(version, ".")
	if len(splited) != 3 {
		return ErrFormatInvalid{Target: "openapi version", Format: "X.Y.Z"}
	}
	major, err := strconv.Atoi(splited[0])
	if err != nil {
		return ErrFormatInvalid{Target: "major part of openapi version"}
	}
	minor, err := strconv.Atoi(splited[1])
	if err != nil {
		return ErrFormatInvalid{Target: "minor part of openapi version"}
	}
	_, err = strconv.Atoi(splited[2])
	if err != nil {
		return ErrFormatInvalid{Target: "patch part of openapi version"}
	}
	if major == 3 && 0 <= minor {
		return nil
	}
	return UnsupportedVersionError
}
<|endoftext|>


<commit_before>package agent

import (
	"bytes"
	"encoding/json"
	"log"
	"time"

	mqtt "git.eclipse.org/gitroot/paho/org.eclipse.paho.mqtt.golang.git"
)

const (
	updateTopic = "$hardware/status/+"
	statusTopic = "$sphere/leds/status"
)

/*
 Just manages all the data going into out of this service.
*/
type Bus struct {
	conf   *Config
	agent  *Agent
	client *mqtt.MqttClient
	ticker *time.Ticker
}

type updateRequest struct {
	Topic      string
	Brightness int    `json:"brightness"`
	On         bool   `json:"on"`
	Color      string `json:"color"`
	Flash      bool   `json:"flash"`
}

type statusEvent struct {
	Status string `json:"status"`
}

type statsEvent struct {

	// memory related information
	Alloc      uint64 `json:"alloc"`
	HeapAlloc  uint64 `json:"heapAlloc"`
	TotalAlloc uint64 `json:"totalAlloc"`
}

func createBus(conf *Config, agent *Agent) *Bus {

	return &Bus{conf: conf, agent: agent}
}

func (b *Bus) listen() {
	logger.Infof("connecting to the bus")

	opts := mqtt.NewClientOptions().AddBroker(b.conf.LocalUrl).SetClientId("mqtt-bridgeify")

	// shut up
	b.client = mqtt.NewClient(opts)

	_, err := b.client.Start()
	if err != nil {
		log.Fatalf("error starting connection: %s", err)
	} else {
		logger.Infof("Connected as %s\n", b.conf.LocalUrl)
	}

	topicFilter, _ := mqtt.NewTopicFilter(updateTopic, 0)
	if _, err := b.client.StartSubscription(b.handleUpdate, topicFilter); err != nil {
		log.Fatalf("error starting subscription: %s", err)
	}

	b.setupBackgroundJob()

}

func (b *Bus) handleUpdate(client *mqtt.MqttClient, msg mqtt.Message) {
	logger.Debugf("handleUpdate")
	req := &updateRequest{}
	err := b.decodeRequest(&msg, req)
	if err != nil {
		logger.Errorf("Unable to decode connect request %s", err)
	}
	req.Topic = msg.Topic()
	b.agent.updateLeds(req)

}

func (b *Bus) setupBackgroundJob() {
	b.ticker = time.NewTicker(10 * time.Second)

	for {
		select {
		case <-b.ticker.C:
			// emit the status
			status := b.agent.getStatus()
			// log.Printf("[DEBUG] status %+v", status)
			b.client.PublishMessage(statusTopic, b.encodeRequest(status))
		}
	}

}

func (b *Bus) encodeRequest(data interface{}) *mqtt.Message {
	buf := bytes.NewBuffer(nil)
	json.NewEncoder(buf).Encode(data)
	return mqtt.NewMessage(buf.Bytes())
}

func (b *Bus) decodeRequest(msg *mqtt.Message, data interface{}) error {
	return json.NewDecoder(bytes.NewBuffer(msg.Payload())).Decode(data)
}
<commit_msg>Strip enclosing [] from payload.<commit_after>package agent

import (
	"bytes"
	"encoding/json"
	"log"
	"strings"
	"time"

	mqtt "git.eclipse.org/gitroot/paho/org.eclipse.paho.mqtt.golang.git"
)

const (
	updateTopic = "$hardware/status/+"
	statusTopic = "$sphere/leds/status"
)

/*
 Just manages all the data going into out of this service.
*/
type Bus struct {
	conf   *Config
	agent  *Agent
	client *mqtt.MqttClient
	ticker *time.Ticker
}

type updateRequest struct {
	Topic      string
	Brightness int    `json:"brightness"`
	On         bool   `json:"on"`
	Color      string `json:"color"`
	Flash      bool   `json:"flash"`
}

type statusEvent struct {
	Status string `json:"status"`
}

type statsEvent struct {

	// memory related information
	Alloc      uint64 `json:"alloc"`
	HeapAlloc  uint64 `json:"heapAlloc"`
	TotalAlloc uint64 `json:"totalAlloc"`
}

func createBus(conf *Config, agent *Agent) *Bus {

	return &Bus{conf: conf, agent: agent}
}

func (b *Bus) listen() {
	logger.Infof("connecting to the bus")

	opts := mqtt.NewClientOptions().AddBroker(b.conf.LocalUrl).SetClientId("mqtt-bridgeify")

	// shut up
	b.client = mqtt.NewClient(opts)

	_, err := b.client.Start()
	if err != nil {
		log.Fatalf("error starting connection: %s", err)
	} else {
		logger.Infof("Connected as %s\n", b.conf.LocalUrl)
	}

	topicFilter, _ := mqtt.NewTopicFilter(updateTopic, 0)
	if _, err := b.client.StartSubscription(b.handleUpdate, topicFilter); err != nil {
		log.Fatalf("error starting subscription: %s", err)
	}

	b.setupBackgroundJob()

}

func (b *Bus) handleUpdate(client *mqtt.MqttClient, msg mqtt.Message) {
	logger.Debugf("handleUpdate")
	req := &updateRequest{}
	err := b.decodeRequest(&msg, req)
	if err != nil {
		logger.Errorf("Unable to decode connect request %s", err)
	}
	req.Topic = msg.Topic()
	b.agent.updateLeds(req)

}

func (b *Bus) setupBackgroundJob() {
	b.ticker = time.NewTicker(10 * time.Second)

	for {
		select {
		case <-b.ticker.C:
			// emit the status
			status := b.agent.getStatus()
			// log.Printf("[DEBUG] status %+v", status)
			b.client.PublishMessage(statusTopic, b.encodeRequest(status))
		}
	}

}

func (b *Bus) encodeRequest(data interface{}) *mqtt.Message {
	buf := bytes.NewBuffer(nil)
	json.NewEncoder(buf).Encode(data)
	return mqtt.NewMessage(buf.Bytes())
}

func (b *Bus) decodeRequest(msg *mqtt.Message, data interface{}) error {
	payload := string(msg.Payload())
	if strings.HasPrefix(payload, "[") && strings.HasSuffix(payload, "]") {
		// go rpc can't handle enclosing parameters
		payload = payload[1 : len(payload)-1]
	}
	return json.NewDecoder(bytes.NewBuffer([]byte(payload))).Decode(data)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"path"

	"github.com/codegangsta/cli"
	"github.com/lgtmco/lgtm-go/lgtm"
)

var ListCmd = cli.Command{
	Name:  "ls",
	Usage: "list repositories",
	Action: func(c *cli.Context) {
		handle(c, listCmd)
	},
	Flags: []cli.Flag{
		cli.BoolTFlag{
			Name:  "active",
			Usage: "list active repositories",
		},
		cli.BoolFlag{
			Name:  "inactive",
			Usage: "list inactive repositories",
		},
		cli.StringFlag{
			Name:  "exclude",
			Usage: "exclude repositories matching the pattern",
		},
	},
}

func listCmd(c *cli.Context, client lgtm.Client) error {
	var (
		active   = c.BoolT("active")
		inactive = c.Bool("inactive")
		exclude  = c.String("exclude")
	)
	repos, err := client.Repos()
	if err != nil {
		return err
	}
	for _, repo := range repos {
		match, _ := path.Match(exclude, repo.Slug)
		switch {
		case !match && active && repo.ID != 0:
			fmt.Println(repo.Slug)
		case !match && inactive && repo.ID == 0:
			fmt.Println(repo.Slug)
		}
	}
	return nil
}
<commit_msg>fixed bug with inactive exclusion<commit_after>package main

import (
	"fmt"
	"path"

	"github.com/codegangsta/cli"
	"github.com/lgtmco/lgtm-go/lgtm"
)

var ListCmd = cli.Command{
	Name:  "ls",
	Usage: "list repositories",
	Action: func(c *cli.Context) {
		handle(c, listCmd)
	},
	Flags: []cli.Flag{
		cli.BoolTFlag{
			Name:  "active",
			Usage: "list active repositories",
		},
		cli.BoolFlag{
			Name:  "inactive",
			Usage: "list inactive repositories",
		},
		cli.StringFlag{
			Name:  "exclude",
			Usage: "exclude repositories matching the pattern",
		},
	},
}

func listCmd(c *cli.Context, client lgtm.Client) error {
	var (
		active   = c.BoolT("active")
		inactive = c.Bool("inactive")
		exclude  = c.String("exclude")
	)
	if inactive {
		active = false
	}
	repos, err := client.Repos()
	if err != nil {
		return err
	}
	for _, repo := range repos {
		match, _ := path.Match(exclude, repo.Slug)
		if len(exclude) != 0 && !match {
			continue
		}

		switch {
		case active && repo.ID != 0:
			fmt.Println(repo.Slug)
		case inactive && repo.ID == 0:
			fmt.Println(repo.Slug)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package command

import (
	"testing"

	"github.com/hashicorp/vault/http"
	"github.com/hashicorp/vault/meta"
	"github.com/hashicorp/vault/vault"
	"github.com/mitchellh/cli"
)

func TestSeal(t *testing.T) {
	core, _, token := vault.TestCoreUnsealed(t)
	ln, addr := http.TestServer(t, core)
	defer ln.Close()

	ui := new(cli.MockUi)
	c := &SealCommand{
		Meta: meta.Meta{
			ClientToken: token,
			Ui:          ui,
		},
	}

	args := []string{"-address", addr}
	if code := c.Run(args); code != 0 {
		t.Fatalf("bad: %d\n\n%s", code, ui.ErrorWriter.String())
	}

	sealed, err := core.Sealed()
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	if !sealed {
		t.Fatal("should be sealed")
	}
}
<commit_msg>Change seal test name in command package<commit_after>package command

import (
	"testing"

	"github.com/hashicorp/vault/http"
	"github.com/hashicorp/vault/meta"
	"github.com/hashicorp/vault/vault"
	"github.com/mitchellh/cli"
)

func Test_Seal(t *testing.T) {
	core, _, token := vault.TestCoreUnsealed(t)
	ln, addr := http.TestServer(t, core)
	defer ln.Close()

	ui := new(cli.MockUi)
	c := &SealCommand{
		Meta: meta.Meta{
			ClientToken: token,
			Ui:          ui,
		},
	}

	args := []string{"-address", addr}
	if code := c.Run(args); code != 0 {
		t.Fatalf("bad: %d\n\n%s", code, ui.ErrorWriter.String())
	}

	sealed, err := core.Sealed()
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	if !sealed {
		t.Fatal("should be sealed")
	}
}
<|endoftext|>