<commit_before>package main

import (
	"encoding/hex"
	"log"
	"math/rand"
	"reflect"
	"sort"
	"time"

	"github.com/foursquare/fsgo/report"
	"github.com/foursquare/quiver/gen"
	"github.com/foursquare/quiver/util"
)

func renderErr(raw error) string {
	msg := raw.Error()
	switch e := raw.(type) {
	case *gen.HFileServiceException:
		msg = e.GetMessage()
	default:
	}
	return msg
}

// starts count worker processes, each with their own thttp client(s), watching the work chan.
func (l *Load) startWorkers(count int, maxQps bool) {
	for i := 0; i < count; i++ {
		go func() {
			client := GetQuiverClient(l.server)
			var diff *gen.HFileServiceClient
			if l.diffing {
				diff = GetQuiverClient(l.diff)
			}
			for {
				if !maxQps {
					<-l.work
				}
				l.sendOne(client, diff)
			}
		}()
	}
}

// Pick a random request type to generate and send.
func (l *Load) sendOne(client *gen.HFileServiceClient, diff *gen.HFileServiceClient) {
	i := rand.Int31n(100)
	switch {
	case i < l.mixPrefix:
		l.sendPrefixes(client, diff)
	case i < l.mixIterator:
		l.sendGetIterator(client, diff)
	case i < l.mixMulti:
		l.sendMulti(client, diff)
	default:
		l.sendSingle(client, diff)
	}

}

// Generate and send a random GetValuesSingle request.
func (l *Load) sendSingle(client *gen.HFileServiceClient, diff *gen.HFileServiceClient) {
	keys := l.randomKeys()
	r := &gen.SingleHFileKeyRequest{HfileName: &l.collection, SortedKeys: keys}

	before := time.Now()
	resp, err := client.GetValuesSingle(r)
	if err != nil {
		log.Println("[GetValuesSingle] Error fetching value:", renderErr(err), util.PrettyKeys(keys))
	}
	report.TimeSince(l.rtt+".overall", before)
	report.TimeSince(l.rtt+".getValuesSingle", before)

	if diff != nil {
		beforeDiff := time.Now()
		diffResp, diffErr := diff.GetValuesSingle(r)
		if diffErr != nil {

			log.Println("[GetValuesSingle] Error fetching diff value:", renderErr(diffErr), util.PrettyKeys(keys))
		}
		report.TimeSince(l.diffRtt+".overall", beforeDiff)
		report.TimeSince(l.diffRtt+".getValuesSingle", beforeDiff)

		if err == nil && diffErr == nil && !reflect.DeepEqual(resp, diffResp) {
			report.Inc("diffs")
			report.Inc("diffs.getValuesSingle")
			log.Printf("[DIFF-getValuesSingle] req: %v\n%s\torig (%d): %v\n\tdiff (%d): %v\n", r, util.PrettyKeys(keys), resp.GetKeyCount(), resp, diffResp.GetKeyCount(), diffResp)
		}
	}
}

// Generate and send a random GetValuesSingle request.
func (l *Load) sendMulti(client *gen.HFileServiceClient, diff *gen.HFileServiceClient) {
	keys := l.randomKeys()
	r := &gen.SingleHFileKeyRequest{HfileName: &l.collection, SortedKeys: keys}

	before := time.Now()
	resp, err := client.GetValuesMulti(r)
	if err != nil {
		log.Println("[GetValuesMulti] Error fetching value:", renderErr(err), util.PrettyKeys(keys))
	}
	report.TimeSince(l.rtt+".overall", before)
	report.TimeSince(l.rtt+".getValuesMulti", before)

	if diff != nil {
		beforeDiff := time.Now()
		diffResp, diffErr := diff.GetValuesMulti(r)
		if diffErr != nil {
			log.Println("[GetValuesMulti] Error fetching diff value:", renderErr(diffErr), util.PrettyKeys(keys))
		}
		report.TimeSince(l.diffRtt+".overall", beforeDiff)
		report.TimeSince(l.diffRtt+".getValuesMulti", beforeDiff)

		if err == nil && diffErr == nil && !reflect.DeepEqual(resp, diffResp) {
			report.Inc("diffs")
			report.Inc("diffs.getValuesMulti")
			log.Printf("[DIFF-getValuesMulti] req: %v\n \torig: %v\n\n\tdiff: %v\n", r, resp, diffResp)
		}
	}
}

// Generate and send a random GetValuesSingle request.
func (l *Load) sendPrefixes(client *gen.HFileServiceClient, diff *gen.HFileServiceClient) {

	fullKeys := l.randomKeys()
	prefixes := make([][]byte, len(fullKeys))

	for i, v := range fullKeys {
		prefixes[i] = v[:len(v)/2]
	}
	sort.Sort(util.Keys(prefixes))
	limit := int32(10) // request a max of 10 k/v pairs
	r := &gen.PrefixRequest{HfileName: &l.collection, SortedKeys: prefixes, ValueLimit: &limit}

	before := time.Now()
	resp, err := client.GetValuesForPrefixes(r)
	if err != nil {
		log.Println("[GetValuesForPrefixes] Error fetching value:", renderErr(err), util.PrettyKeys(prefixes))
	}
	report.TimeSince(l.rtt+".overall", before)
	report.TimeSince(l.rtt+".getValuesForPrefixes", before)

	if diff != nil {
		beforeDiff := time.Now()
		diffResp, diffErr := diff.GetValuesForPrefixes(r)
		if diffErr != nil {
			log.Println("[GetValuesForPrefixes] Error fetching diff value:", renderErr(diffErr), util.PrettyKeys(prefixes))
		}
		report.TimeSince(l.diffRtt+".overall", beforeDiff)
		report.TimeSince(l.diffRtt+".getValuesForPrefixes", beforeDiff)

		if err == nil && diffErr == nil && !reflect.DeepEqual(resp, diffResp) {
			report.Inc("diffs")
			report.Inc("diffs.getValuesForPrefixes")
			if resp != nil && diffResp != nil {
				log.Printf("[DIFF-GetValuesForPrefixes] req: %v\n", r)

				log.Printf("[DIFF-GetValuesForPrefixes] orig len: %d\n", len(resp.GetValues()))
				log.Printf("[DIFF-GetValuesForPrefixes] diff len: %d\n", len(diffResp.GetValues()))

				log.Printf("[DIFF-GetValuesForPrefixes] orig lastKey: %s\n", hex.EncodeToString(resp.GetLastKey()))
				log.Printf("[DIFF-GetValuesForPrefixes] diff lastKey: %s\n", hex.EncodeToString(diffResp.GetLastKey()))

				a := resp.GetValues()
				b := diffResp.GetValues()
				for k, values := range a {
					if other, ok := b[k]; !ok {
						log.Printf("[DIFF-GetValuesForPrefixes] Missing from diff: %s", hex.EncodeToString([]byte(k)))
					} else if !reflect.DeepEqual(values, other) {
						log.Printf("[DIFF-GetValuesForPrefixes] Different values for: %s", hex.EncodeToString([]byte(k)))
						log.Printf("[DIFF-GetValuesForPrefixes] orig (%d): %s", len(values), util.PrettyKeys(values))
						log.Printf("[DIFF-GetValuesForPrefixes] diff (%d): %s", len(other), util.PrettyKeys(other))
					}
				}
				for k, _ := range b {
					if _, ok := a[k]; !ok {
						log.Printf("[DIFF-GetValuesForPrefixes] Missing from orig: %s", hex.EncodeToString([]byte(k)))
					}
				}
				log.Println("\n")
			} else {
				log.Printf("[DIFF-GetValuesForPrefixes] req: %v\n \torig: %v\n\n\tdiff: %v\n", r, resp, diffResp)
			}
		}
	}
}

// Generate and send a random GetValuesSingle request.
func (l *Load) sendGetIterator(client *gen.HFileServiceClient, diff *gen.HFileServiceClient) {

	includeValues := true
	k := l.randomKey()
	lim := int32(10)
	r := &gen.IteratorRequest{HfileName: &l.collection, IncludeValues: &includeValues, LastKey: k, ResponseLimit: &lim}

	before := time.Now()
	resp, err := client.GetIterator(r)
	if err != nil {
		log.Println("[GetIterator] Error fetching value:", renderErr(err), k)
	}
	report.TimeSince(l.rtt+".overall", before)
	report.TimeSince(l.rtt+".getIterator", before)

	if diff != nil {
		beforeDiff := time.Now()
		diffResp, diffErr := diff.GetIterator(r)
		if diffErr != nil {
			log.Println("[GetIterator] Error fetching diff value:", renderErr(diffErr), k)
		}
		report.TimeSince(l.diffRtt+".overall", beforeDiff)
		report.TimeSince(l.diffRtt+".getIterator", beforeDiff)

		if err == nil && diffErr == nil && !reflect.DeepEqual(resp, diffResp) {
			report.Inc("diffs")
			report.Inc("diffs.GetIterator")
			log.Printf("[DIFF-GetIterator] req: %v\n", r)
			log.Printf("[DIFF-GetIterator] orig (skip %d): %v\n", resp.GetSkipKeys(), resp)
			log.Printf("[DIFF-GetIterator] diff (skip %d): %v\n", diffResp.GetSkipKeys(), diffResp)
		}
	}
}
<commit_msg>if diffing, fire both requests at the same time<commit_after>package main

import (
	"encoding/hex"
	"log"
	"math/rand"
	"reflect"
	"sort"
	"sync"
	"time"

	"github.com/foursquare/fsgo/report"
	"github.com/foursquare/quiver/gen"
	"github.com/foursquare/quiver/util"
)

func renderErr(raw error) string {
	msg := raw.Error()
	switch e := raw.(type) {
	case *gen.HFileServiceException:
		msg = e.GetMessage()
	default:
	}
	return msg
}

// starts count worker processes, each with their own thttp client(s), watching the work chan.
func (l *Load) startWorkers(count int, maxQps bool) {
	for i := 0; i < count; i++ {
		go func() {
			client := GetQuiverClient(l.server)
			var diff *gen.HFileServiceClient
			if l.diffing {
				diff = GetQuiverClient(l.diff)
			}
			for {
				if !maxQps {
					<-l.work
				}
				l.sendOne(client, diff)
			}
		}()
	}
}

// Pick a random request type to generate and send.
func (l *Load) sendOne(client *gen.HFileServiceClient, diff *gen.HFileServiceClient) {
	i := rand.Int31n(100)
	switch {
	case i < l.mixPrefix:
		l.sendPrefixes(client, diff)
	case i < l.mixIterator:
		l.sendGetIterator(client, diff)
	case i < l.mixMulti:
		l.sendMulti(client, diff)
	default:
		l.sendSingle(client, diff)
	}

}

// Generate and send a random GetValuesSingle request.
func (l *Load) sendSingle(client *gen.HFileServiceClient, diff *gen.HFileServiceClient) {
	var wg sync.WaitGroup
	keys := l.randomKeys()
	r := &gen.SingleHFileKeyRequest{HfileName: &l.collection, SortedKeys: keys}

	var diffResp *gen.SingleHFileKeyResponse
	var diffErr error
	if diff != nil {
		wg.Add(1)
		go func() {
			defer wg.Done()
			beforeDiff := time.Now()
			diffResp, diffErr = diff.GetValuesSingle(r)
			if diffErr != nil {
				log.Println("[GetValuesSingle] Error fetching diff value:", renderErr(diffErr), util.PrettyKeys(keys))
			}
			report.TimeSince(l.diffRtt+".overall", beforeDiff)
			report.TimeSince(l.diffRtt+".getValuesSingle", beforeDiff)
		}()
	}

	before := time.Now()
	resp, err := client.GetValuesSingle(r)
	if err != nil {
		log.Println("[GetValuesSingle] Error fetching value:", renderErr(err), util.PrettyKeys(keys))
	}
	report.TimeSince(l.rtt+".overall", before)
	report.TimeSince(l.rtt+".getValuesSingle", before)

	if diff != nil {
		wg.Wait()
		if err == nil && diffErr == nil && !reflect.DeepEqual(resp, diffResp) {
			report.Inc("diffs")
			report.Inc("diffs.getValuesSingle")
			log.Printf("[DIFF-getValuesSingle] req: %v\n%s\torig (%d): %v\n\tdiff (%d): %v\n", r, util.PrettyKeys(keys), resp.GetKeyCount(), resp, diffResp.GetKeyCount(), diffResp)
		}
	}
}

// Generate and send a random GetValuesSingle request.
func (l *Load) sendMulti(client *gen.HFileServiceClient, diff *gen.HFileServiceClient) {
	var wg sync.WaitGroup

	keys := l.randomKeys()
	r := &gen.SingleHFileKeyRequest{HfileName: &l.collection, SortedKeys: keys}

	var diffResp *gen.MultiHFileKeyResponse
	var diffErr error
	if diff != nil {
		wg.Add(1)
		go func() {
			defer wg.Done()
			beforeDiff := time.Now()
			diffResp, diffErr = diff.GetValuesMulti(r)
			if diffErr != nil {
				log.Println("[GetValuesMulti] Error fetching diff value:", renderErr(diffErr), util.PrettyKeys(keys))
			}
			report.TimeSince(l.diffRtt+".overall", beforeDiff)
			report.TimeSince(l.diffRtt+".getValuesMulti", beforeDiff)
		}()
	}

	before := time.Now()
	resp, err := client.GetValuesMulti(r)
	if err != nil {
		log.Println("[GetValuesMulti] Error fetching value:", renderErr(err), util.PrettyKeys(keys))
	}
	report.TimeSince(l.rtt+".overall", before)
	report.TimeSince(l.rtt+".getValuesMulti", before)

	if diff != nil {
		wg.Wait()
		if err == nil && diffErr == nil && !reflect.DeepEqual(resp, diffResp) {
			report.Inc("diffs")
			report.Inc("diffs.getValuesMulti")
			log.Printf("[DIFF-getValuesMulti] req: %v\n \torig: %v\n\n\tdiff: %v\n", r, resp, diffResp)
		}
	}
}

// Generate and send a random GetValuesSingle request.
func (l *Load) sendPrefixes(client *gen.HFileServiceClient, diff *gen.HFileServiceClient) {
	var wg sync.WaitGroup

	fullKeys := l.randomKeys()
	prefixes := make([][]byte, len(fullKeys))

	for i, v := range fullKeys {
		prefixes[i] = v[:len(v)/2]
	}
	sort.Sort(util.Keys(prefixes))
	limit := int32(10) // request a max of 10 k/v pairs
	r := &gen.PrefixRequest{HfileName: &l.collection, SortedKeys: prefixes, ValueLimit: &limit}

	var diffResp *gen.PrefixResponse
	var diffErr error
	if diff != nil {
		wg.Add(1)
		go func() {
			defer wg.Done()
			beforeDiff := time.Now()
			diffResp, diffErr = diff.GetValuesForPrefixes(r)
			if diffErr != nil {
				log.Println("[GetValuesForPrefixes] Error fetching diff value:", renderErr(diffErr), util.PrettyKeys(prefixes))
			}
			report.TimeSince(l.diffRtt+".overall", beforeDiff)
			report.TimeSince(l.diffRtt+".getValuesForPrefixes", beforeDiff)
		}()
	}

	before := time.Now()
	resp, err := client.GetValuesForPrefixes(r)
	if err != nil {
		log.Println("[GetValuesForPrefixes] Error fetching value:", renderErr(err), util.PrettyKeys(prefixes))
	}
	report.TimeSince(l.rtt+".overall", before)
	report.TimeSince(l.rtt+".getValuesForPrefixes", before)

	if diff != nil {
		wg.Wait()

		if err == nil && diffErr == nil && !reflect.DeepEqual(resp, diffResp) {
			report.Inc("diffs")
			report.Inc("diffs.getValuesForPrefixes")
			if resp != nil && diffResp != nil {
				log.Printf("[DIFF-GetValuesForPrefixes] req: %v\n", r)

				log.Printf("[DIFF-GetValuesForPrefixes] orig len: %d\n", len(resp.GetValues()))
				log.Printf("[DIFF-GetValuesForPrefixes] diff len: %d\n", len(diffResp.GetValues()))

				log.Printf("[DIFF-GetValuesForPrefixes] orig lastKey: %s\n", hex.EncodeToString(resp.GetLastKey()))
				log.Printf("[DIFF-GetValuesForPrefixes] diff lastKey: %s\n", hex.EncodeToString(diffResp.GetLastKey()))

				a := resp.GetValues()
				b := diffResp.GetValues()
				for k, values := range a {
					if other, ok := b[k]; !ok {
						log.Printf("[DIFF-GetValuesForPrefixes] Missing from diff: %s", hex.EncodeToString([]byte(k)))
					} else if !reflect.DeepEqual(values, other) {
						log.Printf("[DIFF-GetValuesForPrefixes] Different values for: %s", hex.EncodeToString([]byte(k)))
						log.Printf("[DIFF-GetValuesForPrefixes] orig (%d): %s", len(values), util.PrettyKeys(values))
						log.Printf("[DIFF-GetValuesForPrefixes] diff (%d): %s", len(other), util.PrettyKeys(other))
					}
				}
				for k, _ := range b {
					if _, ok := a[k]; !ok {
						log.Printf("[DIFF-GetValuesForPrefixes] Missing from orig: %s", hex.EncodeToString([]byte(k)))
					}
				}
				log.Println("\n")
			} else {
				log.Printf("[DIFF-GetValuesForPrefixes] req: %v\n \torig: %v\n\n\tdiff: %v\n", r, resp, diffResp)
			}
		}
	}
}

// Generate and send a random GetValuesSingle request.
func (l *Load) sendGetIterator(client *gen.HFileServiceClient, diff *gen.HFileServiceClient) {
	var wg sync.WaitGroup
	includeValues := true
	k := l.randomKey()
	lim := int32(10)
	r := &gen.IteratorRequest{HfileName: &l.collection, IncludeValues: &includeValues, LastKey: k, ResponseLimit: &lim}

	var diffResp *gen.IteratorResponse
	var diffErr error
	if diff != nil {
		wg.Add(1)
		go func() {
			defer wg.Done()
			beforeDiff := time.Now()
			diffResp, diffErr = diff.GetIterator(r)
			if diffErr != nil {
				log.Println("[GetIterator] Error fetching diff value:", renderErr(diffErr), k)
			}
			report.TimeSince(l.diffRtt+".overall", beforeDiff)
			report.TimeSince(l.diffRtt+".getIterator", beforeDiff)

		}()
	}

	before := time.Now()
	resp, err := client.GetIterator(r)
	if err != nil {
		log.Println("[GetIterator] Error fetching value:", renderErr(err), k)
	}
	report.TimeSince(l.rtt+".overall", before)
	report.TimeSince(l.rtt+".getIterator", before)

	if diff != nil {
		wg.Wait()
		if err == nil && diffErr == nil && !reflect.DeepEqual(resp, diffResp) {
			report.Inc("diffs")
			report.Inc("diffs.GetIterator")
			log.Printf("[DIFF-GetIterator] req: %v\n", r)
			log.Printf("[DIFF-GetIterator] orig (skip %d): %v\n", resp.GetSkipKeys(), resp)
			log.Printf("[DIFF-GetIterator] diff (skip %d): %v\n", diffResp.GetSkipKeys(), diffResp)
		}
	}
}
<|endoftext|>


<commit_before>// The fmt command applies standard formatting to text proto files and preserves
// comments.
package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"flag"
	log "github.com/golang/glog"
	"github.com/protocolbuffers/txtpbfmt/parser"
)

var (
	// Top level flags.
	dryRun                   = flag.Bool("dry_run", false, "Enable dry run mode.")
	expandAllChildren        = flag.Bool("expand_all_children", false, "Expand all children irrespective of initial state.")
	skipAllColons            = flag.Bool("skip_all_colons", false, "Skip colons whenever possible.")
	allowTripleQuotedStrings = flag.Bool("allow_triple_quoted_strings", false,
		`Allow Python-style """ or ''' delimited strings in input.`)
)

const stdinPath = "<stdin>"

func read(path string) ([]byte, error) {
	if path == stdinPath {
		return ioutil.ReadAll(bufio.NewReader(os.Stdin))
	}
	return ioutil.ReadFile(path)
}

func errorf(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, format+"\n", args...)
}

func contentForLogging(content []byte) string {
	res := string(content)
	if len(res) > 100 {
		res = res[:100] + " ... <snip> ..."
	}
	return res
}

func main() {
	flag.Parse()
	paths := flag.Args()
	if len(paths) == 0 {
		paths = append(paths, stdinPath)
	}
	log.Info("paths: ", paths)
	errs := 0
	for _, path := range paths {
		if strings.HasPrefix(path, "//depot/google3/") {
			path = strings.Replace(path, "//depot/google3/", "", 1)
		}
		log.Info("path ", path)
		content, err := read(path)
		if os.IsNotExist(err) {
			log.Error("Ignoring path: ", err)
			errs++
			continue
		} else if err != nil {
			log.Exit(err)
		}

		newContent, err := parser.FormatWithConfig(content, parser.Config{
			ExpandAllChildren:        *expandAllChildren,
			SkipAllColons:            *skipAllColons,
			AllowTripleQuotedStrings: *allowTripleQuotedStrings,
		})
		if err != nil {
			errorf("parser.Format for path %v with content %q returned err %v", path, contentForLogging(content), err)
			errs++
			continue
		}
		log.V(2).Infof("New content for path %s: %q", path, newContent)

		if path == stdinPath {
			fmt.Print(string(newContent))
			continue
		}
		if bytes.Equal(content, newContent) {
			log.Info("No change for path ", path)
			continue
		}
		if *dryRun {
			fmt.Println(string(newContent))
			continue
		}
		if err := ioutil.WriteFile(path, newContent, 0664); err != nil {
			log.Exit(err)
		}
	}
	if errs > 0 {
		log.Exit(errs, " error(s) encountered during execution")
	}
}
<commit_msg>Internal change<commit_after>// The fmt command applies standard formatting to text proto files and preserves
// comments.
package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"flag"
	// Google internal base/go package, commented out by copybara
	log "github.com/golang/glog"
	"github.com/protocolbuffers/txtpbfmt/parser"
)

var (
	// Top level flags.
	dryRun                   = flag.Bool("dry_run", false, "Enable dry run mode.")
	expandAllChildren        = flag.Bool("expand_all_children", false, "Expand all children irrespective of initial state.")
	skipAllColons            = flag.Bool("skip_all_colons", false, "Skip colons whenever possible.")
	allowTripleQuotedStrings = flag.Bool("allow_triple_quoted_strings", false,
		`Allow Python-style """ or ''' delimited strings in input.`)
)

const stdinPath = "<stdin>"

func read(path string) ([]byte, error) {
	if path == stdinPath {
		return ioutil.ReadAll(bufio.NewReader(os.Stdin))
	}
	return ioutil.ReadFile(path)
}

func errorf(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, format+"\n", args...)
}

func contentForLogging(content []byte) string {
	res := string(content)
	if len(res) > 100 {
		res = res[:100] + " ... <snip> ..."
	}
	return res
}

func main() {
	flag.Parse()
	paths := flag.Args()
	if len(paths) == 0 {
		paths = append(paths, stdinPath)
	}
	log.Info("paths: ", paths)
	errs := 0
	for _, path := range paths {
		if strings.HasPrefix(path, "//depot/google3/") {
			path = strings.Replace(path, "//depot/google3/", "", 1)
		}
		log.Info("path ", path)
		content, err := read(path)
		if os.IsNotExist(err) {
			log.Error("Ignoring path: ", err)
			errs++
			continue
		} else if err != nil {
			log.Exit(err)
		}

		newContent, err := parser.FormatWithConfig(content, parser.Config{
			ExpandAllChildren:        *expandAllChildren,
			SkipAllColons:            *skipAllColons,
			AllowTripleQuotedStrings: *allowTripleQuotedStrings,
		})
		if err != nil {
			errorf("parser.Format for path %v with content %q returned err %v", path, contentForLogging(content), err)
			errs++
			continue
		}
		log.V(2).Infof("New content for path %s: %q", path, newContent)

		if path == stdinPath {
			fmt.Print(string(newContent))
			continue
		}
		if bytes.Equal(content, newContent) {
			log.Info("No change for path ", path)
			continue
		}
		if *dryRun {
			fmt.Println(string(newContent))
			continue
		}
		if err := ioutil.WriteFile(path, newContent, 0664); err != nil {
			log.Exit(err)
		}
	}
	if errs > 0 {
		log.Exit(errs, " error(s) encountered during execution")
	}
}
<|endoftext|>


<commit_before>// Copyright 2012-2017 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Read the system log.
//
// Synopsis:
//     dmesg [-c]
//
// Options:
//     -c: clear the log
package main

import (
	"flag"
	"log"
	"os"
	"unsafe"

	"golang.org/x/sys/unix"
)

var clearSyslog = flag.Bool("c", false, "Clear the log")

func main() {
	flag.Parse()
	l := uintptr(3)
	if *clearSyslog {
		l = 4
	}
	b := make([]byte, 256*1024)
	if amt, _, err := unix.Syscall(unix.SYS_SYSLOG, l, uintptr(unsafe.Pointer(&b[0])), uintptr(len(b))); err == 0 {
		os.Stdout.Write(b[:amt])
	} else {
		log.Fatalf("syslog failed: %v", err)
	}
}
<commit_msg>cleanup: dmesg<commit_after>// Copyright 2012-2017 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Read the system log.
//
// Synopsis:
//     dmesg [-clear|-read-clear]
//
// Options:
//     -clear: clear the log
//     -read-clear: clear the log after printing
package main

import (
	"flag"
	"log"
	"os"
	"syscall"
	"unsafe"
)

const (
	_SYSLOG_ACTION_READ       = 2
	_SYSLOG_ACTION_READ_ALL   = 3
	_SYSLOG_ACTION_READ_CLEAR = 4
	_SYSLOG_ACTION_CLEAR      = 5
)

var (
	clear     bool
	readClear bool
)

func init() {
	flag.BoolVar(&clear, "clear", false, "Clear the log")
	flag.BoolVar(&readClear, "read-clear", false, "Clear the log after printing")
	flag.BoolVar(&readClear, "c", false, "Clear the log after printing")
}

func main() {
	flag.Parse()
	if clear && readClear {
		log.Fatalf("cannot specify both -clear and -read-clear")
	}

	level := uintptr(_SYSLOG_ACTION_READ_ALL)
	if clear {
		level = _SYSLOG_ACTION_CLEAR
	}
	if readClear {
		level = _SYSLOG_ACTION_READ_CLEAR
	}

	b := make([]byte, 256*1024)
	amt, _, err := syscall.Syscall(syscall.SYS_SYSLOG, level, uintptr(unsafe.Pointer(&b[0])), uintptr(len(b)))
	if err != 0 {
		log.Fatalf("syslog failed: %v", err)
	}

	os.Stdout.Write(b[:amt])
}
<|endoftext|>


<commit_before>package river

import (
	"flag"
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/siddontang/go-mysql-elasticsearch/elastic"
	"github.com/siddontang/go-mysql/client"
	. "gopkg.in/check.v1"
)

var my_addr = flag.String("my_addr", "127.0.0.1:3306", "MySQL addr")
var es_addr = flag.String("es_addr", "127.0.0.1:9200", "Elasticsearch addr")

func Test(t *testing.T) {
	TestingT(t)
}

type riverTestSuite struct {
	c *client.Conn
	r *River
}

var _ = Suite(&riverTestSuite{})

func (s *riverTestSuite) SetUpSuite(c *C) {
	var err error
	s.c, err = client.Connect(*my_addr, "root", "", "test")
	c.Assert(err, IsNil)

	s.testExecute(c, "SET SESSION binlog_format = 'ROW'")

	schema := `
        CREATE TABLE IF NOT EXISTS %s (
            id INT,
            title VARCHAR(256),
            content VARCHAR(256),
            mylist VARCHAR(256),
            tenum ENUM("e1", "e2", "e3"),
            tset SET("a", "b", "c"),
            tbit BIT(1) default 1,
            PRIMARY KEY(id)) ENGINE=INNODB;
    `

	schema_json := `
	CREATE TABLE IF NOT EXISTS %s (
	    id INT,
	    info JSON,
	    PRIMARY KEY(id)) ENGINE=INNODB;
    `

	s.testExecute(c, "DROP TABLE IF EXISTS test_river")
	s.testExecute(c, "DROP TABLE IF EXISTS test_for_id")
	s.testExecute(c, "DROP TABLE IF EXISTS test_for_json")
	s.testExecute(c, "DROP TABLE IF EXISTS test_river")
	s.testExecute(c, fmt.Sprintf(schema, "test_river"))
	s.testExecute(c, fmt.Sprintf(schema, "test_for_id"))
	s.testExecute(c, fmt.Sprintf(schema_json, "test_for_json"))

	for i := 0; i < 10; i++ {
		table := fmt.Sprintf("test_river_%04d", i)
		s.testExecute(c, fmt.Sprintf("DROP TABLE IF EXISTS %s", table))
		s.testExecute(c, fmt.Sprintf(schema, table))
	}

	cfg := new(Config)
	cfg.MyAddr = *my_addr
	cfg.MyUser = "root"
	cfg.MyPassword = ""
	cfg.MyCharset = "utf8"
	cfg.ESAddr = *es_addr

	cfg.ServerID = 1001
	cfg.Flavor = "mysql"

	cfg.DataDir = "/tmp/test_river"
	cfg.DumpExec = "mysqldump"

	cfg.StatAddr = "127.0.0.1:12800"
	cfg.BulkSize = 1
	cfg.FlushBulkTime = TomlDuration{3 * time.Millisecond}

	os.RemoveAll(cfg.DataDir)

	cfg.Sources = []SourceConfig{SourceConfig{Schema: "test", Tables: []string{"test_river", "test_river_[0-9]{4}", "test_for_id", "test_for_json"}}}

	cfg.Rules = []*Rule{
		&Rule{Schema: "test",
			Table:        "test_river",
			Index:        "river",
			Type:         "river",
			FieldMapping: map[string]string{"title": "es_title", "mylist": "es_mylist,list"},
		},

		&Rule{Schema: "test",
			Table:        "test_for_id",
			Index:        "river",
			Type:         "river",
			ID:           []string{"id", "title"},
			FieldMapping: map[string]string{"title": "es_title", "mylist": "es_mylist,list"},
		},

		&Rule{Schema: "test",
			Table:        "test_river_[0-9]{4}",
			Index:        "river",
			Type:         "river",
			FieldMapping: map[string]string{"title": "es_title", "mylist": "es_mylist,list"},
		},

		&Rule{Schema: "test",
			Table:        "test_for_json",
			Index:        "river",
			Type:         "river",
		},
	}

	s.r, err = NewRiver(cfg)
	c.Assert(err, IsNil)

	err = s.r.es.DeleteIndex("river")
	c.Assert(err, IsNil)
}

func (s *riverTestSuite) TearDownSuite(c *C) {
	if s.c != nil {
		s.c.Close()
	}

	if s.r != nil {
		s.r.Close()
	}
}

func (s *riverTestSuite) TestConfig(c *C) {
	str := `
my_addr = "127.0.0.1:3306"
my_user = "root"
my_pass = ""
my_charset = "utf8"
es_addr = "127.0.0.1:9200"
es_user = ""
es_pass = ""
data_dir = "./var"

[[source]]
schema = "test"

tables = ["test_river", "test_river_[0-9]{4}", "test_for_id", "test_for_json"]

[[rule]]
schema = "test"
table = "test_river"
index = "river"
type = "river"
parent = "pid"

    [rule.field]
    title = "es_title"
    mylist = "es_mylist,list"

[[rule]]
schema = "test"
table = "test_for_id"
index = "river"
type = "river"
parent = "pid"
id = ["id", "title"]
    [rule.field]
    title = "es_title"
    mylist = "es_mylist,list"

[[rule]]
schema = "test"
table = "test_river_[0-9]{4}"
index = "river"
type = "river"

    [rule.field]
    title = "es_title"
    mylist = "es_mylist,list"

[[rule]]
schema = "test"
table = "test_for_json"
index = "river"
type = "river"
`

	cfg, err := NewConfig(str)
	c.Assert(err, IsNil)
	c.Assert(cfg.Sources, HasLen, 1)
	c.Assert(cfg.Sources[0].Tables, HasLen, 4)
	c.Assert(cfg.Rules, HasLen, 4)
}

func (s *riverTestSuite) testExecute(c *C, query string, args ...interface{}) {
	_, err := s.c.Execute(query, args...)
	c.Assert(err, IsNil)
}

func (s *riverTestSuite) testPrepareData(c *C) {
	s.testExecute(c, "INSERT INTO test_river (id, title, content, tenum, tset) VALUES (?, ?, ?, ?, ?)", 1, "first", "hello go 1", "e1", "a,b")
	s.testExecute(c, "INSERT INTO test_river (id, title, content, tenum, tset) VALUES (?, ?, ?, ?, ?)", 2, "second", "hello mysql 2", "e2", "b,c")
	s.testExecute(c, "INSERT INTO test_river (id, title, content, tenum, tset) VALUES (?, ?, ?, ?, ?)", 3, "third", "hello elaticsearch 3", "e3", "c")
	s.testExecute(c, "INSERT INTO test_river (id, title, content, tenum, tset, tbit) VALUES (?, ?, ?, ?, ?, ?)", 4, "fouth", "hello go-mysql-elasticserach 4", "e1", "a,b,c", 0)
	s.testExecute(c, "INSERT INTO test_for_id (id, title, content, tenum, tset) VALUES (?, ?, ?, ?, ?)", 1, "first", "hello go 1", "e1", "a,b")
	s.testExecute(c, "INSERT INTO test_for_json (id, json) VALUES (?, ?)", 9200, "{\"first\": \"a\", \"second\": \"b\"}")

	for i := 0; i < 10; i++ {
		table := fmt.Sprintf("test_river_%04d", i)
		s.testExecute(c, fmt.Sprintf("INSERT INTO %s (id, title, content, tenum, tset) VALUES (?, ?, ?, ?, ?)", table), 5+i, "abc", "hello", "e1", "a,b,c")
	}
}

func (s *riverTestSuite) testElasticGet(c *C, id string) *elastic.Response {
	index := "river"
	docType := "river"

	r, err := s.r.es.Get(index, docType, id)
	c.Assert(err, IsNil)

	return r
}

func testWaitSyncDone(c *C, r *River) {
	<-r.canal.WaitDumpDone()

	err := r.canal.CatchMasterPos(10 * time.Second)
	c.Assert(err, IsNil)

	for i := 0; i < 1000; i++ {
		if len(r.syncCh) == 0 {
			return
		}

		time.Sleep(10 * time.Millisecond)
	}

	c.Fatalf("wait 1s but still have %d items to be synced", len(r.syncCh))
}

func (s *riverTestSuite) TestRiver(c *C) {
	s.testPrepareData(c)

	s.r.Start()

	testWaitSyncDone(c, s.r)

	var r *elastic.Response
	r = s.testElasticGet(c, "1")
	c.Assert(r.Found, Equals, true)
	c.Assert(r.Source["tenum"], Equals, "e1")
	c.Assert(r.Source["tset"], Equals, "a,b")

	r = s.testElasticGet(c, "1:first")
	c.Assert(r.Found, Equals, true)

	r = s.testElasticGet(c, "9200")
	c.Assert(r.Found, Equals, true)
	c.Assert(r.Source["info"]["first"], Equals, "a")
	c.Assert(r.Source["info"]["second"], Equals, "b")

	r = s.testElasticGet(c, "100")
	c.Assert(r.Found, Equals, false)

	for i := 0; i < 10; i++ {
		r = s.testElasticGet(c, fmt.Sprintf("%d", 5+i))
		c.Assert(r.Found, Equals, true)
		c.Assert(r.Source["es_title"], Equals, "abc")
	}

	s.testExecute(c, "UPDATE test_river SET title = ?, tenum = ?, tset = ?, mylist = ? WHERE id = ?", "second 2", "e3", "a,b,c", "a,b,c", 2)
	s.testExecute(c, "DELETE FROM test_river WHERE id = ?", 1)
	s.testExecute(c, "UPDATE test_river SET title = ?, id = ? WHERE id = ?", "second 30", 30, 3)

	// so we can insert invalid data
	s.testExecute(c, `SET SESSION sql_mode="NO_ENGINE_SUBSTITUTION";`)

	// bad insert
	s.testExecute(c, "UPDATE test_river SET title = ?, tenum = ?, tset = ? WHERE id = ?", "second 2", "e5", "a,b,c,d", 4)

	for i := 0; i < 10; i++ {
		table := fmt.Sprintf("test_river_%04d", i)
		s.testExecute(c, fmt.Sprintf("UPDATE %s SET title = ? WHERE id = ?", table), "hello", 5+i)
	}

	testWaitSyncDone(c, s.r)

	r = s.testElasticGet(c, "1")
	c.Assert(r.Found, Equals, false)

	r = s.testElasticGet(c, "2")
	c.Assert(r.Found, Equals, true)
	c.Assert(r.Source["es_title"], Equals, "second 2")
	c.Assert(r.Source["tenum"], Equals, "e3")
	c.Assert(r.Source["tset"], Equals, "a,b,c")
	c.Assert(r.Source["es_mylist"], DeepEquals, []interface{}{"a", "b", "c"})
	c.Assert(r.Source["tbit"], Equals, float64(1))

	r = s.testElasticGet(c, "4")
	c.Assert(r.Found, Equals, true)
	c.Assert(r.Source["tenum"], Equals, "")
	c.Assert(r.Source["tset"], Equals, "a,b,c")
	c.Assert(r.Source["tbit"], Equals, float64(0))

	r = s.testElasticGet(c, "3")
	c.Assert(r.Found, Equals, false)

	r = s.testElasticGet(c, "30")
	c.Assert(r.Found, Equals, true)
	c.Assert(r.Source["es_title"], Equals, "second 30")

	for i := 0; i < 10; i++ {
		r = s.testElasticGet(c, fmt.Sprintf("%d", 5+i))
		c.Assert(r.Found, Equals, true)
		c.Assert(r.Source["es_title"], Equals, "hello")
	}
}
<commit_msg>fix the bug of test<commit_after>package river

import (
	"flag"
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/siddontang/go-mysql-elasticsearch/elastic"
	"github.com/siddontang/go-mysql/client"
	. "gopkg.in/check.v1"
)

var my_addr = flag.String("my_addr", "127.0.0.1:3306", "MySQL addr")
var es_addr = flag.String("es_addr", "127.0.0.1:9200", "Elasticsearch addr")

func Test(t *testing.T) {
	TestingT(t)
}

type riverTestSuite struct {
	c *client.Conn
	r *River
}

var _ = Suite(&riverTestSuite{})

func (s *riverTestSuite) SetUpSuite(c *C) {
	var err error
	s.c, err = client.Connect(*my_addr, "root", "", "test")
	c.Assert(err, IsNil)

	s.testExecute(c, "SET SESSION binlog_format = 'ROW'")

	schema := `
        CREATE TABLE IF NOT EXISTS %s (
            id INT,
            title VARCHAR(256),
            content VARCHAR(256),
            mylist VARCHAR(256),
            tenum ENUM("e1", "e2", "e3"),
            tset SET("a", "b", "c"),
            tbit BIT(1) default 1,
            PRIMARY KEY(id)) ENGINE=INNODB;
    `

	schema_json := `
	CREATE TABLE IF NOT EXISTS %s (
	    id INT,
	    info JSON,
	    PRIMARY KEY(id)) ENGINE=INNODB;
    `

	s.testExecute(c, "DROP TABLE IF EXISTS test_river")
	s.testExecute(c, "DROP TABLE IF EXISTS test_for_id")
	s.testExecute(c, "DROP TABLE IF EXISTS test_for_json")
	s.testExecute(c, "DROP TABLE IF EXISTS test_river")
	s.testExecute(c, fmt.Sprintf(schema, "test_river"))
	s.testExecute(c, fmt.Sprintf(schema, "test_for_id"))
	s.testExecute(c, fmt.Sprintf(schema_json, "test_for_json"))

	for i := 0; i < 10; i++ {
		table := fmt.Sprintf("test_river_%04d", i)
		s.testExecute(c, fmt.Sprintf("DROP TABLE IF EXISTS %s", table))
		s.testExecute(c, fmt.Sprintf(schema, table))
	}

	cfg := new(Config)
	cfg.MyAddr = *my_addr
	cfg.MyUser = "root"
	cfg.MyPassword = ""
	cfg.MyCharset = "utf8"
	cfg.ESAddr = *es_addr

	cfg.ServerID = 1001
	cfg.Flavor = "mysql"

	cfg.DataDir = "/tmp/test_river"
	cfg.DumpExec = "mysqldump"

	cfg.StatAddr = "127.0.0.1:12800"
	cfg.BulkSize = 1
	cfg.FlushBulkTime = TomlDuration{3 * time.Millisecond}

	os.RemoveAll(cfg.DataDir)

	cfg.Sources = []SourceConfig{SourceConfig{Schema: "test", Tables: []string{"test_river", "test_river_[0-9]{4}", "test_for_id", "test_for_json"}}}

	cfg.Rules = []*Rule{
		&Rule{Schema: "test",
			Table:        "test_river",
			Index:        "river",
			Type:         "river",
			FieldMapping: map[string]string{"title": "es_title", "mylist": "es_mylist,list"},
		},

		&Rule{Schema: "test",
			Table:        "test_for_id",
			Index:        "river",
			Type:         "river",
			ID:           []string{"id", "title"},
			FieldMapping: map[string]string{"title": "es_title", "mylist": "es_mylist,list"},
		},

		&Rule{Schema: "test",
			Table:        "test_river_[0-9]{4}",
			Index:        "river",
			Type:         "river",
			FieldMapping: map[string]string{"title": "es_title", "mylist": "es_mylist,list"},
		},

		&Rule{Schema: "test",
			Table:        "test_for_json",
			Index:        "river",
			Type:         "river",
		},
	}

	s.r, err = NewRiver(cfg)
	c.Assert(err, IsNil)

	err = s.r.es.DeleteIndex("river")
	c.Assert(err, IsNil)
}

func (s *riverTestSuite) TearDownSuite(c *C) {
	if s.c != nil {
		s.c.Close()
	}

	if s.r != nil {
		s.r.Close()
	}
}

func (s *riverTestSuite) TestConfig(c *C) {
	str := `
my_addr = "127.0.0.1:3306"
my_user = "root"
my_pass = ""
my_charset = "utf8"
es_addr = "127.0.0.1:9200"
es_user = ""
es_pass = ""
data_dir = "./var"

[[source]]
schema = "test"

tables = ["test_river", "test_river_[0-9]{4}", "test_for_id", "test_for_json"]

[[rule]]
schema = "test"
table = "test_river"
index = "river"
type = "river"
parent = "pid"

    [rule.field]
    title = "es_title"
    mylist = "es_mylist,list"

[[rule]]
schema = "test"
table = "test_for_id"
index = "river"
type = "river"
parent = "pid"
id = ["id", "title"]
    [rule.field]
    title = "es_title"
    mylist = "es_mylist,list"

[[rule]]
schema = "test"
table = "test_river_[0-9]{4}"
index = "river"
type = "river"

    [rule.field]
    title = "es_title"
    mylist = "es_mylist,list"

[[rule]]
schema = "test"
table = "test_for_json"
index = "river"
type = "river"
`

	cfg, err := NewConfig(str)
	c.Assert(err, IsNil)
	c.Assert(cfg.Sources, HasLen, 1)
	c.Assert(cfg.Sources[0].Tables, HasLen, 4)
	c.Assert(cfg.Rules, HasLen, 4)
}

func (s *riverTestSuite) testExecute(c *C, query string, args ...interface{}) {
	_, err := s.c.Execute(query, args...)
	c.Assert(err, IsNil)
}

func (s *riverTestSuite) testPrepareData(c *C) {
	s.testExecute(c, "INSERT INTO test_river (id, title, content, tenum, tset) VALUES (?, ?, ?, ?, ?)", 1, "first", "hello go 1", "e1", "a,b")
	s.testExecute(c, "INSERT INTO test_river (id, title, content, tenum, tset) VALUES (?, ?, ?, ?, ?)", 2, "second", "hello mysql 2", "e2", "b,c")
	s.testExecute(c, "INSERT INTO test_river (id, title, content, tenum, tset) VALUES (?, ?, ?, ?, ?)", 3, "third", "hello elaticsearch 3", "e3", "c")
	s.testExecute(c, "INSERT INTO test_river (id, title, content, tenum, tset, tbit) VALUES (?, ?, ?, ?, ?, ?)", 4, "fouth", "hello go-mysql-elasticserach 4", "e1", "a,b,c", 0)
	s.testExecute(c, "INSERT INTO test_for_id (id, title, content, tenum, tset) VALUES (?, ?, ?, ?, ?)", 1, "first", "hello go 1", "e1", "a,b")
	s.testExecute(c, "INSERT INTO test_for_json (id, json) VALUES (?, ?)", 9200, "{\"first\": \"a\", \"second\": \"b\"}")

	for i := 0; i < 10; i++ {
		table := fmt.Sprintf("test_river_%04d", i)
		s.testExecute(c, fmt.Sprintf("INSERT INTO %s (id, title, content, tenum, tset) VALUES (?, ?, ?, ?, ?)", table), 5+i, "abc", "hello", "e1", "a,b,c")
	}
}

func (s *riverTestSuite) testElasticGet(c *C, id string) *elastic.Response {
	index := "river"
	docType := "river"

	r, err := s.r.es.Get(index, docType, id)
	c.Assert(err, IsNil)

	return r
}

func testWaitSyncDone(c *C, r *River) {
	<-r.canal.WaitDumpDone()

	err := r.canal.CatchMasterPos(10 * time.Second)
	c.Assert(err, IsNil)

	for i := 0; i < 1000; i++ {
		if len(r.syncCh) == 0 {
			return
		}

		time.Sleep(10 * time.Millisecond)
	}

	c.Fatalf("wait 1s but still have %d items to be synced", len(r.syncCh))
}

func (s *riverTestSuite) TestRiver(c *C) {
	s.testPrepareData(c)

	s.r.Start()

	testWaitSyncDone(c, s.r)

	var r *elastic.Response
	r = s.testElasticGet(c, "1")
	c.Assert(r.Found, Equals, true)
	c.Assert(r.Source["tenum"], Equals, "e1")
	c.Assert(r.Source["tset"], Equals, "a,b")

	r = s.testElasticGet(c, "1:first")
	c.Assert(r.Found, Equals, true)

	r = s.testElasticGet(c, "9200")
	c.Assert(r.Found, Equals, true)
	switch v := r.source["info"].(type) {
	case map[string]string:
	    c.Assert(v["first"], Equals, "a")
	    c.Assert(v["second"], Equals, "b")
	default:
	    c.Assert(true, Equals, false)
	}

	r = s.testElasticGet(c, "100")
	c.Assert(r.Found, Equals, false)

	for i := 0; i < 10; i++ {
		r = s.testElasticGet(c, fmt.Sprintf("%d", 5+i))
		c.Assert(r.Found, Equals, true)
		c.Assert(r.Source["es_title"], Equals, "abc")
	}

	s.testExecute(c, "UPDATE test_river SET title = ?, tenum = ?, tset = ?, mylist = ? WHERE id = ?", "second 2", "e3", "a,b,c", "a,b,c", 2)
	s.testExecute(c, "DELETE FROM test_river WHERE id = ?", 1)
	s.testExecute(c, "UPDATE test_river SET title = ?, id = ? WHERE id = ?", "second 30", 30, 3)

	// so we can insert invalid data
	s.testExecute(c, `SET SESSION sql_mode="NO_ENGINE_SUBSTITUTION";`)

	// bad insert
	s.testExecute(c, "UPDATE test_river SET title = ?, tenum = ?, tset = ? WHERE id = ?", "second 2", "e5", "a,b,c,d", 4)

	for i := 0; i < 10; i++ {
		table := fmt.Sprintf("test_river_%04d", i)
		s.testExecute(c, fmt.Sprintf("UPDATE %s SET title = ? WHERE id = ?", table), "hello", 5+i)
	}

	testWaitSyncDone(c, s.r)

	r = s.testElasticGet(c, "1")
	c.Assert(r.Found, Equals, false)

	r = s.testElasticGet(c, "2")
	c.Assert(r.Found, Equals, true)
	c.Assert(r.Source["es_title"], Equals, "second 2")
	c.Assert(r.Source["tenum"], Equals, "e3")
	c.Assert(r.Source["tset"], Equals, "a,b,c")
	c.Assert(r.Source["es_mylist"], DeepEquals, []interface{}{"a", "b", "c"})
	c.Assert(r.Source["tbit"], Equals, float64(1))

	r = s.testElasticGet(c, "4")
	c.Assert(r.Found, Equals, true)
	c.Assert(r.Source["tenum"], Equals, "")
	c.Assert(r.Source["tset"], Equals, "a,b,c")
	c.Assert(r.Source["tbit"], Equals, float64(0))

	r = s.testElasticGet(c, "3")
	c.Assert(r.Found, Equals, false)

	r = s.testElasticGet(c, "30")
	c.Assert(r.Found, Equals, true)
	c.Assert(r.Source["es_title"], Equals, "second 30")

	for i := 0; i < 10; i++ {
		r = s.testElasticGet(c, fmt.Sprintf("%d", 5+i))
		c.Assert(r.Found, Equals, true)
		c.Assert(r.Source["es_title"], Equals, "hello")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"errors"
	"flag"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"strings"
	"math"
	"time"

	_ "github.com/mattn/go-sqlite3"
	"github.com/ungerik/go-rss"
)

// CREATE TABLE feeds (name TEXT PRIMARY KEY, url TEXT NOT NULL, dayOfWeek INTEGER NOT NULL,
// seconds INTEGER NOT NULL, lastTitle TEXT NOT NULL);

type updatedTitleMessage struct {
	Name  string
	Title string
}

// Flag specifications.
var dbFilename = flag.String("db_file", "feeds.db", "filename of database to use")
var logFilename = flag.String("log_file", "", "filename to log to")
var target = flag.String("target", "", "target directory to download to")
var checkInterval = flag.Int(
	"check_interval", 3600, "seconds between checks during normal operation")
var rapidCheckInterval = flag.Int(
	"rapid_check_interval", 60, "seconds between checks when we suspect there will be a new item")
var rapidCheckDuration = flag.Int(
	"rapid_check_duration", 3600, "seconds that we suspect there will be a new item")
var downloadDelay = flag.Int(
	"download_delay", 30, "seconds to wait before downloading the file")
var requestDelay = flag.Int(
	"request_delay", 5, "seconds to wait between requests")

var requestDelayTicker <-chan time.Time

func downloadUrl(url string) error {
	// Figure out the filename to download to.
	lastSeparatorIndex := strings.LastIndex(url, "/")
	if lastSeparatorIndex == -1 {
		return errors.New("malformed url (no slash!?)")
	}
	filename := url[lastSeparatorIndex+1:]
	if len(filename) == 0 {
		return errors.New("malformed url (no filename)")
	}
	filepath := path.Join(*target, filename)

	// Actually download it.
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	file, err := os.Create(filepath)
	if err != nil {
		return err
	}
	defer file.Close()

	_, err = io.Copy(file, resp.Body)
	if err != nil {
		return err
	}
	return nil
}

func lastRapidStartTime(fromTime time.Time, dayOfWeek int, seconds int) time.Time {
	dayDiff := dayOfWeek - int(fromTime.Weekday())
	if dayDiff > 0 {
		dayDiff -= 7
	}

	if dayDiff == 0 {
		if fromTime.Before(time.Date(fromTime.Year(), fromTime.Month(), fromTime.Day(), 0, 0, seconds, 0, time.Local)) {
			dayDiff -= 7
		}
	}

	return time.Date(
		fromTime.Year(), fromTime.Month(), fromTime.Day()+dayDiff, 0, 0, seconds, 0, time.Local)
}

func nextRapidStartTime(fromTime time.Time, dayOfWeek int, seconds int) time.Time {
	return lastRapidStartTime(fromTime.AddDate(0, 0, 7), dayOfWeek, seconds)
}

func isRapid(fromTime time.Time, dayOfWeek int, seconds int) bool {
	rapidStartTime := lastRapidStartTime(fromTime, dayOfWeek, seconds)
	return fromTime.Equal(rapidStartTime) || (fromTime.After(rapidStartTime) && fromTime.Before(rapidStartTime.Add(time.Duration(*rapidCheckDuration)*time.Second)))
}

func nextCheckTime(lastCheckTime time.Time, dayOfWeek int, seconds int) time.Time {
	var nextCheckTime time.Time

	if isRapid(lastCheckTime, dayOfWeek, seconds) {
		nextCheckTime = lastCheckTime.Add(time.Duration(*rapidCheckInterval) * time.Second)
	} else {
		nextCheckTime = lastCheckTime.Add(time.Duration(*checkInterval) * time.Second)
	}

	nextRapidTime := nextRapidStartTime(lastCheckTime, dayOfWeek, seconds)
	if nextCheckTime.After(nextRapidTime) {
		nextCheckTime = nextRapidTime
	}

	return nextCheckTime
}

func firstCheckTime(startTime time.Time, dayOfWeek int, seconds int) time.Time {
	// Grab info from last rapid start time.
	baseTime := lastRapidStartTime(startTime, dayOfWeek, seconds)
	var currentCheckInterval float64
	if isRapid(startTime, dayOfWeek, seconds) {
		currentCheckInterval = float64(*rapidCheckInterval)
	} else {
		baseTime = baseTime.Add(time.Duration(*rapidCheckDuration) * time.Second)
		currentCheckInterval = float64(*checkInterval)
	}

	// Calculate next check time.
	secondsOffsetFromBase := startTime.Sub(baseTime).Seconds()
	nextCheckOffsetFromBase := currentCheckInterval * math.Ceil(secondsOffsetFromBase / currentCheckInterval)
	nextCheckTime := baseTime.Add(time.Duration(nextCheckOffsetFromBase) * time.Second)

	// Fixup check time if it happens to be after the next rapid start time.
	nextRapidTime := nextRapidStartTime(startTime, dayOfWeek, seconds)
	if nextCheckTime.After(nextRapidTime) {
		nextCheckTime = nextRapidTime
	}

	return nextCheckTime
}

func watchFeed(
	messages chan updatedTitleMessage, name string, feedUrl string, dayOfWeek int, seconds int,
	lastTitle string) {
	log.Printf("[%s] Starting watch.", name)

	checkTime := firstCheckTime(time.Now(), dayOfWeek, seconds)

	// Main loop.
	for {
		// Wait until the next check time.
		time.Sleep(checkTime.Sub(time.Now()))
		checkTime = nextCheckTime(checkTime, dayOfWeek, seconds)

		// Fetch RSS.
		<-requestDelayTicker
		log.Printf("[%s] Checking for new items.", name)
		feed, err := rss.Read(feedUrl)
		if err != nil {
			log.Printf("[%s] Error fetching RSS: %s", name, err)
		} else {
			// Download any new files.
			for i := 0; i < len(feed.Item); i++ {
				if feed.Item[i].Title == lastTitle {
					break
				}

				log.Printf("[%s] Fetching %s.", name, feed.Item[i].Title)
				go func(title string, url string) {
					if *downloadDelay > 0 {
						time.Sleep(time.Duration(*downloadDelay) * time.Second)
					}
					<-requestDelayTicker
					err := downloadUrl(url)
					if err != nil {
						log.Printf("[%s] Error fetching %s: %s", name, url, err)
					} else {
						log.Printf("[%s] Fetched %s.", name, title)
					}
				}(feed.Item[i].Title, feed.Item[i].Link)
			}

			// Update last seen title.
			if len(feed.Item) > 0 {
				newTitle := feed.Item[0].Title
				if lastTitle != newTitle {
					lastTitle = newTitle
					messages <- updatedTitleMessage{name, lastTitle}
				}
			}
		}
	}
}

func main() {
	// Check flags.
	flag.Parse()
	if *target == "" {
		log.Fatal("--target is required.")
	}

	// Set up logging.
	if *logFilename != "" {
		logWriter, err := os.OpenFile(*logFilename, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666)
		if err != nil {
			log.Fatal("--log_file could not be opened")
		}

		log.SetOutput(logWriter)
		defer logWriter.Close()
	}

	log.Print("Starting rss-downloader.")
	requestDelayTicker = time.Tick(time.Duration(*requestDelay) * time.Second)

	// Connect to database.
	db, err := sql.Open("sqlite3", *dbFilename)
	if err != nil {
		log.Fatalf("Error opening database connection: %s", err)
	}
	defer db.Close()

	// Start watching.
	messages := make(chan updatedTitleMessage)
	rows, err := db.Query("SELECT name, url, dayOfWeek, seconds, lastTitle FROM feeds")
	if err != nil {
		log.Fatalf("Error reading RSS feeds: %s", err)
	}
	for rows.Next() {
		var name string
		var url string
		var dayOfWeek int
		var seconds int
		var lastTitle string

		if err := rows.Scan(&name, &url, &dayOfWeek, &seconds, &lastTitle); err != nil {
			log.Fatalf("Error reading RSS feeds: %s", err)
		}

		go watchFeed(messages, name, url, dayOfWeek, seconds, lastTitle)
	}
	if err := rows.Err(); err != nil {
		log.Fatalf("Error reading RSS feeds: %s", err)
	}

	for {
		msg := <-messages
		_, err := db.Exec(
			"UPDATE feeds SET lastTitle = ? WHERE name = ?", msg.Title, msg.Name)
		if err != nil {
			log.Printf("[%s] Error updating last title: %s", err)
		}
	}
}
<commit_msg>go fmt<commit_after>package main

import (
	"database/sql"
	"errors"
	"flag"
	"io"
	"log"
	"math"
	"net/http"
	"os"
	"path"
	"strings"
	"time"

	_ "github.com/mattn/go-sqlite3"
	"github.com/ungerik/go-rss"
)

// CREATE TABLE feeds (name TEXT PRIMARY KEY, url TEXT NOT NULL, dayOfWeek INTEGER NOT NULL,
// seconds INTEGER NOT NULL, lastTitle TEXT NOT NULL);

type updatedTitleMessage struct {
	Name  string
	Title string
}

// Flag specifications.
var dbFilename = flag.String("db_file", "feeds.db", "filename of database to use")
var logFilename = flag.String("log_file", "", "filename to log to")
var target = flag.String("target", "", "target directory to download to")
var checkInterval = flag.Int(
	"check_interval", 3600, "seconds between checks during normal operation")
var rapidCheckInterval = flag.Int(
	"rapid_check_interval", 60, "seconds between checks when we suspect there will be a new item")
var rapidCheckDuration = flag.Int(
	"rapid_check_duration", 3600, "seconds that we suspect there will be a new item")
var downloadDelay = flag.Int(
	"download_delay", 30, "seconds to wait before downloading the file")
var requestDelay = flag.Int(
	"request_delay", 5, "seconds to wait between requests")

var requestDelayTicker <-chan time.Time

func downloadUrl(url string) error {
	// Figure out the filename to download to.
	lastSeparatorIndex := strings.LastIndex(url, "/")
	if lastSeparatorIndex == -1 {
		return errors.New("malformed url (no slash!?)")
	}
	filename := url[lastSeparatorIndex+1:]
	if len(filename) == 0 {
		return errors.New("malformed url (no filename)")
	}
	filepath := path.Join(*target, filename)

	// Actually download it.
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	file, err := os.Create(filepath)
	if err != nil {
		return err
	}
	defer file.Close()

	_, err = io.Copy(file, resp.Body)
	if err != nil {
		return err
	}
	return nil
}

func lastRapidStartTime(fromTime time.Time, dayOfWeek int, seconds int) time.Time {
	dayDiff := dayOfWeek - int(fromTime.Weekday())
	if dayDiff > 0 {
		dayDiff -= 7
	}

	if dayDiff == 0 {
		if fromTime.Before(time.Date(fromTime.Year(), fromTime.Month(), fromTime.Day(), 0, 0, seconds, 0, time.Local)) {
			dayDiff -= 7
		}
	}

	return time.Date(
		fromTime.Year(), fromTime.Month(), fromTime.Day()+dayDiff, 0, 0, seconds, 0, time.Local)
}

func nextRapidStartTime(fromTime time.Time, dayOfWeek int, seconds int) time.Time {
	return lastRapidStartTime(fromTime.AddDate(0, 0, 7), dayOfWeek, seconds)
}

func isRapid(fromTime time.Time, dayOfWeek int, seconds int) bool {
	rapidStartTime := lastRapidStartTime(fromTime, dayOfWeek, seconds)
	return fromTime.Equal(rapidStartTime) || (fromTime.After(rapidStartTime) && fromTime.Before(rapidStartTime.Add(time.Duration(*rapidCheckDuration)*time.Second)))
}

func nextCheckTime(lastCheckTime time.Time, dayOfWeek int, seconds int) time.Time {
	var nextCheckTime time.Time

	if isRapid(lastCheckTime, dayOfWeek, seconds) {
		nextCheckTime = lastCheckTime.Add(time.Duration(*rapidCheckInterval) * time.Second)
	} else {
		nextCheckTime = lastCheckTime.Add(time.Duration(*checkInterval) * time.Second)
	}

	nextRapidTime := nextRapidStartTime(lastCheckTime, dayOfWeek, seconds)
	if nextCheckTime.After(nextRapidTime) {
		nextCheckTime = nextRapidTime
	}

	return nextCheckTime
}

func firstCheckTime(startTime time.Time, dayOfWeek int, seconds int) time.Time {
	// Grab info from last rapid start time.
	baseTime := lastRapidStartTime(startTime, dayOfWeek, seconds)
	var currentCheckInterval float64
	if isRapid(startTime, dayOfWeek, seconds) {
		currentCheckInterval = float64(*rapidCheckInterval)
	} else {
		baseTime = baseTime.Add(time.Duration(*rapidCheckDuration) * time.Second)
		currentCheckInterval = float64(*checkInterval)
	}

	// Calculate next check time.
	secondsOffsetFromBase := startTime.Sub(baseTime).Seconds()
	nextCheckOffsetFromBase := currentCheckInterval * math.Ceil(secondsOffsetFromBase/currentCheckInterval)
	nextCheckTime := baseTime.Add(time.Duration(nextCheckOffsetFromBase) * time.Second)

	// Fixup check time if it happens to be after the next rapid start time.
	nextRapidTime := nextRapidStartTime(startTime, dayOfWeek, seconds)
	if nextCheckTime.After(nextRapidTime) {
		nextCheckTime = nextRapidTime
	}

	return nextCheckTime
}

func watchFeed(
	messages chan updatedTitleMessage, name string, feedUrl string, dayOfWeek int, seconds int,
	lastTitle string) {
	log.Printf("[%s] Starting watch.", name)

	checkTime := firstCheckTime(time.Now(), dayOfWeek, seconds)

	// Main loop.
	for {
		// Wait until the next check time.
		time.Sleep(checkTime.Sub(time.Now()))
		checkTime = nextCheckTime(checkTime, dayOfWeek, seconds)

		// Fetch RSS.
		<-requestDelayTicker
		log.Printf("[%s] Checking for new items.", name)
		feed, err := rss.Read(feedUrl)
		if err != nil {
			log.Printf("[%s] Error fetching RSS: %s", name, err)
		} else {
			// Download any new files.
			for i := 0; i < len(feed.Item); i++ {
				if feed.Item[i].Title == lastTitle {
					break
				}

				log.Printf("[%s] Fetching %s.", name, feed.Item[i].Title)
				go func(title string, url string) {
					if *downloadDelay > 0 {
						time.Sleep(time.Duration(*downloadDelay) * time.Second)
					}
					<-requestDelayTicker
					err := downloadUrl(url)
					if err != nil {
						log.Printf("[%s] Error fetching %s: %s", name, url, err)
					} else {
						log.Printf("[%s] Fetched %s.", name, title)
					}
				}(feed.Item[i].Title, feed.Item[i].Link)
			}

			// Update last seen title.
			if len(feed.Item) > 0 {
				newTitle := feed.Item[0].Title
				if lastTitle != newTitle {
					lastTitle = newTitle
					messages <- updatedTitleMessage{name, lastTitle}
				}
			}
		}
	}
}

func main() {
	// Check flags.
	flag.Parse()
	if *target == "" {
		log.Fatal("--target is required.")
	}

	// Set up logging.
	if *logFilename != "" {
		logWriter, err := os.OpenFile(*logFilename, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666)
		if err != nil {
			log.Fatal("--log_file could not be opened")
		}

		log.SetOutput(logWriter)
		defer logWriter.Close()
	}

	log.Print("Starting rss-downloader.")
	requestDelayTicker = time.Tick(time.Duration(*requestDelay) * time.Second)

	// Connect to database.
	db, err := sql.Open("sqlite3", *dbFilename)
	if err != nil {
		log.Fatalf("Error opening database connection: %s", err)
	}
	defer db.Close()

	// Start watching.
	messages := make(chan updatedTitleMessage)
	rows, err := db.Query("SELECT name, url, dayOfWeek, seconds, lastTitle FROM feeds")
	if err != nil {
		log.Fatalf("Error reading RSS feeds: %s", err)
	}
	for rows.Next() {
		var name string
		var url string
		var dayOfWeek int
		var seconds int
		var lastTitle string

		if err := rows.Scan(&name, &url, &dayOfWeek, &seconds, &lastTitle); err != nil {
			log.Fatalf("Error reading RSS feeds: %s", err)
		}

		go watchFeed(messages, name, url, dayOfWeek, seconds, lastTitle)
	}
	if err := rows.Err(); err != nil {
		log.Fatalf("Error reading RSS feeds: %s", err)
	}

	for {
		msg := <-messages
		_, err := db.Exec(
			"UPDATE feeds SET lastTitle = ? WHERE name = ?", msg.Title, msg.Name)
		if err != nil {
			log.Printf("[%s] Error updating last title: %s", err)
		}
	}
}
<|endoftext|>


<commit_before>package rpc

import (
	"errors"
	"github.com/mitchellh/packer/packer"
	"net"
	"net/rpc"
)

// A Server is a Golang RPC server that has helper methods for automatically
// setting up the endpoints for Packer interfaces.
type Server struct {
	listener net.Listener
	server *rpc.Server
}

// Creates and returns a new Server.
func NewServer() *Server {
	return &Server{
		server: rpc.NewServer(),
	}
}

func (s *Server) Address() string {
	if s.listener == nil {
		panic("Server not listening.")
	}

	return s.listener.Addr().String()
}

func (s *Server) RegisterUi(ui packer.Ui) {
	s.server.RegisterName("Ui", &UiServer{ui})
}

func (s *Server) Start() error {
	if s.listener != nil {
		return errors.New("Server already started.")
	}

	// Start the TCP listener and a goroutine responsible for cleaning up the
	// listener.
	s.listener = netListenerInRange(portRangeMin, portRangeMax)
	if s.listener == nil {
		return errors.New("Could not open a port ot listen on.")
	}

	// Start accepting connections
	go func() {
		for {
			conn, err := s.listener.Accept()
			if err != nil {
				break
			}

			go s.server.ServeConn(conn)
		}
	}()

	return nil
}

func (s *Server) Stop() {
	if s.listener != nil {
		s.listener.Close()
		s.listener = nil
	}
}
<commit_msg>Fix race condition issue where listner could become nil<commit_after>package rpc

import (
	"errors"
	"github.com/mitchellh/packer/packer"
	"net"
	"net/rpc"
)

// A Server is a Golang RPC server that has helper methods for automatically
// setting up the endpoints for Packer interfaces.
type Server struct {
	listener net.Listener
	server *rpc.Server
}

// Creates and returns a new Server.
func NewServer() *Server {
	return &Server{
		server: rpc.NewServer(),
	}
}

func (s *Server) Address() string {
	if s.listener == nil {
		panic("Server not listening.")
	}

	return s.listener.Addr().String()
}

func (s *Server) RegisterUi(ui packer.Ui) {
	s.server.RegisterName("Ui", &UiServer{ui})
}

func (s *Server) Start() error {
	if s.listener != nil {
		return errors.New("Server already started.")
	}

	// Start the TCP listener and a goroutine responsible for cleaning up the
	// listener.
	s.listener = netListenerInRange(portRangeMin, portRangeMax)
	if s.listener == nil {
		return errors.New("Could not open a port ot listen on.")
	}

	// Start accepting connections
	go func(l net.Listener) {
		for {
			conn, err := l.Accept()
			if err != nil {
				break
			}

			go s.server.ServeConn(conn)
		}
	}(s.listener)

	return nil
}

func (s *Server) Stop() {
	if s.listener != nil {
		s.listener.Close()
		s.listener = nil
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package hbasekv

import (
	"strings"
	"sync"
	"time"

	"github.com/juju/errors"
	"github.com/ngaut/log"
	"github.com/pingcap/go-hbase"
	"github.com/pingcap/go-themis"
	"github.com/pingcap/tidb/kv"
)

const (
	// hbaseColFamily is the hbase column family name.
	hbaseColFamily = "f"
	// hbaseQualifier is the hbase column name.
	hbaseQualifier = "q"
	// hbaseFmlAndQual is a shortcut.
	hbaseFmlAndQual = hbaseColFamily + ":" + hbaseQualifier
	// fix length conn pool
	hbaseConnPoolSize = 10
)

var (
	hbaseColFamilyBytes = []byte(hbaseColFamily)
	hbaseQualifierBytes = []byte(hbaseQualifier)
)

var (
	_ kv.Storage = (*hbaseStore)(nil)
)

var (
	// ErrZkInvalid is returned when zookeeper info is invalid.
	ErrZkInvalid = errors.New("zk info invalid")
)

type storeCache struct {
	mu    sync.Mutex
	cache map[string]*hbaseStore
}

var mc storeCache

func init() {
	mc.cache = make(map[string]*hbaseStore)
}

type hbaseStore struct {
	mu        sync.Mutex
	zkInfo    string
	storeName string
	conns     []hbase.HBaseClient
}

func (s *hbaseStore) Begin() (kv.Transaction, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	// get random conn
	hbaseCli := s.conns[time.Now().UnixNano()%hbaseConnPoolSize]
	t := themis.NewTxn(hbaseCli)
	txn := newHbaseTxn(t, s.storeName)
	txn.UnionStore = kv.NewUnionStore(newHbaseSnapshot(t, s.storeName))
	return txn, nil
}

func (s *hbaseStore) GetSnapshot(ver kv.Version) (kv.MvccSnapshot, error) {
	// get random conn
	hbaseCli := s.conns[time.Now().UnixNano()%hbaseConnPoolSize]
	t := themis.NewTxn(hbaseCli)
	return newHbaseSnapshot(t, s.storeName), nil
}

func (s *hbaseStore) Close() error {
	mc.mu.Lock()
	defer mc.mu.Unlock()

	delete(mc.cache, s.zkInfo)

	var err error
	for _, conn := range s.conns {
		err = conn.Close()
		if err != nil {
			log.Error(err)
		}
	}
	// return last error
	return err
}

func (s *hbaseStore) UUID() string {
	return "hbase." + s.storeName + "." + s.zkInfo
}

func (s *hbaseStore) CurrentVersion() (kv.Version, error) {
	hbaseCli := s.conns[time.Now().UnixNano()%hbaseConnPoolSize]
	t := themis.NewTxn(hbaseCli)
	defer t.Release()

	return kv.Version{Ver: t.GetStartTS()}, nil
}

// Driver implements engine Driver.
type Driver struct {
}

// Open opens or creates a storage database with given path.
func (d Driver) Open(zkInfo string) (kv.Storage, error) {
	mc.mu.Lock()
	defer mc.mu.Unlock()

	if len(zkInfo) == 0 {
		return nil, errors.Trace(ErrZkInvalid)
	}
	pos := strings.LastIndex(zkInfo, "/")
	if pos == -1 {
		return nil, errors.Trace(ErrZkInvalid)
	}
	tableName := zkInfo[pos+1:]
	zks := strings.Split(zkInfo[:pos], ",")

	if store, ok := mc.cache[zkInfo]; ok {
		// TODO: check the cache store has the same engine with this Driver.
		return store, nil
	}

	// create buffered HBase connections, HBaseClient is goroutine-safe, so
	// it's OK to redistribute to transactions.
	conns := make([]hbase.HBaseClient, 0, hbaseConnPoolSize)
	for i := 0; i < hbaseConnPoolSize; i++ {
		c, err := hbase.NewClient(zks, "/hbase")
		if err != nil {
			return nil, errors.Trace(err)
		}
		conns = append(conns, c)
	}

	c := conns[0]
	if !c.TableExists(tableName) {
		// Create new hbase table for store.
		t := hbase.NewTableDesciptor(hbase.NewTableNameWithDefaultNS(tableName))
		cf := hbase.NewColumnFamilyDescriptor(hbaseColFamily)
		cf.AddStrAddr("THEMIS_ENABLE", "true")
		t.AddColumnDesc(cf)
		//TODO: specify split?
		if err := c.CreateTable(t, nil); err != nil {
			return nil, errors.Trace(err)
		}
	}

	s := &hbaseStore{
		zkInfo:    zkInfo,
		storeName: tableName,
		conns:     conns,
	}
	mc.cache[zkInfo] = s
	return s, nil
}
<commit_msg>hbase-store: use rand instead of timestamp for selecing connection randomly<commit_after>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package hbasekv

import (
	"math/rand"
	"strings"
	"sync"
	"time"

	"github.com/juju/errors"
	"github.com/ngaut/log"
	"github.com/pingcap/go-hbase"
	"github.com/pingcap/go-themis"
	"github.com/pingcap/tidb/kv"
)

const (
	// hbaseColFamily is the hbase column family name.
	hbaseColFamily = "f"
	// hbaseQualifier is the hbase column name.
	hbaseQualifier = "q"
	// hbaseFmlAndQual is a shortcut.
	hbaseFmlAndQual = hbaseColFamily + ":" + hbaseQualifier
	// fix length conn pool
	hbaseConnPoolSize = 10
)

var (
	hbaseColFamilyBytes = []byte(hbaseColFamily)
	hbaseQualifierBytes = []byte(hbaseQualifier)
)

var (
	_ kv.Storage = (*hbaseStore)(nil)
)

var (
	// ErrZkInvalid is returned when zookeeper info is invalid.
	ErrZkInvalid = errors.New("zk info invalid")
)

type storeCache struct {
	mu    sync.Mutex
	cache map[string]*hbaseStore
}

var mc storeCache

func init() {
	mc.cache = make(map[string]*hbaseStore)
	rand.Seed(time.Now().UnixNano())
}

type hbaseStore struct {
	mu        sync.Mutex
	zkInfo    string
	storeName string
	conns     []hbase.HBaseClient
}

func (s *hbaseStore) Begin() (kv.Transaction, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	// get random conn
	hbaseCli := s.conns[rand.Intn(hbaseConnPoolSize)]
	t := themis.NewTxn(hbaseCli)
	txn := newHbaseTxn(t, s.storeName)
	txn.UnionStore = kv.NewUnionStore(newHbaseSnapshot(t, s.storeName))
	return txn, nil
}

func (s *hbaseStore) GetSnapshot(ver kv.Version) (kv.MvccSnapshot, error) {
	// get random conn
	hbaseCli := s.conns[rand.Intn(hbaseConnPoolSize)]
	t := themis.NewTxn(hbaseCli)
	return newHbaseSnapshot(t, s.storeName), nil
}

func (s *hbaseStore) Close() error {
	mc.mu.Lock()
	defer mc.mu.Unlock()

	delete(mc.cache, s.zkInfo)

	var err error
	for _, conn := range s.conns {
		err = conn.Close()
		if err != nil {
			log.Error(err)
		}
	}
	// return last error
	return err
}

func (s *hbaseStore) UUID() string {
	return "hbase." + s.storeName + "." + s.zkInfo
}

func (s *hbaseStore) CurrentVersion() (kv.Version, error) {
	hbaseCli := s.conns[rand.Intn(hbaseConnPoolSize)]
	t := themis.NewTxn(hbaseCli)
	defer t.Release()

	return kv.Version{Ver: t.GetStartTS()}, nil
}

// Driver implements engine Driver.
type Driver struct {
}

// Open opens or creates a storage database with given path.
func (d Driver) Open(zkInfo string) (kv.Storage, error) {
	mc.mu.Lock()
	defer mc.mu.Unlock()

	if len(zkInfo) == 0 {
		return nil, errors.Trace(ErrZkInvalid)
	}
	pos := strings.LastIndex(zkInfo, "/")
	if pos == -1 {
		return nil, errors.Trace(ErrZkInvalid)
	}
	tableName := zkInfo[pos+1:]
	zks := strings.Split(zkInfo[:pos], ",")

	if store, ok := mc.cache[zkInfo]; ok {
		// TODO: check the cache store has the same engine with this Driver.
		return store, nil
	}

	// create buffered HBase connections, HBaseClient is goroutine-safe, so
	// it's OK to redistribute to transactions.
	conns := make([]hbase.HBaseClient, 0, hbaseConnPoolSize)
	for i := 0; i < hbaseConnPoolSize; i++ {
		c, err := hbase.NewClient(zks, "/hbase")
		if err != nil {
			return nil, errors.Trace(err)
		}
		conns = append(conns, c)
	}

	c := conns[0]
	if !c.TableExists(tableName) {
		// Create new hbase table for store.
		t := hbase.NewTableDesciptor(hbase.NewTableNameWithDefaultNS(tableName))
		cf := hbase.NewColumnFamilyDescriptor(hbaseColFamily)
		cf.AddStrAddr("THEMIS_ENABLE", "true")
		t.AddColumnDesc(cf)
		//TODO: specify split?
		if err := c.CreateTable(t, nil); err != nil {
			return nil, errors.Trace(err)
		}
	}

	s := &hbaseStore{
		zkInfo:    zkInfo,
		storeName: tableName,
		conns:     conns,
	}
	mc.cache[zkInfo] = s
	return s, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package memfs

import (
	"fmt"
	"os"
	"time"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/timeutil"
	"golang.org/x/net/context"
)

type memFS struct {
	fuseutil.NotImplementedFileSystem

	/////////////////////////
	// Dependencies
	/////////////////////////

	clock timeutil.Clock

	/////////////////////////
	// Mutable state
	/////////////////////////

	// When acquiring this lock, the caller must hold no inode locks.
	mu syncutil.InvariantMutex

	// The collection of live inodes, indexed by ID. IDs of free inodes that may
	// be re-used have nil entries. No ID less than fuse.RootInodeID is ever used.
	//
	// INVARIANT: len(inodes) > fuse.RootInodeID
	// INVARIANT: For all i < fuse.RootInodeID, inodes[i] == nil
	// INVARIANT: inodes[fuse.RootInodeID] != nil
	// INVARIANT: inodes[fuse.RootInodeID].dir is true
	inodes []*inode // GUARDED_BY(mu)

	// A list of inode IDs within inodes available for reuse, not including the
	// reserved IDs less than fuse.RootInodeID.
	//
	// INVARIANT: This is all and only indices i of 'inodes' such that i >
	// fuse.RootInodeID and inodes[i] == nil
	freeInodes []fuse.InodeID // GUARDED_BY(mu)
}

// Create a file system that stores data and metadata in memory.
func NewMemFS(
	clock timeutil.Clock) fuse.FileSystem {
	// Set up the basic struct.
	fs := &memFS{
		clock:  clock,
		inodes: make([]*inode, fuse.RootInodeID+1),
	}

	// Set up the root inode.
	fs.inodes[fuse.RootInodeID] = newInode(0777 | os.ModeDir)

	// Set up invariant checking.
	fs.mu = syncutil.NewInvariantMutex(fs.checkInvariants)

	return fs
}

func (fs *memFS) checkInvariants() {
	// Check reserved inodes.
	for i := 0; i < fuse.RootInodeID; i++ {
		if fs.inodes[i] != nil {
			panic(fmt.Sprintf("Non-nil inode for ID: %v", i))
		}
	}

	// Check the root inode.
	if !fs.inodes[fuse.RootInodeID].dir {
		panic("Expected root to be a directory.")
	}

	// Build our own list of free IDs.
	freeIDsEncountered := make(map[fuse.InodeID]struct{})
	for i := fuse.RootInodeID + 1; i < len(fs.inodes); i++ {
		inode := fs.inodes[i]
		if inode == nil {
			freeIDsEncountered[fuse.InodeID(i)] = struct{}{}
			continue
		}
	}

	// Check fs.freeInodes.
	if len(fs.freeInodes) != len(freeIDsEncountered) {
		panic(
			fmt.Sprintf(
				"Length mismatch: %v vs. %v",
				len(fs.freeInodes),
				len(freeIDsEncountered)))
	}

	for _, id := range fs.freeInodes {
		if _, ok := freeIDsEncountered[id]; !ok {
			panic(fmt.Sprintf("Unexected free inode ID: %v", id))
		}
	}
}

func (fs *memFS) Init(
	ctx context.Context,
	req *fuse.InitRequest) (resp *fuse.InitResponse, err error) {
	resp = &fuse.InitResponse{}
	return
}

// Find the given inode and return it with its lock held. Panic if it doesn't
// exist.
//
// SHARED_LOCKS_REQUIRED(fs.mu)
// EXCLUSIVE_LOCK_FUNCTION(inode.mu)
func (fs *memFS) getInodeForModifyingOrDie(id fuse.InodeID) (inode *inode) {
	inode = fs.inodes[id]
	if inode == nil {
		panic(fmt.Sprintf("Unknown inode: %v", id))
	}

	inode.mu.Lock()
	return
}

// Find the given inode and return it with its lock held for reading. Panic if
// it doesn't exist.
//
// SHARED_LOCKS_REQUIRED(fs.mu)
// SHARED_LOCK_FUNCTION(inode.mu)
func (fs *memFS) getInodeForReadingOrDie(id fuse.InodeID) (inode *inode) {
	inode = fs.inodes[id]
	if inode == nil {
		panic(fmt.Sprintf("Unknown inode: %v", id))
	}

	inode.mu.RLock()
	return
}

// Allocate a new inode, assigning it an ID that is not in use. Return it with
// its lock held.
//
// EXCLUSIVE_LOCKS_REQUIRED(fs.mu)
// EXCLUSIVE_LOCK_FUNCTION(inode.mu)
func (fs *memFS) allocateInode(
	mode os.FileMode) (id fuse.InodeID, inode *inode) {
	// Create and lock the inode.
	inode = newInode(mode)
	inode.mu.Lock()

	// Re-use a free ID if possible. Otherwise mint a new one.
	numFree := len(fs.freeInodes)
	if numFree != 0 {
		id = fs.freeInodes[numFree-1]
		fs.freeInodes = fs.freeInodes[:numFree-1]
		fs.inodes[id] = inode
	} else {
		id = fuse.InodeID(len(fs.inodes))
		fs.inodes = append(fs.inodes, inode)
	}

	return
}

func (fs *memFS) LookUpInode(
	ctx context.Context,
	req *fuse.LookUpInodeRequest) (resp *fuse.LookUpInodeResponse, err error) {
	resp = &fuse.LookUpInodeResponse{}

	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// Grab the parent directory.
	inode := fs.getInodeForReadingOrDie(req.Parent)
	defer inode.mu.RUnlock()

	// Does the directory have an entry with the given name?
	childID, ok := inode.LookUpChild(req.Name)
	if !ok {
		err = fuse.ENOENT
		return
	}

	// Grab the child.
	child := fs.getInodeForReadingOrDie(childID)
	defer child.mu.RUnlock()

	// Fill in the response.
	resp.Entry.Attributes = child.attributes

	// We don't spontaneously mutate, so the kernel can cache as long as it wants
	// (since it also handles invalidation).
	resp.Entry.AttributesExpiration = fs.clock.Now().Add(365 * 24 * time.Hour)
	resp.Entry.EntryExpiration = resp.Entry.EntryExpiration

	return
}

func (fs *memFS) GetInodeAttributes(
	ctx context.Context,
	req *fuse.GetInodeAttributesRequest) (
	resp *fuse.GetInodeAttributesResponse, err error) {
	resp = &fuse.GetInodeAttributesResponse{}

	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// Grab the inode.
	inode := fs.getInodeForReadingOrDie(req.Inode)
	defer inode.mu.RUnlock()

	// Fill in the response.
	resp.Attributes = inode.attributes

	// We don't spontaneously mutate, so the kernel can cache as long as it wants
	// (since it also handles invalidation).
	resp.AttributesExpiration = fs.clock.Now().Add(365 * 24 * time.Hour)

	return
}

func (fs *memFS) MkDir(
	ctx context.Context,
	req *fuse.MkDirRequest) (resp *fuse.MkDirResponse, err error) {
	resp = &fuse.MkDirResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Grab the parent, which we will update shortly.
	parent := fs.getInodeForModifyingOrDie(req.Parent)
	defer parent.mu.Unlock()

	// Allocate a child.
	childID, child := fs.allocateInode(req.Mode)
	defer child.mu.Unlock()

	// Add an entry in the parent.
	parent.AddChild(childID, req.Name, fuseutil.DT_Directory)

	// Fill in the response.
	resp.Entry.Attributes = child.attributes

	// We don't spontaneously mutate, so the kernel can cache as long as it wants
	// (since it also handles invalidation).
	resp.Entry.AttributesExpiration = fs.clock.Now().Add(365 * 24 * time.Hour)
	resp.Entry.EntryExpiration = resp.Entry.EntryExpiration

	return
}

func (fs *memFS) OpenDir(
	ctx context.Context,
	req *fuse.OpenDirRequest) (resp *fuse.OpenDirResponse, err error) {
	resp = &fuse.OpenDirResponse{}

	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// We don't mutate spontaneosuly, so if the VFS layer has asked for an
	// inode that doesn't exist, something screwed up earlier (a lookup, a
	// cache invalidation, etc.).
	inode := fs.getInodeForReadingOrDie(req.Inode)
	defer inode.mu.RUnlock()

	if !inode.dir {
		panic("Found non-dir.")
	}

	return
}

func (fs *memFS) ReadDir(
	ctx context.Context,
	req *fuse.ReadDirRequest) (resp *fuse.ReadDirResponse, err error) {
	resp = &fuse.ReadDirResponse{}

	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// Grab the directory.
	inode := fs.getInodeForReadingOrDie(req.Inode)
	defer inode.mu.RUnlock()

	// Serve the request.
	resp.Data, err = inode.ReadDir(int(req.Offset), req.Size)
	if err != nil {
		err = fmt.Errorf("inode.ReadDir: %v", err)
		return
	}

	return
}
<commit_msg>Fixed two bugs where ChildInodeEntry.Child was not filled in.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package memfs

import (
	"fmt"
	"os"
	"time"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/timeutil"
	"golang.org/x/net/context"
)

type memFS struct {
	fuseutil.NotImplementedFileSystem

	/////////////////////////
	// Dependencies
	/////////////////////////

	clock timeutil.Clock

	/////////////////////////
	// Mutable state
	/////////////////////////

	// When acquiring this lock, the caller must hold no inode locks.
	mu syncutil.InvariantMutex

	// The collection of live inodes, indexed by ID. IDs of free inodes that may
	// be re-used have nil entries. No ID less than fuse.RootInodeID is ever used.
	//
	// INVARIANT: len(inodes) > fuse.RootInodeID
	// INVARIANT: For all i < fuse.RootInodeID, inodes[i] == nil
	// INVARIANT: inodes[fuse.RootInodeID] != nil
	// INVARIANT: inodes[fuse.RootInodeID].dir is true
	inodes []*inode // GUARDED_BY(mu)

	// A list of inode IDs within inodes available for reuse, not including the
	// reserved IDs less than fuse.RootInodeID.
	//
	// INVARIANT: This is all and only indices i of 'inodes' such that i >
	// fuse.RootInodeID and inodes[i] == nil
	freeInodes []fuse.InodeID // GUARDED_BY(mu)
}

// Create a file system that stores data and metadata in memory.
func NewMemFS(
	clock timeutil.Clock) fuse.FileSystem {
	// Set up the basic struct.
	fs := &memFS{
		clock:  clock,
		inodes: make([]*inode, fuse.RootInodeID+1),
	}

	// Set up the root inode.
	fs.inodes[fuse.RootInodeID] = newInode(0777 | os.ModeDir)

	// Set up invariant checking.
	fs.mu = syncutil.NewInvariantMutex(fs.checkInvariants)

	return fs
}

func (fs *memFS) checkInvariants() {
	// Check reserved inodes.
	for i := 0; i < fuse.RootInodeID; i++ {
		if fs.inodes[i] != nil {
			panic(fmt.Sprintf("Non-nil inode for ID: %v", i))
		}
	}

	// Check the root inode.
	if !fs.inodes[fuse.RootInodeID].dir {
		panic("Expected root to be a directory.")
	}

	// Build our own list of free IDs.
	freeIDsEncountered := make(map[fuse.InodeID]struct{})
	for i := fuse.RootInodeID + 1; i < len(fs.inodes); i++ {
		inode := fs.inodes[i]
		if inode == nil {
			freeIDsEncountered[fuse.InodeID(i)] = struct{}{}
			continue
		}
	}

	// Check fs.freeInodes.
	if len(fs.freeInodes) != len(freeIDsEncountered) {
		panic(
			fmt.Sprintf(
				"Length mismatch: %v vs. %v",
				len(fs.freeInodes),
				len(freeIDsEncountered)))
	}

	for _, id := range fs.freeInodes {
		if _, ok := freeIDsEncountered[id]; !ok {
			panic(fmt.Sprintf("Unexected free inode ID: %v", id))
		}
	}
}

func (fs *memFS) Init(
	ctx context.Context,
	req *fuse.InitRequest) (resp *fuse.InitResponse, err error) {
	resp = &fuse.InitResponse{}
	return
}

// Find the given inode and return it with its lock held. Panic if it doesn't
// exist.
//
// SHARED_LOCKS_REQUIRED(fs.mu)
// EXCLUSIVE_LOCK_FUNCTION(inode.mu)
func (fs *memFS) getInodeForModifyingOrDie(id fuse.InodeID) (inode *inode) {
	inode = fs.inodes[id]
	if inode == nil {
		panic(fmt.Sprintf("Unknown inode: %v", id))
	}

	inode.mu.Lock()
	return
}

// Find the given inode and return it with its lock held for reading. Panic if
// it doesn't exist.
//
// SHARED_LOCKS_REQUIRED(fs.mu)
// SHARED_LOCK_FUNCTION(inode.mu)
func (fs *memFS) getInodeForReadingOrDie(id fuse.InodeID) (inode *inode) {
	inode = fs.inodes[id]
	if inode == nil {
		panic(fmt.Sprintf("Unknown inode: %v", id))
	}

	inode.mu.RLock()
	return
}

// Allocate a new inode, assigning it an ID that is not in use. Return it with
// its lock held.
//
// EXCLUSIVE_LOCKS_REQUIRED(fs.mu)
// EXCLUSIVE_LOCK_FUNCTION(inode.mu)
func (fs *memFS) allocateInode(
	mode os.FileMode) (id fuse.InodeID, inode *inode) {
	// Create and lock the inode.
	inode = newInode(mode)
	inode.mu.Lock()

	// Re-use a free ID if possible. Otherwise mint a new one.
	numFree := len(fs.freeInodes)
	if numFree != 0 {
		id = fs.freeInodes[numFree-1]
		fs.freeInodes = fs.freeInodes[:numFree-1]
		fs.inodes[id] = inode
	} else {
		id = fuse.InodeID(len(fs.inodes))
		fs.inodes = append(fs.inodes, inode)
	}

	return
}

func (fs *memFS) LookUpInode(
	ctx context.Context,
	req *fuse.LookUpInodeRequest) (resp *fuse.LookUpInodeResponse, err error) {
	resp = &fuse.LookUpInodeResponse{}

	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// Grab the parent directory.
	inode := fs.getInodeForReadingOrDie(req.Parent)
	defer inode.mu.RUnlock()

	// Does the directory have an entry with the given name?
	childID, ok := inode.LookUpChild(req.Name)
	if !ok {
		err = fuse.ENOENT
		return
	}

	// Grab the child.
	child := fs.getInodeForReadingOrDie(childID)
	defer child.mu.RUnlock()

	// Fill in the response.
	resp.Entry.Child = childID
	resp.Entry.Attributes = child.attributes

	// We don't spontaneously mutate, so the kernel can cache as long as it wants
	// (since it also handles invalidation).
	resp.Entry.AttributesExpiration = fs.clock.Now().Add(365 * 24 * time.Hour)
	resp.Entry.EntryExpiration = resp.Entry.EntryExpiration

	return
}

func (fs *memFS) GetInodeAttributes(
	ctx context.Context,
	req *fuse.GetInodeAttributesRequest) (
	resp *fuse.GetInodeAttributesResponse, err error) {
	resp = &fuse.GetInodeAttributesResponse{}

	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// Grab the inode.
	inode := fs.getInodeForReadingOrDie(req.Inode)
	defer inode.mu.RUnlock()

	// Fill in the response.
	resp.Attributes = inode.attributes

	// We don't spontaneously mutate, so the kernel can cache as long as it wants
	// (since it also handles invalidation).
	resp.AttributesExpiration = fs.clock.Now().Add(365 * 24 * time.Hour)

	return
}

func (fs *memFS) MkDir(
	ctx context.Context,
	req *fuse.MkDirRequest) (resp *fuse.MkDirResponse, err error) {
	resp = &fuse.MkDirResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Grab the parent, which we will update shortly.
	parent := fs.getInodeForModifyingOrDie(req.Parent)
	defer parent.mu.Unlock()

	// Allocate a child.
	childID, child := fs.allocateInode(req.Mode)
	defer child.mu.Unlock()

	// Add an entry in the parent.
	parent.AddChild(childID, req.Name, fuseutil.DT_Directory)

	// Fill in the response.
	resp.Entry.Child = childID
	resp.Entry.Attributes = child.attributes

	// We don't spontaneously mutate, so the kernel can cache as long as it wants
	// (since it also handles invalidation).
	resp.Entry.AttributesExpiration = fs.clock.Now().Add(365 * 24 * time.Hour)
	resp.Entry.EntryExpiration = resp.Entry.EntryExpiration

	return
}

func (fs *memFS) OpenDir(
	ctx context.Context,
	req *fuse.OpenDirRequest) (resp *fuse.OpenDirResponse, err error) {
	resp = &fuse.OpenDirResponse{}

	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// We don't mutate spontaneosuly, so if the VFS layer has asked for an
	// inode that doesn't exist, something screwed up earlier (a lookup, a
	// cache invalidation, etc.).
	inode := fs.getInodeForReadingOrDie(req.Inode)
	defer inode.mu.RUnlock()

	if !inode.dir {
		panic("Found non-dir.")
	}

	return
}

func (fs *memFS) ReadDir(
	ctx context.Context,
	req *fuse.ReadDirRequest) (resp *fuse.ReadDirResponse, err error) {
	resp = &fuse.ReadDirResponse{}

	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// Grab the directory.
	inode := fs.getInodeForReadingOrDie(req.Inode)
	defer inode.mu.RUnlock()

	// Serve the request.
	resp.Data, err = inode.ReadDir(int(req.Offset), req.Size)
	if err != nil {
		err = fmt.Errorf("inode.ReadDir: %v", err)
		return
	}

	return
}
<|endoftext|>


<commit_before>package storage

import (
	"context"
	"testing"
	"time"

	"github.com/ohsu-comp-bio/funnel/config"
)

func TestHTTP(t *testing.T) {
	store, err := NewHTTP(config.HTTPStorage{
		Timeout: config.Duration(2 * time.Second),
	})
	if err != nil {
		t.Fatal("Error creating HTTP backend:", err)
	}

	// test client timeout
	err = store.UnsupportedOperations("https://fakeurl.com").Get
	if err == nil {
		t.Error("Expected timeout error")
	}

	// test Get is supported
	err = store.UnsupportedOperations("https://google.com").Get
	if err != nil {
		t.Error("Unexpected error for unsupported.Get call:", err)
	}

	// test Get on file that requires auth
	err = store.UnsupportedOperations("https://s3.amazonaws.com/private").Get
	if err == nil {
		t.Error("Expected error for unsupported.Get call")
	}

	// test Get succeeds
	_, err = store.Get(context.Background(), "https://google.com", "_test_download/test_https_download.html")
	if err != nil {
		t.Error("Unexpected error downloading file:", err)
	}

	// test List not supported
	_, err = store.List(context.Background(), "https://google.com")
	if err == nil {
		t.Error("Expected error for List irectory")
	}

	// test Put not supported
	_, err = store.Put(context.Background(), "https://fakeurl.com", "_test_download/test_https_download.html")
	if err == nil {
		t.Error("Expected error for Put call")
	}
}
<commit_msg>storage/http: fix test<commit_after>package storage

import (
	"context"
	"testing"
	"time"

	"github.com/ohsu-comp-bio/funnel/config"
)

func TestHTTP(t *testing.T) {
	store, err := NewHTTP(config.HTTPStorage{
		Timeout: config.Duration(2 * time.Second),
	})
	if err != nil {
		t.Fatal("Error creating HTTP backend:", err)
	}

	// test client timeout
	err = store.UnsupportedOperations("https://fakeurl1234.com").Get
	if err == nil {
		t.Error("Expected timeout error")
	}

	// test Get is supported
	err = store.UnsupportedOperations("https://google.com").Get
	if err != nil {
		t.Error("Unexpected error for unsupported.Get call:", err)
	}

	// test Get on file that requires auth
	err = store.UnsupportedOperations("https://s3.amazonaws.com/private").Get
	if err == nil {
		t.Error("Expected error for unsupported.Get call")
	}

	// test Get succeeds
	_, err = store.Get(context.Background(), "https://google.com", "_test_download/test_https_download.html")
	if err != nil {
		t.Error("Unexpected error downloading file:", err)
	}

	// test List not supported
	_, err = store.List(context.Background(), "https://google.com")
	if err == nil {
		t.Error("Expected error for List irectory")
	}

	// test Put not supported
	_, err = store.Put(context.Background(), "https://fakeurl1234.com", "_test_download/test_https_download.html")
	if err == nil {
		t.Error("Expected error for Put call")
	}
}
<|endoftext|>


<commit_before>// This file is generated from mgl32/shapes_test.go; DO NOT EDIT

// Copyright 2018 The go-gl Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.package mgl64_test

package mgl64

import (
	"testing"
)

func TestScreenToGLCoords(t *testing.T) {
	// use a small screen size in order to minimize errors due to fp rounding.
	const (
		sw = 100
		sh = 100
	)
	x, y := ScreenToGLCoords(0, sh-1, sw, sh)
	if x != -1 {
		t.Errorf("x = %f, expected -1.0", x)
	}
	if y != -1 {
		t.Errorf("y = %f, expected -1.0", y)
	}

	x, y = ScreenToGLCoords(sw-1, 0, sw, sh)
	if x != 1 {
		t.Errorf("x = %f, expected 1.0", x)
	}
	if y != 1 {
		t.Errorf("y = %f, expected 1.0", y)
	}
}

func TestGLToScreenCoords(t *testing.T) {
	const (
		sw = 100
		sh = 100
	)
	x, y := GLToScreenCoords(-1, -1, sw, sh)
	if x != 0 {
		t.Errorf("x = %d, expected 0", x)
	}
	if y != sh-1 {
		t.Errorf("y = %d, expected %d", y, sh-1)
	}

	x, y = GLToScreenCoords(1, 1, sw, sh)
	if x != sw-1 {
		t.Errorf("x = %d, expected %d", x, sw-1)
	}
	if y != 0 {
		t.Errorf("y = %d, expected 0", y)
	}
}

func Test_choose(t *testing.T) {

	tests := []struct {
		name string
		n    int
		want []int
	}{
		// test cases.
		{
			name: "C(2,k)",
			n:    2,
			want: []int{1, 2, 1},
		},
		{
			name: "C(3,k)",
			n:    3,
			want: []int{1, 3, 3, 1},
		},
		{
			name: "C(4,k)",
			n:    4,
			want: []int{1, 4, 6, 4, 1},
		},
		{
			name: "C(5,k)",
			n:    5,
			want: []int{1, 5, 10, 10, 5, 1},
		},
		{
			name: "C(6,k)",
			n:    6,
			want: []int{1, 6, 15, 20, 25, 6, 1},
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := []int{}
			fail := false
			for k := 0; k < tt.n; k++ {
				got = append(got, choose(tt.n, k))
				fail = got[k] != tt.want[k]
			}
			if fail {
				t.Errorf("choose() = %v, want %v", got, tt.want)
			}
		})
	}
}
<commit_msg>removed t.Run() from Test_choose() so it'll pass the Go 1.6 test on travis-ci...<commit_after>// This file is generated from mgl32/shapes_test.go; DO NOT EDIT

// Copyright 2018 The go-gl Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.package mgl64_test

package mgl64

import (
	"testing"
)

func TestScreenToGLCoords(t *testing.T) {
	// use a small screen size in order to minimize errors due to fp rounding.
	const (
		sw = 100
		sh = 100
	)
	x, y := ScreenToGLCoords(0, sh-1, sw, sh)
	if x != -1 {
		t.Errorf("x = %f, expected -1.0", x)
	}
	if y != -1 {
		t.Errorf("y = %f, expected -1.0", y)
	}

	x, y = ScreenToGLCoords(sw-1, 0, sw, sh)
	if x != 1 {
		t.Errorf("x = %f, expected 1.0", x)
	}
	if y != 1 {
		t.Errorf("y = %f, expected 1.0", y)
	}
}

func TestGLToScreenCoords(t *testing.T) {
	const (
		sw = 100
		sh = 100
	)
	x, y := GLToScreenCoords(-1, -1, sw, sh)
	if x != 0 {
		t.Errorf("x = %d, expected 0", x)
	}
	if y != sh-1 {
		t.Errorf("y = %d, expected %d", y, sh-1)
	}

	x, y = GLToScreenCoords(1, 1, sw, sh)
	if x != sw-1 {
		t.Errorf("x = %d, expected %d", x, sw-1)
	}
	if y != 0 {
		t.Errorf("y = %d, expected 0", y)
	}
}

func Test_choose(t *testing.T) {

	tests := []struct {
		name string
		n    int
		want []int
	}{
		// test cases.
		{
			name: "C(2,k)",
			n:    2,
			want: []int{1, 2, 1},
		},
		{
			name: "C(3,k)",
			n:    3,
			want: []int{1, 3, 3, 1},
		},
		{
			name: "C(4,k)",
			n:    4,
			want: []int{1, 4, 6, 4, 1},
		},
		{
			name: "C(5,k)",
			n:    5,
			want: []int{1, 5, 10, 10, 5, 1},
		},
		{
			name: "C(6,k)",
			n:    6,
			want: []int{1, 6, 15, 20, 25, 6, 1},
		},
	}
	for _, tt := range tests {
		// t.Run(tt.name, func(t *testing.T) {
		got := []int{}
		fail := false
		for k := 0; k < tt.n; k++ {
			got = append(got, choose(tt.n, k))
			fail = got[k] != tt.want[k]
		}
		if fail {
			t.Errorf("choose() = %v, want %v", got, tt.want)
		}
		// })
	}
}
<|endoftext|>


<commit_before>package tests

import (
	"context"
	"fmt"
	"math"
	"math/rand"
	"testing"

	"github.com/pborman/uuid"

	"gopkg.in/BTrDB/btrdb.v4"
)

func helperConnect(t *testing.T, ctx context.Context) *btrdb.BTrDB {
	db, err := btrdb.Connect(ctx, btrdb.EndpointsFromEnv()...)
	if err != nil {
		t.Fatalf("Unexpected connection error: %v", err)
	}
	return db
}

func helperGetCollection(uu uuid.UUID) string {
	return fmt.Sprintf("test.%x", uu[:])
}

func helperCreateStream(t *testing.T, ctx context.Context, db *btrdb.BTrDB, uu uuid.UUID, coll string, tags map[string]string, ann map[string]string) *btrdb.Stream {
	stream, err := db.Create(ctx, uu, coll, tags, ann)
	if err != nil {
		t.Fatalf("create error %v", err)
	}
	return stream
}

func helperCreateDefaultStream(t *testing.T, ctx context.Context, db *btrdb.BTrDB, tags map[string]string, ann map[string]string) *btrdb.Stream {
	uu := uuid.NewRandom()
	coll := helperGetCollection(uu)
	s := helperCreateStream(t, ctx, db, uu, coll, tags, ann)
	suu := s.UUID()
	if len(suu) != len(uu) {
		t.Fatal("Bad UUID")
	}
	for i, b := range suu {
		if b != uu[i] {
			t.Fatal("UUID of created stream does not match provided UUID")
		}
	}
	return s
}

func helperWaitAfterInsert(t *testing.T, ctx context.Context, s *btrdb.Stream) {
	err := s.Flush(ctx)
	if err != nil {
		t.Fatalf("Error from Flush %v", err)
	}
}

func helperInsert(t *testing.T, ctx context.Context, s *btrdb.Stream, data []btrdb.RawPoint) {
	err := s.Insert(ctx, data)
	if err != nil {
		t.Fatalf("Error from Insert %v", err)
	}
	helperWaitAfterInsert(t, ctx, s)
}

func helperRandomData(start int64, end int64, gap int64) []btrdb.RawPoint {
	numpts := (end - start) / gap
	pts := make([]btrdb.RawPoint, numpts)
	for i, _ := range pts {
		pts[i].Time = start + (int64(i) * gap)
		pts[i].Value = rand.NormFloat64()
	}
	return pts
}

func helperRandomDataCount(start int64, end int64, numpts int64) []btrdb.RawPoint {
	gap := (end - start) / numpts
	return helperRandomData(start, end, gap)
}

func helperFloatEquals(x float64, y float64) bool {
	return math.Abs(x-y) < 1e-10*math.Max(math.Abs(x), math.Abs(y))
}

func helperMakeStatPoints(points []btrdb.RawPoint, start int64, end int64, queryWidth int64, aligned bool) []btrdb.StatPoint {
	if !aligned {
		end = start + queryWidth*((end-start)/queryWidth)
	}
	numPoints := int((end - start) / queryWidth)
	var statPoints []btrdb.StatPoint
	index := 0
	for index < len(points) {
		offset := index
		windowStart := start + int64(len(statPoints))*queryWidth

		/* For aligned queries, queryWidth will be 1<<pw,
		 * and we want to clear the pw bits from the window start
		 * ( e.g. start & ~((1<<pw) - 1) )
		 */
		if aligned {
			windowStart = windowStart &^ (queryWidth - 1)
		}

		for offset < len(points) && points[offset].Time < windowStart+queryWidth {
			offset++
		}

		pointsSlice := points[index:offset]

		// Unaligned queries return empty stat points
		if !aligned && len(pointsSlice) == 0 {
			empty := btrdb.StatPoint{windowStart, 0, 0, 0, 0}
			statPoints = append(statPoints, empty)
			continue
		}
		min := math.Inf(1)
		max := math.Inf(-1)
		sum := 0.0
		for _, point := range pointsSlice {
			min = math.Min(min, point.Value)
			sum += point.Value
			max = math.Max(max, point.Value)
		}
		mean := sum / float64(len(pointsSlice))
		statPoint := btrdb.StatPoint{windowStart, min, mean, max, uint64(len(pointsSlice))}
		statPoints = append(statPoints, statPoint)
		index = offset
	}

	/* Unaligned queries always return the correct number of stat
	 * points, even if there is no data
	 */
	for !aligned && len(statPoints) < numPoints {
		windowStart := start + int64(len(statPoints))*queryWidth
		empty := btrdb.StatPoint{windowStart, 0, 0, 0, 0}
		statPoints = append(statPoints, empty)
	}
	return statPoints
}

func helperCheckStatisticalEqual(firstPoints []btrdb.StatPoint, secondPoints []btrdb.StatPoint) error {
	if len(firstPoints) != len(secondPoints) {
		return fmt.Errorf("Stat point lenghts did not match %v %v", len(firstPoints), len(secondPoints))
	}

	for i := range firstPoints {
		first := firstPoints[i]
		second := secondPoints[i]
		if first.Time != second.Time {
			return fmt.Errorf("First time %v did not equal second time %v", first.Time, second.Time)
		}
		if !helperFloatEquals(first.Min, second.Min) {
			return fmt.Errorf("First min %v did not equal second min %v", first.Min, second.Min)
		}
		if !helperFloatEquals(first.Max, second.Max) {
			return fmt.Errorf("First max %v did not equal second max %v", first.Min, second.Max)
		}
		if first.Count != second.Count {
			return fmt.Errorf("First point count %v did not equal second point %v", first.Count, second.Count)
		}
		if !helperFloatEquals(first.Mean, second.Mean) {
			return fmt.Errorf("First mean %v did not equal second mean %v", first.Mean, second.Mean)
		}
	}

	return nil
}

func helperWindowQuery(t *testing.T, ctx context.Context, s *btrdb.Stream, start int64, end int64, width uint64, depth uint8, version uint64) ([]btrdb.StatPoint, uint64) {
	spc, verc, errc := s.Windows(ctx, start, end, width, depth, version)
	rv := make([]btrdb.StatPoint, 0)
	for sp := range spc {
		rv = append(rv, sp)
	}
	ver := <-verc
	err := <-errc
	if err != nil {
		t.Fatalf("window query error: %v", err)
	}

	return rv, ver
}

func helperStatisticalQuery(t *testing.T, ctx context.Context, s *btrdb.Stream, start int64, end int64, pwe uint8, version uint64) ([]btrdb.StatPoint, uint64) {
	spc, verc, errc := s.AlignedWindows(ctx, start, end, pwe, version)
	rv := make([]btrdb.StatPoint, 0)
	for sp := range spc {
		rv = append(rv, sp)
	}
	ver := <-verc
	err := <-errc
	if err != nil {
		t.Fatalf("statistical query error: %v", err)
	}

	return rv, ver
}
<commit_msg>make comments more clear<commit_after>package tests

import (
	"context"
	"fmt"
	"math"
	"math/rand"
	"testing"

	"github.com/pborman/uuid"

	"gopkg.in/BTrDB/btrdb.v4"
)

func helperConnect(t *testing.T, ctx context.Context) *btrdb.BTrDB {
	db, err := btrdb.Connect(ctx, btrdb.EndpointsFromEnv()...)
	if err != nil {
		t.Fatalf("Unexpected connection error: %v", err)
	}
	return db
}

func helperGetCollection(uu uuid.UUID) string {
	return fmt.Sprintf("test.%x", uu[:])
}

func helperCreateStream(t *testing.T, ctx context.Context, db *btrdb.BTrDB, uu uuid.UUID, coll string, tags map[string]string, ann map[string]string) *btrdb.Stream {
	stream, err := db.Create(ctx, uu, coll, tags, ann)
	if err != nil {
		t.Fatalf("create error %v", err)
	}
	return stream
}

func helperCreateDefaultStream(t *testing.T, ctx context.Context, db *btrdb.BTrDB, tags map[string]string, ann map[string]string) *btrdb.Stream {
	uu := uuid.NewRandom()
	coll := helperGetCollection(uu)
	s := helperCreateStream(t, ctx, db, uu, coll, tags, ann)
	suu := s.UUID()
	if len(suu) != len(uu) {
		t.Fatal("Bad UUID")
	}
	for i, b := range suu {
		if b != uu[i] {
			t.Fatal("UUID of created stream does not match provided UUID")
		}
	}
	return s
}

func helperWaitAfterInsert(t *testing.T, ctx context.Context, s *btrdb.Stream) {
	err := s.Flush(ctx)
	if err != nil {
		t.Fatalf("Error from Flush %v", err)
	}
}

func helperInsert(t *testing.T, ctx context.Context, s *btrdb.Stream, data []btrdb.RawPoint) {
	err := s.Insert(ctx, data)
	if err != nil {
		t.Fatalf("Error from Insert %v", err)
	}
	helperWaitAfterInsert(t, ctx, s)
}

func helperRandomData(start int64, end int64, gap int64) []btrdb.RawPoint {
	numpts := (end - start) / gap
	pts := make([]btrdb.RawPoint, numpts)
	for i, _ := range pts {
		pts[i].Time = start + (int64(i) * gap)
		pts[i].Value = rand.NormFloat64()
	}
	return pts
}

func helperRandomDataCount(start int64, end int64, numpts int64) []btrdb.RawPoint {
	gap := (end - start) / numpts
	return helperRandomData(start, end, gap)
}

func helperFloatEquals(x float64, y float64) bool {
	return math.Abs(x-y) < 1e-10*math.Max(math.Abs(x), math.Abs(y))
}

func helperMakeStatPoints(points []btrdb.RawPoint, start int64, end int64, queryWidth int64, aligned bool) []btrdb.StatPoint {
	if !aligned {
		end = start + queryWidth*((end-start)/queryWidth)
	}
	numPoints := int((end - start) / queryWidth)
	var statPoints []btrdb.StatPoint
	index := 0
	for index < len(points) {
		offset := index
		windowStart := start + int64(len(statPoints))*queryWidth

		/* For aligned queries, queryWidth will be 1<<pw,
		 * and we want to clear the pw bits from the window start
		 * ( e.g. start & ~((1<<pw) - 1) )
		 */
		if aligned {
			windowStart = windowStart &^ (queryWidth - 1)
		}

		for offset < len(points) && points[offset].Time < windowStart+queryWidth {
			offset++
		}

		pointsSlice := points[index:offset]

		/* Unaligned queries return empty stat points
		 * even if there is no data
		 */
		if !aligned && len(pointsSlice) == 0 {
			empty := btrdb.StatPoint{windowStart, 0, 0, 0, 0}
			statPoints = append(statPoints, empty)
			continue
		}
		min := math.Inf(1)
		max := math.Inf(-1)
		sum := 0.0
		for _, point := range pointsSlice {
			min = math.Min(min, point.Value)
			sum += point.Value
			max = math.Max(max, point.Value)
		}
		mean := sum / float64(len(pointsSlice))
		statPoint := btrdb.StatPoint{windowStart, min, mean, max, uint64(len(pointsSlice))}
		statPoints = append(statPoints, statPoint)
		index = offset
	}

	/* Unaligned queries always return the correct number of stat
	 * points, even if the end extends past the data
	 */
	for !aligned && len(statPoints) < numPoints {
		windowStart := start + int64(len(statPoints))*queryWidth
		empty := btrdb.StatPoint{windowStart, 0, 0, 0, 0}
		statPoints = append(statPoints, empty)
	}
	return statPoints
}

func helperCheckStatisticalEqual(firstPoints []btrdb.StatPoint, secondPoints []btrdb.StatPoint) error {
	if len(firstPoints) != len(secondPoints) {
		return fmt.Errorf("Stat point lenghts did not match %v %v", len(firstPoints), len(secondPoints))
	}

	for i := range firstPoints {
		first := firstPoints[i]
		second := secondPoints[i]
		if first.Time != second.Time {
			return fmt.Errorf("First time %v did not equal second time %v", first.Time, second.Time)
		}
		if !helperFloatEquals(first.Min, second.Min) {
			return fmt.Errorf("First min %v did not equal second min %v", first.Min, second.Min)
		}
		if !helperFloatEquals(first.Max, second.Max) {
			return fmt.Errorf("First max %v did not equal second max %v", first.Min, second.Max)
		}
		if first.Count != second.Count {
			return fmt.Errorf("First point count %v did not equal second point %v", first.Count, second.Count)
		}
		if !helperFloatEquals(first.Mean, second.Mean) {
			return fmt.Errorf("First mean %v did not equal second mean %v", first.Mean, second.Mean)
		}
	}

	return nil
}

func helperWindowQuery(t *testing.T, ctx context.Context, s *btrdb.Stream, start int64, end int64, width uint64, depth uint8, version uint64) ([]btrdb.StatPoint, uint64) {
	spc, verc, errc := s.Windows(ctx, start, end, width, depth, version)
	rv := make([]btrdb.StatPoint, 0)
	for sp := range spc {
		rv = append(rv, sp)
	}
	ver := <-verc
	err := <-errc
	if err != nil {
		t.Fatalf("window query error: %v", err)
	}

	return rv, ver
}

func helperStatisticalQuery(t *testing.T, ctx context.Context, s *btrdb.Stream, start int64, end int64, pwe uint8, version uint64) ([]btrdb.StatPoint, uint64) {
	spc, verc, errc := s.AlignedWindows(ctx, start, end, pwe, version)
	rv := make([]btrdb.StatPoint, 0)
	for sp := range spc {
		rv = append(rv, sp)
	}
	ver := <-verc
	err := <-errc
	if err != nil {
		t.Fatalf("statistical query error: %v", err)
	}

	return rv, ver
}
<|endoftext|>