<commit_before>package fetch

import (
	"context"
	"errors"
	"flag"
	"github.com/SlyMarbo/rss"
	log "github.com/golang/glog"
	"github.com/jrupac/goliath/models"
	"github.com/jrupac/goliath/storage"
	"github.com/jrupac/goliath/utils"
	"github.com/mat/besticon/besticon"
	"github.com/microcosm-cc/bluemonday"
	"html"
	"io/ioutil"
	"net/url"
	"strings"
	"sync"
	"time"
)

var (
	parseArticles = flag.Bool("parseArticles", false, "If true, parse article content via Mercury API.")
	sanitizeHTML  = flag.Bool("sanitizeHTML", false, "If true, sanitize HTML content with Bluemonday.")
)

var (
	pauseChan             = make(chan struct{})
	pauseChanDone         = make(chan struct{})
	resumeChan            = make(chan struct{})
	bluemondayTitlePolicy = bluemonday.StrictPolicy()
	bluemondayBodyPolicy  = bluemonday.UGCPolicy()
)

type imagePair struct {
	id      int64
	mime    string
	favicon []byte
}

// Pause stops all continuous feed fetching in a way that is resume-able.
// This call will block until fetching is fully paused. If fetching has not
// started yet, this call will block indefinitely.
func Pause() {
	pauseChan <- struct{}{}
	<-pauseChanDone
}

// Resume resumes continuous feed fetching with a fresh read of feeds.
// If fetching has not started yet, this call will block indefinitely.
func Resume() {
	resumeChan <- struct{}{}
}

// Start starts continuous feed fetching and writes fetched articles to the
// database.
func Start(ctx context.Context, d *storage.Database) {
	log.Infof("Starting continuous feed fetching.")

	// Add additional time layouts that sometimes appear in feeds.
	rss.TimeLayouts = append(rss.TimeLayouts, "2006-01-02")
	rss.TimeLayouts = append(rss.TimeLayouts, "Monday, 02 Jan 2006 15:04:05 MST")

	// Turn off logging of HTTP icon requests.
	besticon.SetLogOutput(ioutil.Discard)

	fctx, cancel := context.WithCancel(ctx)

	wg := &sync.WaitGroup{}
	wg.Add(1)
	go start(fctx, wg, d)

	for {
		select {
		case <-pauseChan:
			cancel()
			wg.Wait()
			log.Info("Fetcher paused.")
			pauseChanDone <- struct{}{}
		case <-resumeChan:
			fctx, cancel = context.WithCancel(ctx)
			wg.Add(1)
			go start(fctx, wg, d)
			log.Info("Fetcher resumed.")
		case <-ctx.Done():
			wg.Wait()
			cancel()
			return
		}
	}
}

func start(ctx context.Context, parent *sync.WaitGroup, d *storage.Database) {
	defer parent.Done()

	feeds, err := d.GetAllFeeds()
	if err != nil {
		log.Errorf("Failed to fetch all feeds: %s", err)
	}
	utils.DebugPrint("Feed list", feeds)

	wg := &sync.WaitGroup{}
	wg.Add(len(feeds))
	ac := make(chan models.Article)
	ic := make(chan imagePair)

	for _, f := range feeds {
		go func(f models.Feed) {
			defer wg.Done()
			fetchLoop(ctx, d, ac, ic, f)
		}(f)
	}

	for {
		select {
		case a := <-ac:
			utils.DebugPrint("Received a new article:", a)
			if err2 := d.InsertArticle(a); err2 != nil {
				log.Warningf("Failed to persist article: %+v: %s", a, err2)
			}
		case ip := <-ic:
			utils.DebugPrint("Received a new image:", ip)
			if err2 := d.InsertFavicon(ip.id, ip.mime, ip.favicon); err2 != nil {
				log.Warningf("Failed to persist icon for feed %d: %s", ip.id, err2)
			}
		case <-ctx.Done():
			log.Infof("Stopping fetching feeds...")
			wg.Wait()
			log.Infof("Stopped fetching feeds.")
			return
		}
	}
}

func fetchLoop(ctx context.Context, d *storage.Database, ac chan models.Article, ic chan imagePair, feed models.Feed) {
	log.Infof("Fetching URL '%s'", feed.URL)
	tick := make(<-chan time.Time)
	initalFetch := make(chan struct{})

	go func() {
		f, err := rss.Fetch(feed.URL)
		if err != nil {
			log.Warningf("Error for feed %d fetching URL '%s': %s", feed.ID, feed.URL, err)
			return
		}
		handleItems(ctx, &feed, d, f.Items, ac)
		handleImage(ctx, feed, f, ic)

		tick = time.After(time.Until(f.Refresh))
		log.Infof("Initial waiting to fetch %s until %s\n", feed.URL, f.Refresh)
		initalFetch <- struct{}{}
	}()

	for {
		select {
		case <-initalFetch:
			// Block on initial fetch here so that we can return early if needed
			continue
		case <-tick:
			log.Infof("Fetching feed %s", feed.URL)
			var refresh time.Time
			if f, err := rss.Fetch(feed.URL); err != nil {
				log.Warningf("Error fetching %s: %s", feed.URL, err)
				// If the request transiently fails, try again after a fixed interval.
				refresh = time.Now().Add(10 * time.Minute)
			} else {
				handleItems(ctx, &feed, d, f.Items, ac)
				refresh = f.Refresh
			}
			log.Infof("Waiting to fetch %s until %s\n", feed.URL, refresh)
			tick = time.After(time.Until(refresh))
		case <-ctx.Done():
			return
		}
	}
}

func handleItems(ctx context.Context, feed *models.Feed, d *storage.Database, items []*rss.Item, send chan models.Article) {
	latest := feed.Latest
	newLatest := latest

Loop:
	for _, item := range items {
		title := item.Title
		content := maybeUnescapeHtml(item.Content)
		summary := maybeUnescapeHtml(item.Summary)

		parsed := ""
		if *parseArticles {
			if p, err := parseArticleContent(item.Link); err != nil {
				log.Warningf("Parsing content failed: %s", err)
			} else {
				parsed = p
			}
		}
		if *sanitizeHTML {
			title = bluemondayTitlePolicy.Sanitize(title)
			content = bluemondayBodyPolicy.Sanitize(content)
			summary = bluemondayBodyPolicy.Sanitize(summary)
			parsed = bluemondayBodyPolicy.Sanitize(parsed)
		}

		a := models.Article{
			FeedID:    feed.ID,
			FolderID:  feed.FolderID,
			Title:     title,
			Summary:   summary,
			Content:   content,
			Parsed:    parsed,
			Link:      item.Link,
			Date:      item.Date,
			Read:      item.Read,
			Retrieved: time.Now(),
		}

		if a.Date.After(latest) {
			select {
			case send <- a:
				break
			case <-ctx.Done():
				// Break out of processing articles and just clean up.
				break Loop
			}
			if a.Date.After(newLatest) {
				newLatest = a.Date
			}
		} else {
			log.V(2).Infof("Not persisting too old article: %+v", a)
		}
	}

	err := d.UpdateLatestTimeForFeed(feed.ID, newLatest)
	if err != nil {
		log.Warningf("Failed to update latest feed time: %s", err)
	} else {
		feed.Latest = newLatest
	}
}

func handleImage(ctx context.Context, feed models.Feed, f *rss.Feed, send chan imagePair) {
	var icon besticon.Icon
	var feedHost string

	u, err := url.Parse(f.Link)
	if err == nil {
		feedHost = u.Hostname()
	}

	if i, err2 := tryIconFetch(f.Image.URL); err2 == nil {
		icon = i
	} else if i, err2 = tryIconFetch(f.Link); err2 == nil {
		icon = i
	} else if i, err2 = tryIconFetch(feedHost); err2 == nil {
		icon = i
	} else {
		return
	}

	select {
	case send <- imagePair{feed.ID, "image/" + icon.Format, icon.ImageData}:
		break
	case <-ctx.Done():
		break
	}
}

func tryIconFetch(link string) (besticon.Icon, error) {
	icon := besticon.Icon{}

	if link == "" {
		return icon, errors.New("invalid URL")
	}

	finder := besticon.IconFinder{}

	icons, err := finder.FetchIcons(link)
	if err != nil {
		return icon, err
	}

	if len(icons) == 0 {
		return icon, errors.New("no icons found")
	}

	for _, i := range icons {
		if i.URL != "" && i.Format != "" {
			return i, nil
		}
	}

	return icon, errors.New("no suitable icons found")
}

// maybeUnescapeHtml looks for occurrences of escaped HTML characters. If more
// than one is found in the given string, an HTML-unescaped string is returned.
// Otherwise, the given input is unmodified.
func maybeUnescapeHtml(content string) string {
	occ := 0
	// The HTML standard defines escape sequences for &, <, and >.
	// Single and double quotes are also escaped in attribute values, represented
	// here in two common forms each.
	escapes := []string{"&", "<", ">", """, "'", "'", """}

	for _, seq := range escapes {
		occ += strings.Count(content, seq)
	}

	if occ > 1 {
		return html.UnescapeString(content)
	}
	return content
}
<commit_msg>core/fetcher: Make some modifications to improve parsing of feeds.<commit_after>package fetch

import (
	"context"
	"errors"
	"flag"
	"github.com/SlyMarbo/rss"
	log "github.com/golang/glog"
	"github.com/jrupac/goliath/models"
	"github.com/jrupac/goliath/storage"
	"github.com/jrupac/goliath/utils"
	"github.com/mat/besticon/besticon"
	"github.com/microcosm-cc/bluemonday"
	"html"
	"io/ioutil"
	"net/url"
	"strings"
	"sync"
	"time"
)

var (
	parseArticles = flag.Bool("parseArticles", false, "If true, parse article content via Mercury API.")
	sanitizeHTML  = flag.Bool("sanitizeHTML", false, "If true, sanitize HTML content with Bluemonday.")
)

var (
	pauseChan             = make(chan struct{})
	pauseChanDone         = make(chan struct{})
	resumeChan            = make(chan struct{})
	bluemondayTitlePolicy = bluemonday.StrictPolicy()
	bluemondayBodyPolicy  = makeBodyPolicy()
)

type imagePair struct {
	id      int64
	mime    string
	favicon []byte
}

func makeBodyPolicy() *bluemonday.Policy {
	p := bluemonday.UGCPolicy()
	p.AllowAttrs("title").OnElements("img")
	return p
}

// Pause stops all continuous feed fetching in a way that is resume-able.
// This call will block until fetching is fully paused. If fetching has not
// started yet, this call will block indefinitely.
func Pause() {
	pauseChan <- struct{}{}
	<-pauseChanDone
}

// Resume resumes continuous feed fetching with a fresh read of feeds.
// If fetching has not started yet, this call will block indefinitely.
func Resume() {
	resumeChan <- struct{}{}
}

// Start starts continuous feed fetching and writes fetched articles to the
// database.
func Start(ctx context.Context, d *storage.Database) {
	log.Infof("Starting continuous feed fetching.")

	// Add additional time layouts that sometimes appear in feeds.
	rss.TimeLayouts = append(rss.TimeLayouts, "2006-01-02")
	rss.TimeLayouts = append(rss.TimeLayouts, "Monday, 02 Jan 2006 15:04:05 MST")
	rss.TimeLayouts = append(rss.TimeLayouts, "Mon, 02 Jan 2006")

	// Turn off logging of HTTP icon requests.
	besticon.SetLogOutput(ioutil.Discard)

	fctx, cancel := context.WithCancel(ctx)

	wg := &sync.WaitGroup{}
	wg.Add(1)
	go start(fctx, wg, d)

	for {
		select {
		case <-pauseChan:
			cancel()
			wg.Wait()
			log.Info("Fetcher paused.")
			pauseChanDone <- struct{}{}
		case <-resumeChan:
			fctx, cancel = context.WithCancel(ctx)
			wg.Add(1)
			go start(fctx, wg, d)
			log.Info("Fetcher resumed.")
		case <-ctx.Done():
			wg.Wait()
			cancel()
			return
		}
	}
}

func start(ctx context.Context, parent *sync.WaitGroup, d *storage.Database) {
	defer parent.Done()

	feeds, err := d.GetAllFeeds()
	if err != nil {
		log.Errorf("Failed to fetch all feeds: %s", err)
	}
	utils.DebugPrint("Feed list", feeds)

	wg := &sync.WaitGroup{}
	wg.Add(len(feeds))
	ac := make(chan models.Article)
	ic := make(chan imagePair)

	for _, f := range feeds {
		go func(f models.Feed) {
			defer wg.Done()
			fetchLoop(ctx, d, ac, ic, f)
		}(f)
	}

	for {
		select {
		case a := <-ac:
			utils.DebugPrint("Received a new article:", a)
			if err2 := d.InsertArticle(a); err2 != nil {
				log.Warningf("Failed to persist article: %+v: %s", a, err2)
			}
		case ip := <-ic:
			utils.DebugPrint("Received a new image:", ip)
			if err2 := d.InsertFavicon(ip.id, ip.mime, ip.favicon); err2 != nil {
				log.Warningf("Failed to persist icon for feed %d: %s", ip.id, err2)
			}
		case <-ctx.Done():
			log.Infof("Stopping fetching feeds...")
			wg.Wait()
			log.Infof("Stopped fetching feeds.")
			return
		}
	}
}

func fetchLoop(ctx context.Context, d *storage.Database, ac chan models.Article, ic chan imagePair, feed models.Feed) {
	log.Infof("Fetching URL '%s'", feed.URL)
	tick := make(<-chan time.Time)
	initalFetch := make(chan struct{})

	go func() {
		f, err := rss.Fetch(feed.URL)
		if err != nil {
			log.Warningf("Error for feed %d fetching URL '%s': %s", feed.ID, feed.URL, err)
			return
		}
		handleItems(ctx, &feed, d, f.Items, ac)
		handleImage(ctx, feed, f, ic)

		tick = time.After(time.Until(f.Refresh))
		log.Infof("Initial waiting to fetch %s until %s\n", feed.URL, f.Refresh)
		initalFetch <- struct{}{}
	}()

	for {
		select {
		case <-initalFetch:
			// Block on initial fetch here so that we can return early if needed
			continue
		case <-tick:
			log.Infof("Fetching feed %s", feed.URL)
			var refresh time.Time
			if f, err := rss.Fetch(feed.URL); err != nil {
				log.Warningf("Error fetching %s: %s", feed.URL, err)
				// If the request transiently fails, try again after a fixed interval.
				refresh = time.Now().Add(10 * time.Minute)
			} else {
				handleItems(ctx, &feed, d, f.Items, ac)
				refresh = f.Refresh
			}
			log.Infof("Waiting to fetch %s until %s\n", feed.URL, refresh)
			tick = time.After(time.Until(refresh))
		case <-ctx.Done():
			return
		}
	}
}

func handleItems(ctx context.Context, feed *models.Feed, d *storage.Database, items []*rss.Item, send chan models.Article) {
	latest := feed.Latest
	newLatest := latest

Loop:
	for _, item := range items {
		title := item.Title
		content := maybeUnescapeHtml(item.Content)
		summary := maybeUnescapeHtml(item.Summary)

		parsed := ""
		if *parseArticles {
			if p, err := parseArticleContent(item.Link); err != nil {
				log.Warningf("Parsing content failed: %s", err)
			} else {
				parsed = p
			}
		}
		if *sanitizeHTML {
			title = bluemondayTitlePolicy.Sanitize(title)
			content = bluemondayBodyPolicy.Sanitize(content)
			summary = bluemondayBodyPolicy.Sanitize(summary)
			parsed = bluemondayBodyPolicy.Sanitize(parsed)
		}

		a := models.Article{
			FeedID:    feed.ID,
			FolderID:  feed.FolderID,
			Title:     title,
			Summary:   summary,
			Content:   content,
			Parsed:    parsed,
			Link:      item.Link,
			Date:      item.Date,
			Read:      item.Read,
			Retrieved: time.Now(),
		}

		if a.Date.After(latest) {
			select {
			case send <- a:
				break
			case <-ctx.Done():
				// Break out of processing articles and just clean up.
				break Loop
			}
			if a.Date.After(newLatest) {
				newLatest = a.Date
			}
		} else {
			log.V(2).Infof("Not persisting too old article: %+v", a)
		}
	}

	err := d.UpdateLatestTimeForFeed(feed.ID, newLatest)
	if err != nil {
		log.Warningf("Failed to update latest feed time: %s", err)
	} else {
		feed.Latest = newLatest
	}
}

func handleImage(ctx context.Context, feed models.Feed, f *rss.Feed, send chan imagePair) {
	var icon besticon.Icon
	var feedHost string

	u, err := url.Parse(f.Link)
	if err == nil {
		feedHost = u.Hostname()
	}

	if i, err2 := tryIconFetch(f.Image.URL); err2 == nil {
		icon = i
	} else if i, err2 = tryIconFetch(f.Link); err2 == nil {
		icon = i
	} else if i, err2 = tryIconFetch(feedHost); err2 == nil {
		icon = i
	} else {
		return
	}

	select {
	case send <- imagePair{feed.ID, "image/" + icon.Format, icon.ImageData}:
		break
	case <-ctx.Done():
		break
	}
}

func tryIconFetch(link string) (besticon.Icon, error) {
	icon := besticon.Icon{}

	if link == "" {
		return icon, errors.New("invalid URL")
	}

	finder := besticon.IconFinder{}

	icons, err := finder.FetchIcons(link)
	if err != nil {
		return icon, err
	}

	if len(icons) == 0 {
		return icon, errors.New("no icons found")
	}

	for _, i := range icons {
		if i.URL != "" && i.Format != "" {
			return i, nil
		}
	}

	return icon, errors.New("no suitable icons found")
}

// maybeUnescapeHtml looks for occurrences of escaped HTML characters. If more
// than one is found in the given string, an HTML-unescaped string is returned.
// Otherwise, the given input is unmodified.
func maybeUnescapeHtml(content string) string {
	occ := 0
	// The HTML standard defines escape sequences for &, <, and >.
	// Single and double quotes are also escaped in attribute values, represented
	// here in two common forms each.
	escapes := []string{"&", "<", ">", """, "'", "'", """}

	for _, seq := range escapes {
		occ += strings.Count(content, seq)
	}

	if occ > 1 {
		return html.UnescapeString(content)
	}
	return content
}
<|endoftext|>


<commit_before>package instance

import (
	"os"
	"time"
)

const FrameworkPrefix = "grnc"

func ExitError() {
	os.Exit(1)
}

func ExitNormal() {
	os.Exit(0)
}

type System struct {
	BlockIntervalMS      time.Duration //The interval (in milliseconds) between checks of blocking components during the start->available lifecycle
	BlockRetries         int           //How many chances blocking components should be given to declare themselves ready before the application exits with an error
	BlockTriesBeforeWarn int           //How many times a blocking component can declare it is blocked before a warning message is logged
	FlushMergedConfig    bool          //If the merged JSON configuration files should be discarded after startup is complete.
	GCAfterConfigure     bool          //If a garbage collection should be invoked after the container has finished populating and configuring the IoC container.
	GCAfterStart         bool          //If a garbage collection should be invoked after the container has called StartComponent on all components (but before AllowAccess).
	StopIntervalMS       time.Duration //The interval (in milliseconds) between checks of stoppable components to see if they are ready to be stopped.
	StopRetries          int           //How many chances stoppable components should be given to declare themselves ready to stop before the container stops them anyway.
	StopTriesBeforeWarn  int           //How many times a stoppable component can declare it is not ready to stop before a warning message is logged

}

const InstanceIDComponent = FrameworkPrefix + "InstanceIdentifier"

type InstanceIdentifier struct {
	ID string
}

type InstanceIdentifierReceiver interface {
	RegisterInstanceID(*InstanceIdentifier)
}
<commit_msg>GoDoc<commit_after>// Copyright 2016 Granitic. All rights reserved.
// Use of this source code is governed by an Apache 2.0 license that can be found in the LICENSE file at the root of this project.

/*
Package instance defines functionality for identifying and tuning an individual instance of a Granitic application.

It is anticipated that most Granitic applications will be deployed as part of a group behind a load-balancer or some other
service discovery mechanism. In these circumstances, it is important that each individual instance can be named. So other
services can identify it.

This package defines a type for storing an instance ID and an interface to be implemented
by any component that needs to be aware of the ID of the current instance. An instance ID is specified when starting
Granitic via a command line argument or an InitialSettings struct. See the documentation for the granitic pacakge for
more detail.

This package also defines the data structure for the implicit System facility (which cannot be disabled, but can be configured).
The System facility controls start up and shutdown behaviour as well as some memory management settings. See
http://granitic.io/1.0/ref/system
*/
package instance

import (
	"os"
	"time"
)

// The prefix for all Granitic owned components stored in the IoC container to namespace them apart from user defined
// components. It is strongly recommended (but not enforced) that user components do not use this prefix for their own names.
const FrameworkPrefix = "grnc"

// Immediately exit the application with the return value 1 to signify a problem. Does not cleanly stop IoC components.
func ExitError() {
	os.Exit(1)
}

// Immediately exit the application with the return value 0 to signify a normal exit. Does not cleanly stop IoC components.
func ExitNormal() {
	os.Exit(0)
}

// A structure used by the implicit System facility to control start, stop and some memory management behaviour.
type System struct {
	//The interval (in milliseconds) between checks of blocking components during the start->available lifecycle phase transition.
	BlockIntervalMS time.Duration

	//How many chances blocking components should be given to declare themselves ready before the application exits with an error.
	BlockRetries int

	//How many times a blocking component can declare it is blocked before a warning message is logged.
	BlockTriesBeforeWarn int

	//If the merged JSON configuration files should be discarded after startup is complete.
	FlushMergedConfig bool

	//If a garbage collection should be invoked after the container has finished populating and configuring the IoC container.
	GCAfterConfigure bool

	//If a garbage collection should be invoked after the container has called StartComponent on all components (but before AllowAccess).
	GCAfterStart bool

	//The interval (in milliseconds) between checks of stoppable components to see if they are ready to be stopped.
	StopIntervalMS time.Duration

	//How many chances stoppable components should be given to declare themselves ready to stop before the container stops them anyway.
	StopRetries int

	//How many times a stoppable component can declare it is not ready to stop before a warning message is logged
	StopTriesBeforeWarn int
}

// The name of the component in the IoC container holding an instance ID.
const InstanceIDComponent = FrameworkPrefix + "InstanceIdentifier"

// A structure used to store the ID of a particular instance of a Granitic application. See the granitic package
// documentation for instructions on how to define the ID at application start time.
type InstanceIdentifier struct {
	// A identifier for this instance application.
	ID string
}

// Implemented by any component that needs to be aware of the ID of the current application instance.
type InstanceIdentifierReceiver interface {
	// RegisterInstanceID is automatically called by the IoC container to inject the instance ID.
	RegisterInstanceID(*InstanceIdentifier)
}
<|endoftext|>


<commit_before>package main

import (
	"strings"
)

type GitController struct {
	Folder string
}

func (this *GitController) ExistsLocal() bool {

	exitcode, _, _, err := CmdRun(this.Folder, "git", "status")

	if err != nil {
		EXIT_ERROR("Error executing command 'git status'\n\n"+err.Error(), EXIT_GIT_ERROR)
	}

	return exitcode == 0
}

func (this *GitController) ExecGitCommand(args ...string) string {
	exitcode, stdout, stderr, err := CmdRun(this.Folder, "git", args...)

	if err != nil {
		EXIT_ERROR("Error executing command 'git "+args[0]+"'\n\n"+err.Error(), EXIT_GIT_ERROR)
	}

	if exitcode != 0 {
		EXIT_ERROR("Error in command 'git "+args[0]+"'\n\n"+stderr, EXIT_GIT_ERROR)
	}

	return stdout
}

func (this *GitController) ExecCredGitCommand(cred GGCredentials, args ...string) string {

	if IsEmpty(cred.Host) || IsEmpty(cred.Username) || IsEmpty(cred.Password) {
		return this.ExecGitCommand(args...)
	}

	credRC := "machine " + cred.Host + " login " + cred.Username + " password " + cred.Password

	EnterNetRCBlock(credRC)

	stdout := this.ExecGitCommand(args...)

	ExitNetRCBlock()

	return stdout
}

func (this *GitController) QueryGitCommand(args ...string) string {
	exitcode, stdout, stderr, err := CmdRun(this.Folder, "git", args...)

	if err != nil {
		EXIT_ERROR("Error executing command 'git "+args[0]+"'\n\n"+err.Error(), EXIT_GIT_ERROR)
	}

	if exitcode != 0 {
		EXIT_ERROR("Error in command 'git "+args[0]+"'\n\n"+stderr, EXIT_GIT_ERROR)
	}

	return stdout
}

func (this *GitController) RemoveAllRemotes() {
	branches := this.QueryGitCommand("remote")

	for _, remote := range strings.Split(branches, "\n") {
		if !IsEmpty(remote) {
			this.ExecGitCommand("remote", "rm", remote)
		}
	}
}

func (this *GitController) CloneOrPull(branch string, remote string, cred GGCredentials) {

	if this.ExistsLocal() {
		this.RemoveAllRemotes()
		this.ExecGitCommand("remote", "add", "origin", remote)

		this.ExecCredGitCommand(cred, "fetch", "--all")
		this.ExecGitCommand("checkout", "-f", branch)
		this.ExecGitCommand("reset", "--hard", "origin/"+branch)

	} else {
		this.ExecCredGitCommand(cred, "clone", remote, ".", "--origin", "origin")
		this.ExecGitCommand("checkout", "-f", "origin/"+branch)
	}

	this.ExecCredGitCommand(cred, "branch", "-u", "origin/"+branch, branch)
	this.ExecCredGitCommand(cred, "clean", "-f", "-d")
}

func (this *GitController) PushBack(branch string, remote string, cred GGCredentials, useForce bool) {
	if this.HasRemoteBranch(branch) {
		this.PushBackExistingBranch(branch, remote, cred, useForce)
	} else {
		this.PushBackNewBranch(branch, remote, cred, useForce)
	}
}

func (this *GitController) PushBackExistingBranch(branch string, remote string, cred GGCredentials, useForce bool) {

	this.RemoveAllRemotes()
	this.ExecGitCommand("remote", "add", "origin", remote)

	this.ExecCredGitCommand(cred, "fetch", "--all")
	this.ExecCredGitCommand(cred, "branch", "-u", "origin/"+branch, branch)
	this.ExecCredGitCommand(cred, "checkout", branch)
	status := this.ExecGitCommand("status")
	LOG_OUT(status)

	var commandoutput string

	if useForce {
		commandoutput = this.ExecCredGitCommand(cred, "push", "origin", "HEAD:"+branch, "--force")
	} else {
		commandoutput = this.ExecCredGitCommand(cred, "push", "origin", "HEAD:"+branch)
	}

	LOG_OUT(commandoutput)
}

func (this *GitController) PushBackNewBranch(branch string, remote string, cred GGCredentials, useForce bool) {

	this.RemoveAllRemotes()
	this.ExecGitCommand("remote", "add", "origin", remote)

	this.ExecCredGitCommand(cred, "fetch", "--all")
	this.ExecCredGitCommand(cred, "checkout", branch)
	status := this.ExecGitCommand("status")
	LOG_OUT(status)

	var commandoutput string

	if useForce {
		commandoutput = this.ExecCredGitCommand(cred, "push", "origin", "HEAD:"+branch, "--force")
	} else {
		commandoutput = this.ExecCredGitCommand(cred, "push", "origin", "HEAD:"+branch)
	}

	LOG_OUT(commandoutput)
}

func (this *GitController) GarbageCollect() {
	this.ExecGitCommand("gc")
}

func (this *GitController) ListLocalBranches() []string {
	stdout := this.QueryGitCommand("branch", "-a", "--list")
	lines := strings.Split(stdout, "\n")

	result := make([]string, 0)

	for _, line := range lines {
		line = strings.TrimSpace(line)
		line = strings.TrimLeft(line, "*")
		line = strings.TrimSpace(line)

		branch := ""

		if strings.Contains(line, " -> ") {
			line = line[:strings.Index(line, " -> ")]
		}

		if strings.HasPrefix(strings.ToLower(line), "remotes/origin/") {
			branch = line[15:]
		} else {
			branch = line
		}

		branch = strings.TrimSpace(branch)

		if !IsEmpty(branch) && !strings.EqualFold(branch, "HEAD") && branch[0:1] != "(" {
			result = AppendIfUniqueCaseInsensitive(result, branch)
		}
	}

	return result
}

func (this *GitController) HasRemoteBranch(branchname string) bool {
	stdout := this.QueryGitCommand("branch", "-r", "--list")
	lines := strings.Split(stdout, "\n")

	for _, line := range lines {
		line = strings.TrimSpace(line)
		line = strings.TrimLeft(line, "*")
		line = strings.TrimSpace(line)

		branch := ""

		if strings.Contains(line, " -> ") {
			line = line[:strings.Index(line, " -> ")]
		}

		if strings.HasPrefix(strings.ToLower(line), "remotes/origin/") {
			branch = line[15:]
		} else {
			branch = line
		}

		branch = strings.TrimSpace(branch)

		if strings.EqualFold(branch, branchname) {
			return true
		}
	}

	return false
}
<commit_msg>added tag support (push all tags to remote)<commit_after>package main

import (
	"strings"
)

type GitController struct {
	Folder string
}

func (this *GitController) ExistsLocal() bool {

	exitcode, _, _, err := CmdRun(this.Folder, "git", "status")

	if err != nil {
		EXIT_ERROR("Error executing command 'git status'\n\n"+err.Error(), EXIT_GIT_ERROR)
	}

	return exitcode == 0
}

func (this *GitController) ExecGitCommand(args ...string) string {
	exitcode, stdout, stderr, err := CmdRun(this.Folder, "git", args...)

	if err != nil {
		EXIT_ERROR("Error executing command 'git "+args[0]+"'\n\n"+err.Error(), EXIT_GIT_ERROR)
	}

	if exitcode != 0 {
		EXIT_ERROR("Error in command 'git "+args[0]+"'\n\n"+stderr, EXIT_GIT_ERROR)
	}

	return stdout
}

func (this *GitController) ExecCredGitCommand(cred GGCredentials, args ...string) string {

	if IsEmpty(cred.Host) || IsEmpty(cred.Username) || IsEmpty(cred.Password) {
		return this.ExecGitCommand(args...)
	}

	credRC := "machine " + cred.Host + " login " + cred.Username + " password " + cred.Password

	EnterNetRCBlock(credRC)

	stdout := this.ExecGitCommand(args...)

	ExitNetRCBlock()

	return stdout
}

func (this *GitController) QueryGitCommand(args ...string) string {
	exitcode, stdout, stderr, err := CmdRun(this.Folder, "git", args...)

	if err != nil {
		EXIT_ERROR("Error executing command 'git "+args[0]+"'\n\n"+err.Error(), EXIT_GIT_ERROR)
	}

	if exitcode != 0 {
		EXIT_ERROR("Error in command 'git "+args[0]+"'\n\n"+stderr, EXIT_GIT_ERROR)
	}

	return stdout
}

func (this *GitController) RemoveAllRemotes() {
	branches := this.QueryGitCommand("remote")

	for _, remote := range strings.Split(branches, "\n") {
		if !IsEmpty(remote) {
			this.ExecGitCommand("remote", "rm", remote)
		}
	}
}

func (this *GitController) CloneOrPull(branch string, remote string, cred GGCredentials) {

	if this.ExistsLocal() {
		this.RemoveAllRemotes()
		this.ExecGitCommand("remote", "add", "origin", remote)

		this.ExecCredGitCommand(cred, "fetch", "--all")
		this.ExecGitCommand("checkout", "-f", branch)
		this.ExecGitCommand("reset", "--hard", "origin/"+branch)

	} else {
		this.ExecCredGitCommand(cred, "clone", remote, ".", "--origin", "origin")
		this.ExecGitCommand("checkout", "-f", "origin/"+branch)
	}

	this.ExecCredGitCommand(cred, "branch", "-u", "origin/"+branch, branch)
	this.ExecCredGitCommand(cred, "clean", "-f", "-d")
}

func (this *GitController) PushBack(branch string, remote string, cred GGCredentials, useForce bool) {
	if this.HasRemoteBranch(branch) {
		this.PushBackExistingBranch(branch, remote, cred, useForce)
	} else {
		this.PushBackNewBranch(branch, remote, cred, useForce)
	}
}

func (this *GitController) PushBackExistingBranch(branch string, remote string, cred GGCredentials, useForce bool) {

	this.RemoveAllRemotes()
	this.ExecGitCommand("remote", "add", "origin", remote)

	this.ExecCredGitCommand(cred, "fetch", "--all")
	this.ExecCredGitCommand(cred, "branch", "-u", "origin/"+branch, branch)
	this.ExecCredGitCommand(cred, "checkout", branch)
	status := this.ExecGitCommand("status")
	LOG_OUT(status)

	var commandoutput string

	if useForce {
		commandoutput = this.ExecCredGitCommand(cred, "push", "origin", "HEAD:"+branch, "--follow-tags", "--force")
	} else {
		commandoutput = this.ExecCredGitCommand(cred, "push", "origin", "HEAD:"+branch, "--follow-tags")
	}

	LOG_OUT(commandoutput)
}

func (this *GitController) PushBackNewBranch(branch string, remote string, cred GGCredentials, useForce bool) {

	this.RemoveAllRemotes()
	this.ExecGitCommand("remote", "add", "origin", remote)

	this.ExecCredGitCommand(cred, "fetch", "--all")
	this.ExecCredGitCommand(cred, "checkout", branch)
	status := this.ExecGitCommand("status")
	LOG_OUT(status)

	var commandoutput string

	if useForce {
		commandoutput = this.ExecCredGitCommand(cred, "push", "origin", "HEAD:"+branch, "--tags", "--force")
	} else {
		commandoutput = this.ExecCredGitCommand(cred, "push", "origin", "HEAD:"+branch, "--tags")
	}

	LOG_OUT(commandoutput)
}

func (this *GitController) GarbageCollect() {
	this.ExecGitCommand("gc")
}

func (this *GitController) ListLocalBranches() []string {
	stdout := this.QueryGitCommand("branch", "-a", "--list")
	lines := strings.Split(stdout, "\n")

	result := make([]string, 0)

	for _, line := range lines {
		line = strings.TrimSpace(line)
		line = strings.TrimLeft(line, "*")
		line = strings.TrimSpace(line)

		branch := ""

		if strings.Contains(line, " -> ") {
			line = line[:strings.Index(line, " -> ")]
		}

		if strings.HasPrefix(strings.ToLower(line), "remotes/origin/") {
			branch = line[15:]
		} else {
			branch = line
		}

		branch = strings.TrimSpace(branch)

		if !IsEmpty(branch) && !strings.EqualFold(branch, "HEAD") && branch[0:1] != "(" {
			result = AppendIfUniqueCaseInsensitive(result, branch)
		}
	}

	return result
}

func (this *GitController) HasRemoteBranch(branchname string) bool {
	stdout := this.QueryGitCommand("branch", "-r", "--list")
	lines := strings.Split(stdout, "\n")

	for _, line := range lines {
		line = strings.TrimSpace(line)
		line = strings.TrimLeft(line, "*")
		line = strings.TrimSpace(line)

		branch := ""

		if strings.Contains(line, " -> ") {
			line = line[:strings.Index(line, " -> ")]
		}

		if strings.HasPrefix(strings.ToLower(line), "remotes/origin/") {
			branch = line[15:]
		} else {
			branch = line
		}

		branch = strings.TrimSpace(branch)

		if strings.EqualFold(branch, branchname) {
			return true
		}
	}

	return false
}
<|endoftext|>


<commit_before>package github

import (
	"github.com/hoisie/web"
	"github.com/plausibility/gitsby/git"
	"github.com/plausibility/gitsby/util"
	"github.com/plausibility/gitsby/server"
)

func Hook(ctx *web.Context) {
	p := NewPayload([]byte(ctx.Params["payload"]))
	owner, name := git.RepoName(p.Url())
	if owner == "" || name == "" {
		util.Infof("Received unparseable payload: '%s/%s', ignoring!", owner, name)
		return
	}
	repo, ok := server.Config.ReposActive[owner + "/" + name]
	if !ok {
		util.Infof("Received payload for unknown repo: '%s/%s', ignoring!", owner, name)
		return
	}
	util.Info("Received payload for: '%s/%s'!", owner, name)
	// goroutines to cheat around `git pull` blocking HTTP.
	go func() {
		if ok, err := repo.Pull(); err == nil && ok {
			// Only deploy if we pull'd successfully.
			repo.Deploy()
		}
	}();
}
<commit_msg>Typo..<commit_after>package github

import (
	"github.com/hoisie/web"
	"github.com/plausibility/gitsby/git"
	"github.com/plausibility/gitsby/util"
	"github.com/plausibility/gitsby/server"
)

func Hook(ctx *web.Context) {
	p := NewPayload([]byte(ctx.Params["payload"]))
	owner, name := git.RepoName(p.Url())
	if owner == "" || name == "" {
		util.Infof("Received unparseable payload: '%s/%s', ignoring!", owner, name)
		return
	}
	repo, ok := server.Config.ReposActive[owner + "/" + name]
	if !ok {
		util.Infof("Received payload for unknown repo: '%s/%s', ignoring!", owner, name)
		return
	}
	util.Infof("Received payload for: '%s/%s'!", owner, name)
	// goroutines to cheat around `git pull` blocking HTTP.
	go func() {
		if ok, err := repo.Pull(); err == nil && ok {
			// Only deploy if we pull'd successfully.
			repo.Deploy()
		}
	}();
}
<|endoftext|>


<commit_before>package internal

import (
	"time"

	"github.com/go-redis/redis/v8/internal/rand"
)

func RetryBackoff(retry int, minBackoff, maxBackoff time.Duration) time.Duration {
	if retry < 0 {
		panic("not reached")
	}
	if minBackoff == 0 {
		return 0
	}

	d := minBackoff << uint(retry)
	d = minBackoff + time.Duration(rand.Int63n(int64(d)))

	if d > maxBackoff || d < minBackoff {
		d = maxBackoff
	}

	return d
}
<commit_msg>Guard against overflow in retry<commit_after>package internal

import (
	"time"

	"github.com/go-redis/redis/v8/internal/rand"
)

func RetryBackoff(retry int, minBackoff, maxBackoff time.Duration) time.Duration {
	if retry < 0 {
		panic("not reached")
	}
	if minBackoff == 0 {
		return 0
	}

	d := minBackoff << uint(retry)
	if d < minBackoff {
		return maxBackoff
	}

	d = minBackoff + time.Duration(rand.Int63n(int64(d)))

	if d > maxBackoff || d < minBackoff {
		d = maxBackoff
	}

	return d
}
<|endoftext|>


<commit_before>package tele

import (
	"context"

	"github.com/juju/errors"
	"github.com/temoto/vender/internal/money"
	"github.com/temoto/vender/internal/state"
	tele_api "github.com/temoto/vender/tele"
)

const logMsgDisabled = "tele disabled"

func (self *tele) CommandReplyErr(c *tele_api.Command, e error) {
	if !self.config.Enabled {
		self.log.Infof(logMsgDisabled)
		return
	}
	errText := ""
	if e != nil {
		errText = e.Error()
	}
	r := tele_api.Response{
		CommandId: c.Id,
		Error:     errText,
	}
	err := self.qpushCommandResponse(c, &r)
	if err != nil {
		self.log.Error(errors.Annotatef(err, "CRITICAL command=%#v response=%#v", c, r))
	}
}

func (self *tele) Error(e error) {
	if !self.config.Enabled {
		self.log.Infof(logMsgDisabled)
		return
	}

	self.log.Debugf("tele.Error: " + errors.ErrorStack(e))
	tm := &tele_api.Telemetry{
		Error:        &tele_api.Telemetry_Error{Message: e.Error()},
		BuildVersion: self.config.BuildVersion,
	}
	if err := self.qpushTelemetry(tm); err != nil {
		self.log.Errorf("CRITICAL qpushTelemetry telemetry_error=%#v err=%v", tm.Error, err)
	}
}

func (self *tele) Report(ctx context.Context, serviceTag bool) error {
	if !self.config.Enabled {
		self.log.Infof(logMsgDisabled)
		return nil
	}

	g := state.GetGlobal(ctx)
	moneysys := money.GetGlobal(ctx)
	tm := &tele_api.Telemetry{
		Inventory:    g.Inventory.Tele(),
		MoneyCashbox: moneysys.TeleCashbox(ctx),
		MoneyChange:  moneysys.TeleChange(ctx),
		AtService:    serviceTag,
		BuildVersion: g.BuildVersion,
	}
	err := self.qpushTelemetry(tm)
	if err != nil {
		self.log.Errorf("CRITICAL qpushTelemetry tm=%#v err=%v", tm, err)
	}
	return err
}

func (self *tele) State(s tele_api.State) {
	if !self.config.Enabled {
		self.log.Infof(logMsgDisabled)
		return
	}

	self.log.Infof("tele.State s=%v", s)
	self.stateCh <- s
}

func (self *tele) StatModify(fun func(s *tele_api.Stat)) {
	if !self.config.Enabled {
		self.log.Infof(logMsgDisabled)
		return
	}

	self.stat.Lock()
	fun(&self.stat)
	self.stat.Unlock()
}

func (self *tele) Transaction(tx tele_api.Telemetry_Transaction) {
	if !self.config.Enabled {
		self.log.Infof(logMsgDisabled)
		return
	}
	err := self.qpushTelemetry(&tele_api.Telemetry{Transaction: &tx})
	if err != nil {
		self.log.Errorf("CRITICAL transaction=%#v err=%v", tx, err)
	}
}
<commit_msg>tele.State() was blocking under slow/dropping network<commit_after>package tele

import (
	"context"
	"time"

	"github.com/juju/errors"
	"github.com/temoto/vender/internal/money"
	"github.com/temoto/vender/internal/state"
	tele_api "github.com/temoto/vender/tele"
)

const logMsgDisabled = "tele disabled"

func (self *tele) CommandReplyErr(c *tele_api.Command, e error) {
	if !self.config.Enabled {
		self.log.Infof(logMsgDisabled)
		return
	}
	errText := ""
	if e != nil {
		errText = e.Error()
	}
	r := tele_api.Response{
		CommandId: c.Id,
		Error:     errText,
	}
	err := self.qpushCommandResponse(c, &r)
	if err != nil {
		self.log.Error(errors.Annotatef(err, "CRITICAL command=%#v response=%#v", c, r))
	}
}

func (self *tele) Error(e error) {
	if !self.config.Enabled {
		self.log.Infof(logMsgDisabled)
		return
	}

	self.log.Debugf("tele.Error: " + errors.ErrorStack(e))
	tm := &tele_api.Telemetry{
		Error:        &tele_api.Telemetry_Error{Message: e.Error()},
		BuildVersion: self.config.BuildVersion,
	}
	if err := self.qpushTelemetry(tm); err != nil {
		self.log.Errorf("CRITICAL qpushTelemetry telemetry_error=%#v err=%v", tm.Error, err)
	}
}

func (self *tele) Report(ctx context.Context, serviceTag bool) error {
	if !self.config.Enabled {
		self.log.Infof(logMsgDisabled)
		return nil
	}

	g := state.GetGlobal(ctx)
	moneysys := money.GetGlobal(ctx)
	tm := &tele_api.Telemetry{
		Inventory:    g.Inventory.Tele(),
		MoneyCashbox: moneysys.TeleCashbox(ctx),
		MoneyChange:  moneysys.TeleChange(ctx),
		AtService:    serviceTag,
		BuildVersion: g.BuildVersion,
	}
	err := self.qpushTelemetry(tm)
	if err != nil {
		self.log.Errorf("CRITICAL qpushTelemetry tm=%#v err=%v", tm, err)
	}
	return err
}

func (self *tele) State(s tele_api.State) {
	if !self.config.Enabled {
		self.log.Infof(logMsgDisabled)
		return
	}

	// FIXME tests expecting blocking behavior and just select default: nothing
	tmr := time.NewTimer(100 * time.Millisecond)
	select {
	case self.stateCh <- s:
		self.log.Infof("tele.State s=%s", s)
		tmr.Stop()

	case <-tmr.C:
		self.log.Infof("tele.State s=%s chan busy, likely network problem", s)
	}
}

func (self *tele) StatModify(fun func(s *tele_api.Stat)) {
	if !self.config.Enabled {
		self.log.Infof(logMsgDisabled)
		return
	}

	self.stat.Lock()
	fun(&self.stat)
	self.stat.Unlock()
}

func (self *tele) Transaction(tx tele_api.Telemetry_Transaction) {
	if !self.config.Enabled {
		self.log.Infof(logMsgDisabled)
		return
	}
	err := self.qpushTelemetry(&tele_api.Telemetry{Transaction: &tx})
	if err != nil {
		self.log.Errorf("CRITICAL transaction=%#v err=%v", tx, err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build js

package ui

import (
	"github.com/gopherjs/gopherjs/js"
	"strconv"
)

func ExecOnUIThread(f func()) {
	f()
}

func Start(width, height, scale int, title string) (actualScale int, err error) {
	return currentUI.start(width, height, scale, title)
}

func Terminate() {
	currentUI.terminate()
}

func DoEvents() error {
	return currentUI.doEvents()
}

func IsClosed() bool {
	return currentUI.isClosed()
}

func SwapBuffers() {
	currentUI.swapBuffers()
}

func SetScreenSize(width, height int) (bool, int) {
	scale := canvas.Get("dataset").Get("ebitenScale").Int()
	result := currentUI.setScreenSize(width, height, scale)
	actualScale := canvas.Get("dataset").Get("ebitenActualScale").Int()
	return result, actualScale
}

func SetScreenScale(scale int) (bool, int) {
	width, height := currentUI.size()
	result := currentUI.setScreenSize(width, height, scale)
	actualScale := canvas.Get("dataset").Get("ebitenActualScale").Int()
	return result, actualScale
}

var canvas js.Object

type userInterface struct{}

var currentUI = &userInterface{}

// NOTE: This returns true even when the browser is not active.
func shown() bool {
	return !js.Global.Get("document").Get("hidden").Bool()
}

func vsync() {
	ch := make(chan struct{})
	// TODO: In iOS8, this is called at every 1/30[sec] frame.
	// Can we use DOMHighResTimeStamp?
	js.Global.Get("window").Call("requestAnimationFrame", func() {
		close(ch)
	})
	<-ch
}

func (*userInterface) doEvents() error {
	vsync()
	for !shown() {
		vsync()
	}
	currentInput.UpdateGamepads()
	return nil
}

func (*userInterface) terminate() {
	// Do nothing.
}

func (*userInterface) isClosed() bool {
	return false
}

func (*userInterface) swapBuffers() {
	// Do nothing.
}

func Init() {
	// Do nothing in node.js.
	if js.Global.Get("require") != js.Undefined {
		return
	}

	doc := js.Global.Get("document")
	window := js.Global.Get("window")
	if doc.Get("body") == nil {
		ch := make(chan struct{})
		window.Call("addEventListener", "load", func() {
			close(ch)
		})
		<-ch
	}

	canvas = doc.Call("createElement", "canvas")
	canvas.Set("width", 16)
	canvas.Set("height", 16)
	doc.Get("body").Call("appendChild", canvas)

	htmlStyle := doc.Get("documentElement").Get("style")
	htmlStyle.Set("height", "100%")
	htmlStyle.Set("margin", "0")
	htmlStyle.Set("padding", "0")

	bodyStyle := doc.Get("body").Get("style")
	bodyStyle.Set("backgroundColor", "#000")
	bodyStyle.Set("position", "relative")
	bodyStyle.Set("height", "100%")
	bodyStyle.Set("margin", "0")
	bodyStyle.Set("padding", "0")
	doc.Get("body").Call("addEventListener", "click", func() {
		canvas.Call("focus")
	})

	canvasStyle := canvas.Get("style")
	canvasStyle.Set("position", "absolute")

	// Make the canvas focusable.
	canvas.Call("setAttribute", "tabindex", 1)
	canvas.Get("style").Set("outline", "none")

	// Keyboard
	canvas.Call("addEventListener", "keydown", func(e js.Object) {
		e.Call("preventDefault")
		code := e.Get("keyCode").Int()
		currentInput.KeyDown(code)
	})
	canvas.Call("addEventListener", "keyup", func(e js.Object) {
		e.Call("preventDefault")
		code := e.Get("keyCode").Int()
		currentInput.KeyUp(code)
	})

	// Mouse
	canvas.Call("addEventListener", "mousedown", func(e js.Object) {
		e.Call("preventDefault")
		button := e.Get("button").Int()
		currentInput.MouseDown(button)
		setMouseCursorFromEvent(e)
	})
	canvas.Call("addEventListener", "mouseup", func(e js.Object) {
		e.Call("preventDefault")
		button := e.Get("button").Int()
		currentInput.MouseUp(button)
		setMouseCursorFromEvent(e)
	})
	canvas.Call("addEventListener", "mousemove", func(e js.Object) {
		e.Call("preventDefault")
		setMouseCursorFromEvent(e)
	})
	canvas.Call("addEventListener", "contextmenu", func(e js.Object) {
		e.Call("preventDefault")
	})

	// Touch (emulating mouse events)
	// TODO: Create indimendent touch functions
	canvas.Call("addEventListener", "touchstart", func(e js.Object) {
		e.Call("preventDefault")
		currentInput.MouseDown(0)
		touches := e.Get("changedTouches")
		touch := touches.Index(0)
		setMouseCursorFromEvent(touch)
	})
	canvas.Call("addEventListener", "touchend", func(e js.Object) {
		e.Call("preventDefault")
		currentInput.MouseUp(0)
		touches := e.Get("changedTouches")
		touch := touches.Index(0)
		setMouseCursorFromEvent(touch)
	})
	canvas.Call("addEventListener", "touchmove", func(e js.Object) {
		e.Call("preventDefault")
		touches := e.Get("changedTouches")
		touch := touches.Index(0)
		setMouseCursorFromEvent(touch)
	})

	// Gamepad
	window.Call("addEventListener", "gamepadconnected", func(e js.Object) {
		// Do nothing.
	})
}

func setMouseCursorFromEvent(e js.Object) {
	scale := canvas.Get("dataset").Get("ebitenScale").Int()
	rect := canvas.Call("getBoundingClientRect")
	x, y := e.Get("clientX").Int(), e.Get("clientY").Int()
	x -= rect.Get("left").Int()
	y -= rect.Get("top").Int()
	currentInput.SetMouseCursor(x/scale, y/scale)
}

func devicePixelRatio() int {
	// TODO: What if ratio is not an integer but a float?
	ratio := js.Global.Get("window").Get("devicePixelRatio").Int()
	if ratio == 0 {
		ratio = 1
	}
	return ratio
}

func (u *userInterface) start(width, height, scale int, title string) (actualScale int, err error) {
	doc := js.Global.Get("document")
	doc.Set("title", title)
	u.setScreenSize(width, height, scale)
	canvas.Call("focus")

	actualScale = canvas.Get("dataset").Get("ebitenActualScale").Int()
	return actualScale, nil
}

func (*userInterface) size() (width, height int) {
	a := canvas.Get("dataset").Get("ebitenActualScale").Int()
	if a == 0 {
		return
	}
	width = canvas.Get("width").Int() / a
	height = canvas.Get("height").Int() / a
	return
}

func (u *userInterface) setScreenSize(width, height, scale int) bool {
	w, h := u.size()
	s := canvas.Get("dataset").Get("ebitenScale").Int()
	if w == width && h == height && s == scale {
		return false
	}

	actualScale := scale * devicePixelRatio()
	canvas.Set("width", width*actualScale)
	canvas.Set("height", height*actualScale)
	canvas.Get("dataset").Set("ebitenScale", scale)
	canvas.Get("dataset").Set("ebitenActualScale", actualScale)
	canvasStyle := canvas.Get("style")

	cssWidth := width * scale
	cssHeight := height * scale
	canvasStyle.Set("width", strconv.Itoa(cssWidth)+"px")
	canvasStyle.Set("height", strconv.Itoa(cssHeight)+"px")
	// CSS calc requires space chars.
	canvasStyle.Set("left", "calc(50% - "+strconv.Itoa(cssWidth/2)+"px)")
	canvasStyle.Set("top", "calc(50% - "+strconv.Itoa(cssHeight/2)+"px)")
	return true
}
<commit_msg>ui: More precise space calc<commit_after>// Copyright 2015 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build js

package ui

import (
	"github.com/gopherjs/gopherjs/js"
	"strconv"
)

func ExecOnUIThread(f func()) {
	f()
}

func Start(width, height, scale int, title string) (actualScale int, err error) {
	return currentUI.start(width, height, scale, title)
}

func Terminate() {
	currentUI.terminate()
}

func DoEvents() error {
	return currentUI.doEvents()
}

func IsClosed() bool {
	return currentUI.isClosed()
}

func SwapBuffers() {
	currentUI.swapBuffers()
}

func SetScreenSize(width, height int) (bool, int) {
	scale := canvas.Get("dataset").Get("ebitenScale").Int()
	result := currentUI.setScreenSize(width, height, scale)
	actualScale := canvas.Get("dataset").Get("ebitenActualScale").Int()
	return result, actualScale
}

func SetScreenScale(scale int) (bool, int) {
	width, height := currentUI.size()
	result := currentUI.setScreenSize(width, height, scale)
	actualScale := canvas.Get("dataset").Get("ebitenActualScale").Int()
	return result, actualScale
}

var canvas js.Object

type userInterface struct{}

var currentUI = &userInterface{}

// NOTE: This returns true even when the browser is not active.
func shown() bool {
	return !js.Global.Get("document").Get("hidden").Bool()
}

func vsync() {
	ch := make(chan struct{})
	// TODO: In iOS8, this is called at every 1/30[sec] frame.
	// Can we use DOMHighResTimeStamp?
	js.Global.Get("window").Call("requestAnimationFrame", func() {
		close(ch)
	})
	<-ch
}

func (*userInterface) doEvents() error {
	vsync()
	for !shown() {
		vsync()
	}
	currentInput.UpdateGamepads()
	return nil
}

func (*userInterface) terminate() {
	// Do nothing.
}

func (*userInterface) isClosed() bool {
	return false
}

func (*userInterface) swapBuffers() {
	// Do nothing.
}

func Init() {
	// Do nothing in node.js.
	if js.Global.Get("require") != js.Undefined {
		return
	}

	doc := js.Global.Get("document")
	window := js.Global.Get("window")
	if doc.Get("body") == nil {
		ch := make(chan struct{})
		window.Call("addEventListener", "load", func() {
			close(ch)
		})
		<-ch
	}

	canvas = doc.Call("createElement", "canvas")
	canvas.Set("width", 16)
	canvas.Set("height", 16)
	doc.Get("body").Call("appendChild", canvas)

	htmlStyle := doc.Get("documentElement").Get("style")
	htmlStyle.Set("height", "100%")
	htmlStyle.Set("margin", "0")
	htmlStyle.Set("padding", "0")

	bodyStyle := doc.Get("body").Get("style")
	bodyStyle.Set("backgroundColor", "#000")
	bodyStyle.Set("position", "relative")
	bodyStyle.Set("height", "100%")
	bodyStyle.Set("margin", "0")
	bodyStyle.Set("padding", "0")
	doc.Get("body").Call("addEventListener", "click", func() {
		canvas.Call("focus")
	})

	canvasStyle := canvas.Get("style")
	canvasStyle.Set("position", "absolute")

	// Make the canvas focusable.
	canvas.Call("setAttribute", "tabindex", 1)
	canvas.Get("style").Set("outline", "none")

	// Keyboard
	canvas.Call("addEventListener", "keydown", func(e js.Object) {
		e.Call("preventDefault")
		code := e.Get("keyCode").Int()
		currentInput.KeyDown(code)
	})
	canvas.Call("addEventListener", "keyup", func(e js.Object) {
		e.Call("preventDefault")
		code := e.Get("keyCode").Int()
		currentInput.KeyUp(code)
	})

	// Mouse
	canvas.Call("addEventListener", "mousedown", func(e js.Object) {
		e.Call("preventDefault")
		button := e.Get("button").Int()
		currentInput.MouseDown(button)
		setMouseCursorFromEvent(e)
	})
	canvas.Call("addEventListener", "mouseup", func(e js.Object) {
		e.Call("preventDefault")
		button := e.Get("button").Int()
		currentInput.MouseUp(button)
		setMouseCursorFromEvent(e)
	})
	canvas.Call("addEventListener", "mousemove", func(e js.Object) {
		e.Call("preventDefault")
		setMouseCursorFromEvent(e)
	})
	canvas.Call("addEventListener", "contextmenu", func(e js.Object) {
		e.Call("preventDefault")
	})

	// Touch (emulating mouse events)
	// TODO: Create indimendent touch functions
	canvas.Call("addEventListener", "touchstart", func(e js.Object) {
		e.Call("preventDefault")
		currentInput.MouseDown(0)
		touches := e.Get("changedTouches")
		touch := touches.Index(0)
		setMouseCursorFromEvent(touch)
	})
	canvas.Call("addEventListener", "touchend", func(e js.Object) {
		e.Call("preventDefault")
		currentInput.MouseUp(0)
		touches := e.Get("changedTouches")
		touch := touches.Index(0)
		setMouseCursorFromEvent(touch)
	})
	canvas.Call("addEventListener", "touchmove", func(e js.Object) {
		e.Call("preventDefault")
		touches := e.Get("changedTouches")
		touch := touches.Index(0)
		setMouseCursorFromEvent(touch)
	})

	// Gamepad
	window.Call("addEventListener", "gamepadconnected", func(e js.Object) {
		// Do nothing.
	})
}

func setMouseCursorFromEvent(e js.Object) {
	scale := canvas.Get("dataset").Get("ebitenScale").Int()
	rect := canvas.Call("getBoundingClientRect")
	x, y := e.Get("clientX").Int(), e.Get("clientY").Int()
	x -= rect.Get("left").Int()
	y -= rect.Get("top").Int()
	currentInput.SetMouseCursor(x/scale, y/scale)
}

func devicePixelRatio() int {
	// TODO: What if ratio is not an integer but a float?
	ratio := js.Global.Get("window").Get("devicePixelRatio").Int()
	if ratio == 0 {
		ratio = 1
	}
	return ratio
}

func (u *userInterface) start(width, height, scale int, title string) (actualScale int, err error) {
	doc := js.Global.Get("document")
	doc.Set("title", title)
	u.setScreenSize(width, height, scale)
	canvas.Call("focus")

	actualScale = canvas.Get("dataset").Get("ebitenActualScale").Int()
	return actualScale, nil
}

func (*userInterface) size() (width, height int) {
	a := canvas.Get("dataset").Get("ebitenActualScale").Int()
	if a == 0 {
		return
	}
	width = canvas.Get("width").Int() / a
	height = canvas.Get("height").Int() / a
	return
}

func (u *userInterface) setScreenSize(width, height, scale int) bool {
	w, h := u.size()
	s := canvas.Get("dataset").Get("ebitenScale").Int()
	if w == width && h == height && s == scale {
		return false
	}

	actualScale := scale * devicePixelRatio()
	canvas.Set("width", width*actualScale)
	canvas.Set("height", height*actualScale)
	canvas.Get("dataset").Set("ebitenScale", scale)
	canvas.Get("dataset").Set("ebitenActualScale", actualScale)
	canvasStyle := canvas.Get("style")

	cssWidth := width * scale
	cssHeight := height * scale
	canvasStyle.Set("width", strconv.Itoa(cssWidth)+"px")
	canvasStyle.Set("height", strconv.Itoa(cssHeight)+"px")
	// CSS calc requires space chars.
	canvasStyle.Set("left", "calc((100% - "+strconv.Itoa(cssWidth)+"px) / 2)")
	canvasStyle.Set("top", "calc((100% - "+strconv.Itoa(cssHeight)+"px) / 2)")
	return true
}
<|endoftext|>


<commit_before>package main

import (
	"testing"
)

func TestItGetsRandomBehaviours(t *testing.T) {
	behaviour1 := new(behaviour)
	behaviour1.Frequency = 0.15
	behaviour1.Status = 123

	behaviour2 := new(behaviour)
	behaviour2.Frequency = 0.2
	behaviour2.Status = 456

	allBehaviours := []behaviour{*behaviour1, *behaviour2}

	result := getBehaviour(allBehaviours, fakeRandomiser{0.9})

	if result != nil {
		t.Error("There shouldnt have been a behaviour returned, but there was", result)
	}

	result = getBehaviour(allBehaviours, fakeRandomiser{0.13})

	if result.Status != behaviour1.Status {
		t.Error("It shouldve found behaviour 1", result, behaviour1)
	}

	result = getBehaviour(allBehaviours, fakeRandomiser{0.19})

	if result.Status != behaviour2.Status {
		t.Error("It should've found behaviour 2", result)
	}
}

func TestItReturnsAnErrorForBadYAML(t *testing.T) {
	yaml := "lol not yaml"

	_, err := monkeyConfigFromYAML([]byte(yaml))

	if err == nil {
		t.Error("Error was not returned for bad YAML")
	}
}

func TestItParsesYAMLIntoBehaviour(t *testing.T) {
	yaml := `
---
# Writes a different body 50% of the time
- body: "This is wrong :( "
  frequency: 0.5

# Delays initial writing of response by a second 20% of the time
- delay: 1000
  frequency: 0.2

# Returns a 404 30% of the time
- status: 404
  frequency: 0.3

# Write 10,000,000 garbage bytes 10% of the time
- garbage: 10000000
  frequency: 0.09
  `
	behaviours, _ := monkeyConfigFromYAML([]byte(yaml))

	if len(behaviours) != 4 {
		t.Error("It didnt load all the behaviours from YAML")
	}
}

type fakeRandomiser struct {
	value float64
}

func (f fakeRandomiser) getFloat() float64 {
	return f.value
}
<commit_msg>Test for config loading<commit_after>package main

import (
	"testing"
)

func TestItGetsRandomBehaviours(t *testing.T) {
	behaviour1 := new(behaviour)
	behaviour1.Frequency = 0.15
	behaviour1.Status = 123

	behaviour2 := new(behaviour)
	behaviour2.Frequency = 0.2
	behaviour2.Status = 456

	allBehaviours := []behaviour{*behaviour1, *behaviour2}

	result := getBehaviour(allBehaviours, fakeRandomiser{0.9})

	if result != nil {
		t.Error("There shouldnt have been a behaviour returned, but there was", result)
	}

	result = getBehaviour(allBehaviours, fakeRandomiser{0.13})

	if result.Status != behaviour1.Status {
		t.Error("It shouldve found behaviour 1", result, behaviour1)
	}

	result = getBehaviour(allBehaviours, fakeRandomiser{0.19})

	if result.Status != behaviour2.Status {
		t.Error("It should've found behaviour 2", result)
	}
}

func TestItLoadsFromFile(t *testing.T) {
	config := loadMonkeyConfig("examples/monkey-business.yaml")
	if len(config) != 4 {
		t.Error("It didnt load all the behaviours from YAML")
	}
}

func TestItReturnsAnErrorForBadYAML(t *testing.T) {
	yaml := "lol not yaml"

	_, err := monkeyConfigFromYAML([]byte(yaml))

	if err == nil {
		t.Error("Error was not returned for bad YAML")
	}
}

func TestItParsesYAMLIntoBehaviour(t *testing.T) {
	yaml := `
---
# Writes a different body 50% of the time
- body: "This is wrong :( "
  frequency: 0.5

# Delays initial writing of response by a second 20% of the time
- delay: 1000
  frequency: 0.2

# Returns a 404 30% of the time
- status: 404
  frequency: 0.3

# Write 10,000,000 garbage bytes 10% of the time
- garbage: 10000000
  frequency: 0.09
  `
	behaviours, _ := monkeyConfigFromYAML([]byte(yaml))

	if len(behaviours) != 4 {
		t.Error("It didnt load all the behaviours from YAML")
	}
}

type fakeRandomiser struct {
	value float64
}

func (f fakeRandomiser) getFloat() float64 {
	return f.value
}
<|endoftext|>


<commit_before>/*
run:
	docker run --name raphanus --rm --publish 8771:8771 msoap/raphanus
	docker run --name redis --rm --publish 6379:6379 redis
	docker run --name memcache --rm --publish 11211:11211 memcached
	make run-benchmark
*/
package raphanus_test

import (
	"os"
	"strconv"
	"testing"

	"github.com/boltdb/bolt"
	"github.com/bradfitz/gomemcache/memcache"
	"github.com/mediocregopher/radix.v2/redis"
	"github.com/msoap/raphanus"
	"github.com/msoap/raphanus/client"
	"github.com/stretchr/testify/require"
)

func Benchmark_raphanusServer(b *testing.B) {
	raph := raphanusclient.New(raphanusclient.Cfg{Address: "http://localhost:8771"})

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err := raph.SetStr("key_"+strI, "bar_"+strI, 0)
		require.NoError(b, err)

		newVal, err := raph.GetStr("key_" + strI)
		require.NoError(b, err)
		require.Equal(b, newVal, "bar_"+strI)
	}
}

func Benchmark_raphanusEmbed(b *testing.B) {
	raph := raphanus.New("", 0)

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err := raph.SetStr("key_"+strI, "bar_"+strI, 0)
		require.NoError(b, err)

		newVal, err := raph.GetStr("key_" + strI)
		require.NoError(b, err)
		require.Equal(b, newVal, "bar_"+strI)
	}
}

func Benchmark_boltdb(b *testing.B) {
	db, err := bolt.Open("bolt_bench_tmp.db", 0600, nil)
	require.NoError(b, err)
	defer func() {
		require.NoError(b, db.Close())
		require.NoError(b, os.Remove("bolt_bench_tmp.db"))
	}()

	_ = db.Update(func(tx *bolt.Tx) error {
		if _, err := tx.CreateBucket([]byte("bucket")); err != nil {
			return err
		}
		return nil
	})

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)

		err := db.Update(func(tx *bolt.Tx) error {
			return tx.Bucket([]byte("bucket")).Put([]byte("key_"+strI), []byte("bar_"+strI))
		})
		require.NoError(b, err)

		err = db.View(func(tx *bolt.Tx) error {
			newVal := tx.Bucket([]byte("bucket")).Get([]byte("key_" + strI))
			require.Equal(b, string(newVal), "bar_"+strI)
			return nil
		})
		require.NoError(b, err)
	}
}

func Benchmark_redis(b *testing.B) {
	redisCli, err := redis.Dial("tcp", "localhost:6379")
	require.NoError(b, err)

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err = redisCli.Cmd("SET", "key_"+strI, "bar_"+strI).Err
		require.NoError(b, err)

		newVal, err := redisCli.Cmd("GET", "key_"+strI).Str()
		require.NoError(b, err)
		require.Equal(b, newVal, "bar_"+strI)
	}
}

func Benchmark_memcache(b *testing.B) {
	mc := memcache.New("localhost:11211")

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err := mc.Set(&memcache.Item{Key: "key_" + strI, Value: []byte("bar_" + strI)})
		require.NoError(b, err)

		item, err := mc.Get("key_" + strI)
		require.NoError(b, err)
		newVal := item.Value
		require.Equal(b, string(newVal), "bar_"+strI)
	}
}

func Benchmark_raphanusServerTTL(b *testing.B) {
	raph := raphanusclient.New(raphanusclient.Cfg{Address: "http://localhost:8771"})

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err := raph.SetStr("key_"+strI, "bar_"+strI, 2)
		require.NoError(b, err)

		newVal, err := raph.GetStr("key_" + strI)
		if err != nil {
			// skip deleted keys error
			continue
		}
		require.Equal(b, newVal, "bar_"+strI)
	}
}

func Benchmark_raphanusEmbedTTL(b *testing.B) {
	raph := raphanus.New("", 0)

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err := raph.SetStr("key_"+strI, "bar_"+strI, 2)
		require.NoError(b, err)

		newVal, err := raph.GetStr("key_" + strI)
		require.NoError(b, err)
		require.Equal(b, newVal, "bar_"+strI)
	}
}

func Benchmark_redisTTL(b *testing.B) {
	redisCli, err := redis.Dial("tcp", "localhost:6379")
	require.NoError(b, err)

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err = redisCli.Cmd("SET", "key_"+strI, "bar_"+strI, "EX", 2).Err
		require.NoError(b, err)

		newVal, err := redisCli.Cmd("GET", "key_"+strI).Str()
		require.NoError(b, err)
		require.Equal(b, newVal, "bar_"+strI)
	}
}

func Benchmark_memcacheTTL(b *testing.B) {
	mc := memcache.New("localhost:11211")

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err := mc.Set(&memcache.Item{Key: "key_" + strI, Value: []byte("bar_" + strI), Expiration: 2})
		require.NoError(b, err)

		item, err := mc.Get("key_" + strI)
		require.NoError(b, err)
		newVal := item.Value
		require.Equal(b, string(newVal), "bar_"+strI)
	}
}
<commit_msg>Update bolt db benchmark test<commit_after>/*
run:
	docker run --name raphanus --rm --publish 8771:8771 msoap/raphanus
	docker run --name redis --rm --publish 6379:6379 redis
	docker run --name memcache --rm --publish 11211:11211 memcached
	make run-benchmark
*/
package raphanus_test

import (
	"os"
	"strconv"
	"testing"

	"github.com/boltdb/bolt"
	"github.com/bradfitz/gomemcache/memcache"
	"github.com/mediocregopher/radix.v2/redis"
	"github.com/msoap/raphanus"
	"github.com/msoap/raphanus/client"
	"github.com/stretchr/testify/require"
)

func Benchmark_raphanusServer(b *testing.B) {
	raph := raphanusclient.New(raphanusclient.Cfg{Address: "http://localhost:8771"})

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err := raph.SetStr("key_"+strI, "bar_"+strI, 0)
		require.NoError(b, err)

		newVal, err := raph.GetStr("key_" + strI)
		require.NoError(b, err)
		require.Equal(b, newVal, "bar_"+strI)
	}
}

func Benchmark_raphanusEmbed(b *testing.B) {
	raph := raphanus.New("", 0)

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err := raph.SetStr("key_"+strI, "bar_"+strI, 0)
		require.NoError(b, err)

		newVal, err := raph.GetStr("key_" + strI)
		require.NoError(b, err)
		require.Equal(b, newVal, "bar_"+strI)
	}
}

func Benchmark_boltdb(b *testing.B) {
	db, err := bolt.Open("bolt_bench_tmp.db", 0600, nil)
	require.NoError(b, err)
	defer func() {
		require.NoError(b, db.Close())
		require.NoError(b, os.Remove("bolt_bench_tmp.db"))
	}()

	err = db.Update(func(tx *bolt.Tx) error {
		if _, err := tx.CreateBucketIfNotExists([]byte("bucket")); err != nil {
			return err
		}
		return nil
	})
	require.NoError(b, err)

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)

		err := db.Update(func(tx *bolt.Tx) error {
			return tx.Bucket([]byte("bucket")).Put([]byte("key_"+strI), []byte("bar_"+strI))
		})
		require.NoError(b, err)

		err = db.View(func(tx *bolt.Tx) error {
			newVal := tx.Bucket([]byte("bucket")).Get([]byte("key_" + strI))
			require.Equal(b, string(newVal), "bar_"+strI)
			return nil
		})
		require.NoError(b, err)
	}
}

func Benchmark_redis(b *testing.B) {
	redisCli, err := redis.Dial("tcp", "localhost:6379")
	require.NoError(b, err)

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err = redisCli.Cmd("SET", "key_"+strI, "bar_"+strI).Err
		require.NoError(b, err)

		newVal, err := redisCli.Cmd("GET", "key_"+strI).Str()
		require.NoError(b, err)
		require.Equal(b, newVal, "bar_"+strI)
	}
}

func Benchmark_memcache(b *testing.B) {
	mc := memcache.New("localhost:11211")

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err := mc.Set(&memcache.Item{Key: "key_" + strI, Value: []byte("bar_" + strI)})
		require.NoError(b, err)

		item, err := mc.Get("key_" + strI)
		require.NoError(b, err)
		newVal := item.Value
		require.Equal(b, string(newVal), "bar_"+strI)
	}
}

func Benchmark_raphanusServerTTL(b *testing.B) {
	raph := raphanusclient.New(raphanusclient.Cfg{Address: "http://localhost:8771"})

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err := raph.SetStr("key_"+strI, "bar_"+strI, 2)
		require.NoError(b, err)

		newVal, err := raph.GetStr("key_" + strI)
		if err != nil {
			// skip deleted keys error
			continue
		}
		require.Equal(b, newVal, "bar_"+strI)
	}
}

func Benchmark_raphanusEmbedTTL(b *testing.B) {
	raph := raphanus.New("", 0)

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err := raph.SetStr("key_"+strI, "bar_"+strI, 2)
		require.NoError(b, err)

		newVal, err := raph.GetStr("key_" + strI)
		require.NoError(b, err)
		require.Equal(b, newVal, "bar_"+strI)
	}
}

func Benchmark_redisTTL(b *testing.B) {
	redisCli, err := redis.Dial("tcp", "localhost:6379")
	require.NoError(b, err)

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err = redisCli.Cmd("SET", "key_"+strI, "bar_"+strI, "EX", 2).Err
		require.NoError(b, err)

		newVal, err := redisCli.Cmd("GET", "key_"+strI).Str()
		require.NoError(b, err)
		require.Equal(b, newVal, "bar_"+strI)
	}
}

func Benchmark_memcacheTTL(b *testing.B) {
	mc := memcache.New("localhost:11211")

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err := mc.Set(&memcache.Item{Key: "key_" + strI, Value: []byte("bar_" + strI), Expiration: 2})
		require.NoError(b, err)

		item, err := mc.Get("key_" + strI)
		require.NoError(b, err)
		newVal := item.Value
		require.Equal(b, string(newVal), "bar_"+strI)
	}
}
<|endoftext|>


<commit_before>/*
run:
	docker run --name raphanus --rm --publish 8771:8771 msoap/raphanus
	docker run --name redis --rm --publish 6379:6379 redis
	docker run --name memcache --rm --publish 11211:11211 memcached
	make run-benchmark
*/
package raphanus_test

import (
	"os"
	"strconv"
	"testing"

	"github.com/boltdb/bolt"
	"github.com/bradfitz/gomemcache/memcache"
	"github.com/mediocregopher/radix.v2/redis"
	"github.com/msoap/raphanus"
	"github.com/msoap/raphanus/client"
	"github.com/stretchr/testify/require"
)

func Benchmark_raphanusServer(b *testing.B) {
	raph := raphanusclient.New(raphanusclient.Cfg{Address: "http://localhost:8771"})

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err := raph.SetStr("key_"+strI, "bar_"+strI, 0)
		require.NoError(b, err)

		newVal, err := raph.GetStr("key_" + strI)
		require.NoError(b, err)
		require.Equal(b, newVal, "bar_"+strI)
	}
}

func Benchmark_raphanusEmbed(b *testing.B) {
	raph := raphanus.New("", 0)

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err := raph.SetStr("key_"+strI, "bar_"+strI, 0)
		require.NoError(b, err)

		newVal, err := raph.GetStr("key_" + strI)
		require.NoError(b, err)
		require.Equal(b, newVal, "bar_"+strI)
	}
}

func Benchmark_boltdb(b *testing.B) {
	db, err := bolt.Open("bolt_bench_tmp.db", 0600, nil)
	require.NoError(b, err)
	defer func() {
		require.NoError(b, db.Close())
		require.NoError(b, os.Remove("bolt_bench_tmp.db"))
	}()

	err = db.Update(func(tx *bolt.Tx) error {
		if _, err := tx.CreateBucketIfNotExists([]byte("bucket")); err != nil {
			return err
		}
		return nil
	})
	require.NoError(b, err)

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)

		err := db.Update(func(tx *bolt.Tx) error {
			return tx.Bucket([]byte("bucket")).Put([]byte("key_"+strI), []byte("bar_"+strI))
		})
		require.NoError(b, err)

		err = db.View(func(tx *bolt.Tx) error {
			newVal := tx.Bucket([]byte("bucket")).Get([]byte("key_" + strI))
			require.Equal(b, string(newVal), "bar_"+strI)
			return nil
		})
		require.NoError(b, err)
	}
}

func Benchmark_redis(b *testing.B) {
	redisCli, err := redis.Dial("tcp", "localhost:6379")
	require.NoError(b, err)

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err = redisCli.Cmd("SET", "key_"+strI, "bar_"+strI).Err
		require.NoError(b, err)

		newVal, err := redisCli.Cmd("GET", "key_"+strI).Str()
		require.NoError(b, err)
		require.Equal(b, newVal, "bar_"+strI)
	}
}

func Benchmark_memcache(b *testing.B) {
	mc := memcache.New("localhost:11211")

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err := mc.Set(&memcache.Item{Key: "key_" + strI, Value: []byte("bar_" + strI)})
		require.NoError(b, err)

		item, err := mc.Get("key_" + strI)
		require.NoError(b, err)
		newVal := item.Value
		require.Equal(b, string(newVal), "bar_"+strI)
	}
}

func Benchmark_raphanusServerTTL(b *testing.B) {
	raph := raphanusclient.New(raphanusclient.Cfg{Address: "http://localhost:8771"})

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err := raph.SetStr("key_"+strI, "bar_"+strI, 2)
		require.NoError(b, err)

		newVal, err := raph.GetStr("key_" + strI)
		if err != nil {
			// skip deleted keys error
			continue
		}
		require.Equal(b, newVal, "bar_"+strI)
	}
}

func Benchmark_raphanusEmbedTTL(b *testing.B) {
	raph := raphanus.New("", 0)

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err := raph.SetStr("key_"+strI, "bar_"+strI, 2)
		require.NoError(b, err)

		newVal, err := raph.GetStr("key_" + strI)
		require.NoError(b, err)
		require.Equal(b, newVal, "bar_"+strI)
	}
}

func Benchmark_redisTTL(b *testing.B) {
	redisCli, err := redis.Dial("tcp", "localhost:6379")
	require.NoError(b, err)

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err = redisCli.Cmd("SET", "key_"+strI, "bar_"+strI, "EX", 2).Err
		require.NoError(b, err)

		newVal, err := redisCli.Cmd("GET", "key_"+strI).Str()
		require.NoError(b, err)
		require.Equal(b, newVal, "bar_"+strI)
	}
}

func Benchmark_memcacheTTL(b *testing.B) {
	mc := memcache.New("localhost:11211")

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err := mc.Set(&memcache.Item{Key: "key_" + strI, Value: []byte("bar_" + strI), Expiration: 2})
		require.NoError(b, err)

		item, err := mc.Get("key_" + strI)
		require.NoError(b, err)
		newVal := item.Value
		require.Equal(b, string(newVal), "bar_"+strI)
	}
}
<commit_msg>Fixed linter warnings<commit_after>/*
run:
	docker run --name raphanus --rm --publish 8771:8771 msoap/raphanus
	docker run --name redis --rm --publish 6379:6379 redis
	docker run --name memcache --rm --publish 11211:11211 memcached
	make run-benchmark
*/
package raphanus_test

import (
	"os"
	"strconv"
	"testing"

	"github.com/boltdb/bolt"
	"github.com/bradfitz/gomemcache/memcache"
	"github.com/mediocregopher/radix.v2/redis"
	"github.com/msoap/raphanus"
	"github.com/msoap/raphanus/client"
	"github.com/stretchr/testify/require"
)

func Benchmark_raphanusServer(b *testing.B) {
	raph := raphanusclient.New(raphanusclient.Cfg{Address: "http://localhost:8771"})

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err := raph.SetStr("key_"+strI, "bar_"+strI, 0)
		require.NoError(b, err)

		newVal, err := raph.GetStr("key_" + strI)
		require.NoError(b, err)
		require.Equal(b, newVal, "bar_"+strI)
	}
}

func Benchmark_raphanusEmbed(b *testing.B) {
	raph := raphanus.New("", 0)

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err := raph.SetStr("key_"+strI, "bar_"+strI, 0)
		require.NoError(b, err)

		newVal, err := raph.GetStr("key_" + strI)
		require.NoError(b, err)
		require.Equal(b, newVal, "bar_"+strI)
	}
}

func Benchmark_boltdb(b *testing.B) {
	db, err := bolt.Open("bolt_bench_tmp.db", 0600, nil)
	require.NoError(b, err)
	defer func() {
		require.NoError(b, db.Close())
		require.NoError(b, os.Remove("bolt_bench_tmp.db"))
	}()

	err = db.Update(func(tx *bolt.Tx) error {
		_, err = tx.CreateBucketIfNotExists([]byte("bucket"))
		return err
	})
	require.NoError(b, err)

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)

		err := db.Update(func(tx *bolt.Tx) error {
			return tx.Bucket([]byte("bucket")).Put([]byte("key_"+strI), []byte("bar_"+strI))
		})
		require.NoError(b, err)

		err = db.View(func(tx *bolt.Tx) error {
			newVal := tx.Bucket([]byte("bucket")).Get([]byte("key_" + strI))
			require.Equal(b, string(newVal), "bar_"+strI)
			return nil
		})
		require.NoError(b, err)
	}
}

func Benchmark_redis(b *testing.B) {
	redisCli, err := redis.Dial("tcp", "localhost:6379")
	require.NoError(b, err)

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err = redisCli.Cmd("SET", "key_"+strI, "bar_"+strI).Err
		require.NoError(b, err)

		newVal, err := redisCli.Cmd("GET", "key_"+strI).Str()
		require.NoError(b, err)
		require.Equal(b, newVal, "bar_"+strI)
	}
}

func Benchmark_memcache(b *testing.B) {
	mc := memcache.New("localhost:11211")

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err := mc.Set(&memcache.Item{Key: "key_" + strI, Value: []byte("bar_" + strI)})
		require.NoError(b, err)

		item, err := mc.Get("key_" + strI)
		require.NoError(b, err)
		newVal := item.Value
		require.Equal(b, string(newVal), "bar_"+strI)
	}
}

func Benchmark_raphanusServerTTL(b *testing.B) {
	raph := raphanusclient.New(raphanusclient.Cfg{Address: "http://localhost:8771"})

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err := raph.SetStr("key_"+strI, "bar_"+strI, 2)
		require.NoError(b, err)

		newVal, err := raph.GetStr("key_" + strI)
		if err != nil {
			// skip deleted keys error
			continue
		}
		require.Equal(b, newVal, "bar_"+strI)
	}
}

func Benchmark_raphanusEmbedTTL(b *testing.B) {
	raph := raphanus.New("", 0)

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err := raph.SetStr("key_"+strI, "bar_"+strI, 2)
		require.NoError(b, err)

		newVal, err := raph.GetStr("key_" + strI)
		require.NoError(b, err)
		require.Equal(b, newVal, "bar_"+strI)
	}
}

func Benchmark_redisTTL(b *testing.B) {
	redisCli, err := redis.Dial("tcp", "localhost:6379")
	require.NoError(b, err)

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err = redisCli.Cmd("SET", "key_"+strI, "bar_"+strI, "EX", 2).Err
		require.NoError(b, err)

		newVal, err := redisCli.Cmd("GET", "key_"+strI).Str()
		require.NoError(b, err)
		require.Equal(b, newVal, "bar_"+strI)
	}
}

func Benchmark_memcacheTTL(b *testing.B) {
	mc := memcache.New("localhost:11211")

	for i := 0; i < b.N; i++ {
		strI := strconv.Itoa(i)
		err := mc.Set(&memcache.Item{Key: "key_" + strI, Value: []byte("bar_" + strI), Expiration: 2})
		require.NoError(b, err)

		item, err := mc.Get("key_" + strI)
		require.NoError(b, err)
		newVal := item.Value
		require.Equal(b, string(newVal), "bar_"+strI)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/HouzuoGuo/tiedot/db"
	"math/rand"
	"os"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"
)

var benchTestSize = 50000

func averageTest(name string, fun func()) {
	numThreads := runtime.GOMAXPROCS(-1)
	wp := new(sync.WaitGroup)
	iter := float64(benchTestSize)
	start := float64(time.Now().UTC().UnixNano())
	// Run function across multiple goroutines
	for i := 0; i < benchTestSize; i += benchTestSize / numThreads {
		wp.Add(1)
		go func() {
			defer wp.Done()
			for j := 0; j < benchTestSize/numThreads; j++ {
				fun()
			}
		}()
	}
	wp.Wait()
	end := float64(time.Now().UTC().UnixNano())
	fmt.Printf("%s %d: %d ns/iter, %d iter/sec\n", name, int(benchTestSize), int((end-start)/iter), int(1000000000/((end-start)/iter)))
}

// benchmark(1) written in test case style
func TestBenchmark1(t *testing.T) {
	runtime.GOMAXPROCS(runtime.NumCPU())
	rand.Seed(time.Now().UnixNano())

	ids := make([]int, 0, benchTestSize)
	// Prepare a collection with two indexes
	tmp := "/tmp/tiedot_test_bench1"
	os.RemoveAll(tmp)
	defer os.RemoveAll(tmp)
	benchDB, col := mkTmpDBAndCol(tmp, "tmp")
	defer benchDB.Close()
	col.Index([]string{"nested", "nested", "str"})
	col.Index([]string{"nested", "nested", "int"})
	col.Index([]string{"nested", "nested", "float"})
	col.Index([]string{"strs"})
	col.Index([]string{"ints"})
	col.Index([]string{"floats"})

	// Benchmark document insert
	average("insert", func() {
		if _, err := col.Insert(sampleDoc()); err != nil {
			panic(err)
		}
	})

	// Collect all document IDs and benchmark document read
	col.ForEachDoc(func(id int, _ []byte) bool {
		ids = append(ids, id)
		return true
	})
	average("read", func() {
		doc, err := col.Read(ids[rand.Intn(benchTestSize)])
		if doc == nil || err != nil {
			panic(err)
		}
	})

	// Benchmark lookup query (two attributes)
	average("lookup", func() {
		result := make(map[int]struct{})
		if err := db.EvalQuery(sampleQuery(), col, &result); err != nil {
			panic(err)
		}
	})

	// Benchmark document update
	average("update", func() {
		if err := col.Update(ids[rand.Intn(benchTestSize)], sampleDoc()); err != nil && !strings.Contains(err.Error(), "locked") {
			panic(err)
		}
	})

	// Benchmark document delete
	var delCount int64
	average("delete", func() {
		if err := col.Delete(ids[rand.Intn(benchTestSize)]); err == nil {
			atomic.AddInt64(&delCount, 1)
		}
	})
	if delCount < int64(benchTestSize/2) {
		t.Fatal("Did not delete enough")
	}
}

// benchmark2 written in test case style
func TestBenchmark2(t *testing.T) {
	runtime.GOMAXPROCS(runtime.NumCPU())
	rand.Seed(time.Now().UnixNano())

	docs := make([]int, 0, benchTestSize*2+1000)
	wp := new(sync.WaitGroup)
	numThreads := runtime.GOMAXPROCS(-1)
	// There are goroutines doing document operations: insert, read, query, update, delete
	wp.Add(5 * numThreads)
	// And one more changing schema and stuff
	wp.Add(1)

	// Prepare a collection with two indexes
	tmp := "/tmp/tiedot_test_bench2"
	os.RemoveAll(tmp)
	defer os.RemoveAll(tmp)
	benchdb, col := mkTmpDBAndCol(tmp, "tmp")
	defer benchdb.Close()
	col.Index([]string{"nested", "nested", "str"})
	col.Index([]string{"nested", "nested", "int"})
	col.Index([]string{"nested", "nested", "float"})
	col.Index([]string{"strs"})
	col.Index([]string{"ints"})
	col.Index([]string{"floats"})

	// Insert 1000 documents to make a start
	for j := 0; j < 1000; j++ {
		if newID, err := col.Insert(sampleDoc()); err == nil {
			docs = append(docs, newID)
		} else {
			panic(err)
		}
	}
	start := float64(time.Now().UTC().UnixNano())

	// Insert benchTestSize * 2 documents
	for i := 0; i < numThreads; i++ {
		go func(i int) {
			fmt.Printf("Insert thread %d starting\n", i)
			defer wp.Done()
			for j := 0; j < benchTestSize/numThreads*2; j++ {
				if newID, err := col.Insert(sampleDoc()); err == nil {
					docs = append(docs, newID)
				} else {
					panic(err)
				}
			}
			fmt.Printf("Insert thread %d completed\n", i)
		}(i)
	}

	// Read benchTestSize * 2 documents
	var readCount int64
	for i := 0; i < numThreads; i++ {
		go func(i int) {
			fmt.Printf("Read thread %d starting\n", i)
			defer wp.Done()
			for j := 0; j < benchTestSize/numThreads*2; j++ {
				if _, err := col.Read(docs[rand.Intn(len(docs))]); err == nil {
					atomic.AddInt64(&readCount, 1)
				}
			}
			fmt.Printf("Read thread %d completed\n", i)
		}(i)
	}

	// Query benchTestSize times (lookup on two attributes)
	for i := 0; i < numThreads; i++ {
		go func(i int) {
			fmt.Printf("Query thread %d starting\n", i)
			defer wp.Done()
			var err error
			for j := 0; j < benchTestSize/numThreads; j++ {
				result := make(map[int]struct{})
				if err = db.EvalQuery(sampleQuery(), col, &result); err != nil {
					panic(err)
				}
			}
			fmt.Printf("Query thread %d completed\n", i)
		}(i)
	}

	// Update benchTestSize documents
	var updateCount int64
	for i := 0; i < numThreads; i++ {
		go func(i int) {
			fmt.Printf("Update thread %d starting\n", i)
			defer wp.Done()
			for j := 0; j < benchTestSize/numThreads; j++ {
				if err := col.Update(docs[rand.Intn(len(docs))], sampleDoc()); err == nil {
					atomic.AddInt64(&updateCount, 1)
				}
			}
			fmt.Printf("Update thread %d completed\n", i)
		}(i)
	}

	// Delete benchTestSize documents
	var delCount int64
	for i := 0; i < numThreads; i++ {
		go func(i int) {
			fmt.Printf("Delete thread %d starting\n", i)
			defer wp.Done()
			for j := 0; j < benchTestSize/numThreads; j++ {
				if err := col.Delete(docs[rand.Intn(len(docs))]); err == nil {
					atomic.AddInt64(&delCount, 1)
				}
			}
			fmt.Printf("Delete thread %d completed\n", i)
		}(i)
	}

	// This one does a bunch of schema-changing stuff, testing the engine while document operations are busy
	go func() {
		time.Sleep(500 * time.Millisecond)
		if err := benchdb.Create("foo"); err != nil {
			panic(err)
		} else if err := benchdb.Rename("foo", "bar"); err != nil {
			panic(err)
		} else if err := benchdb.Truncate("bar"); err != nil {
			panic(err)
		} else if err := benchdb.Scrub("bar"); err != nil {
			panic(err)
		} else if benchdb.Use("bar") == nil {
			panic("Missing collection")
		}
		for _, colName := range benchdb.AllCols() {
			if colName != "bar" && colName != "tmp" {
				panic("Wrong collections in benchmark db")
			}
		}
		os.RemoveAll("/tmp/tiedot_test_bench2_dump")
		defer os.RemoveAll("/tmp/tiedot_test_bench2_dump")
		if err := benchdb.Dump("/tmp/tiedot_test_bench2_dump"); err != nil {
			panic(err)
		} else if err := benchdb.Drop("bar"); err != nil {
			panic(err)
		}
		defer wp.Done()
	}()

	// Wait for all goroutines to finish, then print summary
	wp.Wait()
	end := float64(time.Now().UTC().UnixNano())
	fmt.Printf("Total operations %d: %d ns/iter, %d iter/sec\n", benchTestSize*7, int((end-start)/float64(benchTestSize)/7), int(1000000000/((end-start)/float64(benchTestSize)/7)))
	fmt.Printf("Read %d documents\n", readCount)
	fmt.Printf("Updated %d documents\n", updateCount)
	fmt.Printf("Deleted %d documents\n", delCount)
	if readCount < int64(benchTestSize/3) {
		t.Fatal("Did not read enough documents")
	}
	if updateCount < int64(benchTestSize/8) {
		t.Fatal("Did not update enough documents")
	}
	if delCount < int64(benchTestSize/8) {
		t.Fatal("Did not delete enough documents")
	}
}
<commit_msg>disable benchmark test case for travis<commit_after>package main

import (
	"fmt"
	"github.com/HouzuoGuo/tiedot/db"
	"math/rand"
	"os"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"
)

var benchTestSize = 50000

func averageTest(name string, fun func()) {
	numThreads := runtime.GOMAXPROCS(-1)
	wp := new(sync.WaitGroup)
	iter := float64(benchTestSize)
	start := float64(time.Now().UTC().UnixNano())
	// Run function across multiple goroutines
	for i := 0; i < benchTestSize; i += benchTestSize / numThreads {
		wp.Add(1)
		go func() {
			defer wp.Done()
			for j := 0; j < benchTestSize/numThreads; j++ {
				fun()
			}
		}()
	}
	wp.Wait()
	end := float64(time.Now().UTC().UnixNano())
	fmt.Printf("%s %d: %d ns/iter, %d iter/sec\n", name, int(benchTestSize), int((end-start)/iter), int(1000000000/((end-start)/iter)))
}

// benchmark(1) written in test case style
func TestBenchmark1(t *testing.T) {
	return
	runtime.GOMAXPROCS(runtime.NumCPU())
	rand.Seed(time.Now().UnixNano())

	ids := make([]int, 0, benchTestSize)
	// Prepare a collection with two indexes
	tmp := "/tmp/tiedot_test_bench1"
	os.RemoveAll(tmp)
	defer os.RemoveAll(tmp)
	benchDB, col := mkTmpDBAndCol(tmp, "tmp")
	defer benchDB.Close()
	col.Index([]string{"nested", "nested", "str"})
	col.Index([]string{"nested", "nested", "int"})
	col.Index([]string{"nested", "nested", "float"})
	col.Index([]string{"strs"})
	col.Index([]string{"ints"})
	col.Index([]string{"floats"})

	// Benchmark document insert
	average("insert", func() {
		if _, err := col.Insert(sampleDoc()); err != nil {
			panic(err)
		}
	})

	// Collect all document IDs and benchmark document read
	col.ForEachDoc(func(id int, _ []byte) bool {
		ids = append(ids, id)
		return true
	})
	average("read", func() {
		doc, err := col.Read(ids[rand.Intn(benchTestSize)])
		if doc == nil || err != nil {
			panic(err)
		}
	})

	// Benchmark lookup query (two attributes)
	average("lookup", func() {
		result := make(map[int]struct{})
		if err := db.EvalQuery(sampleQuery(), col, &result); err != nil {
			panic(err)
		}
	})

	// Benchmark document update
	average("update", func() {
		if err := col.Update(ids[rand.Intn(benchTestSize)], sampleDoc()); err != nil && !strings.Contains(err.Error(), "locked") {
			panic(err)
		}
	})

	// Benchmark document delete
	var delCount int64
	average("delete", func() {
		if err := col.Delete(ids[rand.Intn(benchTestSize)]); err == nil {
			atomic.AddInt64(&delCount, 1)
		}
	})
	if delCount < int64(benchTestSize/2) {
		t.Fatal("Did not delete enough")
	}
}

// benchmark2 written in test case style
func TestBenchmark2(t *testing.T) {
	return
	runtime.GOMAXPROCS(runtime.NumCPU())
	rand.Seed(time.Now().UnixNano())

	docs := make([]int, 0, benchTestSize*2+1000)
	wp := new(sync.WaitGroup)
	numThreads := runtime.GOMAXPROCS(-1)
	// There are goroutines doing document operations: insert, read, query, update, delete
	wp.Add(5 * numThreads)
	// And one more changing schema and stuff
	wp.Add(1)

	// Prepare a collection with two indexes
	tmp := "/tmp/tiedot_test_bench2"
	os.RemoveAll(tmp)
	defer os.RemoveAll(tmp)
	benchdb, col := mkTmpDBAndCol(tmp, "tmp")
	defer benchdb.Close()
	col.Index([]string{"nested", "nested", "str"})
	col.Index([]string{"nested", "nested", "int"})
	col.Index([]string{"nested", "nested", "float"})
	col.Index([]string{"strs"})
	col.Index([]string{"ints"})
	col.Index([]string{"floats"})

	// Insert 1000 documents to make a start
	for j := 0; j < 1000; j++ {
		if newID, err := col.Insert(sampleDoc()); err == nil {
			docs = append(docs, newID)
		} else {
			panic(err)
		}
	}
	start := float64(time.Now().UTC().UnixNano())

	// Insert benchTestSize * 2 documents
	for i := 0; i < numThreads; i++ {
		go func(i int) {
			fmt.Printf("Insert thread %d starting\n", i)
			defer wp.Done()
			for j := 0; j < benchTestSize/numThreads*2; j++ {
				if newID, err := col.Insert(sampleDoc()); err == nil {
					docs = append(docs, newID)
				} else {
					panic(err)
				}
			}
			fmt.Printf("Insert thread %d completed\n", i)
		}(i)
	}

	// Read benchTestSize * 2 documents
	var readCount int64
	for i := 0; i < numThreads; i++ {
		go func(i int) {
			fmt.Printf("Read thread %d starting\n", i)
			defer wp.Done()
			for j := 0; j < benchTestSize/numThreads*2; j++ {
				if _, err := col.Read(docs[rand.Intn(len(docs))]); err == nil {
					atomic.AddInt64(&readCount, 1)
				}
			}
			fmt.Printf("Read thread %d completed\n", i)
		}(i)
	}

	// Query benchTestSize times (lookup on two attributes)
	for i := 0; i < numThreads; i++ {
		go func(i int) {
			fmt.Printf("Query thread %d starting\n", i)
			defer wp.Done()
			var err error
			for j := 0; j < benchTestSize/numThreads; j++ {
				result := make(map[int]struct{})
				if err = db.EvalQuery(sampleQuery(), col, &result); err != nil {
					panic(err)
				}
			}
			fmt.Printf("Query thread %d completed\n", i)
		}(i)
	}

	// Update benchTestSize documents
	var updateCount int64
	for i := 0; i < numThreads; i++ {
		go func(i int) {
			fmt.Printf("Update thread %d starting\n", i)
			defer wp.Done()
			for j := 0; j < benchTestSize/numThreads; j++ {
				if err := col.Update(docs[rand.Intn(len(docs))], sampleDoc()); err == nil {
					atomic.AddInt64(&updateCount, 1)
				}
			}
			fmt.Printf("Update thread %d completed\n", i)
		}(i)
	}

	// Delete benchTestSize documents
	var delCount int64
	for i := 0; i < numThreads; i++ {
		go func(i int) {
			fmt.Printf("Delete thread %d starting\n", i)
			defer wp.Done()
			for j := 0; j < benchTestSize/numThreads; j++ {
				if err := col.Delete(docs[rand.Intn(len(docs))]); err == nil {
					atomic.AddInt64(&delCount, 1)
				}
			}
			fmt.Printf("Delete thread %d completed\n", i)
		}(i)
	}

	// This one does a bunch of schema-changing stuff, testing the engine while document operations are busy
	go func() {
		time.Sleep(500 * time.Millisecond)
		if err := benchdb.Create("foo"); err != nil {
			panic(err)
		} else if err := benchdb.Rename("foo", "bar"); err != nil {
			panic(err)
		} else if err := benchdb.Truncate("bar"); err != nil {
			panic(err)
		} else if err := benchdb.Scrub("bar"); err != nil {
			panic(err)
		} else if benchdb.Use("bar") == nil {
			panic("Missing collection")
		}
		for _, colName := range benchdb.AllCols() {
			if colName != "bar" && colName != "tmp" {
				panic("Wrong collections in benchmark db")
			}
		}
		os.RemoveAll("/tmp/tiedot_test_bench2_dump")
		defer os.RemoveAll("/tmp/tiedot_test_bench2_dump")
		if err := benchdb.Dump("/tmp/tiedot_test_bench2_dump"); err != nil {
			panic(err)
		} else if err := benchdb.Drop("bar"); err != nil {
			panic(err)
		}
		defer wp.Done()
	}()

	// Wait for all goroutines to finish, then print summary
	wp.Wait()
	end := float64(time.Now().UTC().UnixNano())
	fmt.Printf("Total operations %d: %d ns/iter, %d iter/sec\n", benchTestSize*7, int((end-start)/float64(benchTestSize)/7), int(1000000000/((end-start)/float64(benchTestSize)/7)))
	fmt.Printf("Read %d documents\n", readCount)
	fmt.Printf("Updated %d documents\n", updateCount)
	fmt.Printf("Deleted %d documents\n", delCount)
	if readCount < int64(benchTestSize/3) {
		t.Fatal("Did not read enough documents")
	}
	if updateCount < int64(benchTestSize/8) {
		t.Fatal("Did not update enough documents")
	}
	if delCount < int64(benchTestSize/8) {
		t.Fatal("Did not delete enough documents")
	}
}
<|endoftext|>