<commit_before>package srcgraph

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/sourcegraph/go-vcs"

	"sourcegraph.com/sourcegraph/srcgraph/buildstore"
	"sourcegraph.com/sourcegraph/srcgraph/config"
	"sourcegraph.com/sourcegraph/srcgraph/repo"
	"sourcegraph.com/sourcegraph/srcgraph/scan"
	"sourcegraph.com/sourcegraph/srcgraph/task2"
)

// repository represents a VCS repository on the filesystem. It can be
// autodetected (using detectRepository) or overridden using
// command-line flags (defined by AddRepositoryFlags).
type repository struct {
	CloneURL    string
	CommitID    string
	vcsTypeName string
	RootDir     string
}

func detectRepository2(dir string) *repository {
	r := detectRepository(dir)
	return &r
}

// detects respository from filesystem layout
func detectRepository(dir string) (dr repository) {
	if !isDir(dir) {
		log.Fatal("dir does not exist: ", dir)
	}

	rootDirCmds := map[string]*exec.Cmd{
		"git": exec.Command("git", "rev-parse", "--show-toplevel"),
		"hg":  exec.Command("hg", "root"),
	}
	for tn, cmd := range rootDirCmds {
		cmd.Dir = dir
		out, err := cmd.Output()
		if err == nil {
			dr.RootDir = strings.TrimSpace(string(out))
			dr.vcsTypeName = tn
			break
		}
	}

	if dr.RootDir == "" {
		if *Verbose {
			log.Printf("warning: failed to detect repository root dir for %q", dir)
		}
		return
	}

	updateVCSIgnore("." + dr.vcsTypeName + "ignore")

	cloneURLCmd := map[string]*exec.Cmd{
		"git": exec.Command("git", "config", "remote.origin.url"),
		"hg":  exec.Command("hg", "paths", "default"),
	}[dr.vcsTypeName]

	vcsType := vcs.VCSByName[dr.vcsTypeName]
	repo, err := vcs.Open(vcsType, dr.RootDir)
	if err != nil {
		if *Verbose {
			log.Printf("warning: failed to open repository at %s: %s", dr.RootDir, err)
		}
		return
	}

	dr.CommitID, err = repo.CurrentCommitID()
	if err != nil {
		return
	}

	cloneURLCmd.Dir = dir
	cloneURL, err := cloneURLCmd.Output()
	if err != nil {
		return
	}
	dr.CloneURL = strings.TrimSpace(string(cloneURL))

	if dr.vcsTypeName == "git" {
		dr.CloneURL = strings.Replace(dr.CloneURL, "git@github.com:", "git://github.com/", 1)
	}

	return
}

func AddRepositoryFlags(fs *flag.FlagSet) *repository {
	r := detectRepository(*dir)
	fs.StringVar(&r.CloneURL, "cloneurl", r.CloneURL, "clone URL of repository")
	fs.StringVar(&r.CommitID, "commit", r.CommitID, "commit ID of current working tree")
	fs.StringVar(&r.vcsTypeName, "vcs", r.vcsTypeName, `VCS type ("git" or "hg")`)
	fs.StringVar(&r.RootDir, "root", r.RootDir, `root directory of repository`)
	return &r
}

func AddRepositoryFlags2(fs *flag.FlagSet, r *repository) {
	fs.StringVar(&r.CloneURL, "cloneurl", r.CloneURL, "clone URL of repository")
	fs.StringVar(&r.CommitID, "commit", r.CommitID, "commit ID of current working tree")
	fs.StringVar(&r.vcsTypeName, "vcs", r.vcsTypeName, `VCS type ("git" or "hg")`)
	fs.StringVar(&r.RootDir, "root", r.RootDir, `root directory of repository`)
}

func AddRepositoryConfigFlags(fs *flag.FlagSet, r *repository) *repositoryConfigurator {
	rc := &repositoryConfigurator{Repository: r}

	var defaultFile string
	if f, err := findCachedRepoConfigFile(r); err == nil && isFile(f) {
		defaultFile = f
	}

	fs.StringVar(&rc.ConfigFile, "conf.cached", defaultFile, "cached repository config to use (if blank, scans repository for source units and reads .sourcegraph in root dir)")
	fs.BoolVar(&rc.cacheConfig, "conf.cache", true, "cache generated config for repository (saves time on subsequent runs)")
	return rc
}

func AddRepositoryConfigFlags2(fs *flag.FlagSet, rc *repositoryConfigurator) {
	fs.StringVar(&rc.ConfigFile, "conf.cached", "", "cached repository config to use (if blank, scans repository for source units and reads .sourcegraph in root dir)")
	fs.BoolVar(&rc.cacheConfig, "conf.cache", true, "cache generated config for repository (saves time on subsequent runs)")
}

func InitializeConfigurator(rc *repositoryConfigurator) {
	if rc.ConfigFile == "" {
		if f, err := findCachedRepoConfigFile(rc.Repository); err == nil && isFile(f) {
			rc.ConfigFile = f
		}
	}
}

// findCachedRepoConfigFile determines the filename where the cached
// repository config will be stored, if config caching is enabled.
func findCachedRepoConfigFile(r *repository) (string, error) {
	if r.RootDir == "" {
		return "", fmt.Errorf("no repository root directory")
	}

	repoStore, err := buildstore.NewRepositoryStore(r.RootDir)
	if err != nil {
		return "", err
	}

	rootDataDir, err := buildstore.RootDir(repoStore)
	if err != nil {
		return "", err
	}

	return filepath.Join(rootDataDir, repoStore.CommitPath(r.CommitID), buildstore.CachedRepositoryConfigFilename), nil
}

// repositoryConfigurator gets the *config.Repository for a
// *repository. It uses a cached config if one exists (and cacheConfig
// is true), and otherwise runs the config and scan steps to obtain
// the *config.Repository.
type repositoryConfigurator struct {
	Repository  *repository
	ConfigFile  string
	cacheConfig bool
}

// GetRepositoryConfig gets the .sourcegraph repository config file
// (parsed into *config.Repository) for the repository on the
// filesystem set in rc.Repository. If a cached repository config
// exists, and cacheConfig is true, that is used; otherwise, it runs
// the config and scan steps to obtain the *config.Repository.
func (rc *repositoryConfigurator) GetRepositoryConfig(x *task2.Context) *config.Repository {
	if rc.ConfigFile != "" {
		f, err := os.Open(rc.ConfigFile)
		if err != nil {
			log.Fatal("error opening repository config file:", err)
		}
		var c *config.Repository
		err = json.NewDecoder(f).Decode(&c)
		if err != nil {
			log.Fatalf("error decoding repository config file %q: %s", rc.ConfigFile, err)
		}
		return c
	}

	c, err := scan.ReadDirConfigAndScan(rc.Repository.RootDir, repo.MakeURI(rc.Repository.CloneURL), x)
	if err != nil {
		log.Fatal(err)
	}

	if rc.cacheConfig {
		configFile, err := findCachedRepoConfigFile(rc.Repository)
		if err != nil {
			log.Fatalf("can't determine filename for repository configuration cache file: %s", err)
		}
		err = os.MkdirAll(filepath.Dir(configFile), 0700)
		if err != nil {
			log.Fatal(err)
		}
		data, err := json.MarshalIndent(c, "", "  ")
		if err != nil {
			log.Fatal(err)
		}
		err = ioutil.WriteFile(configFile, data, 0700)
		if err != nil {
			log.Fatal(err)
		}
	}

	return c
}
<commit_msg>new srcgraph config<commit_after>package srcgraph

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/sourcegraph/go-vcs"

	"sourcegraph.com/sourcegraph/srcgraph/buildstore"
	"sourcegraph.com/sourcegraph/srcgraph/config"
	"sourcegraph.com/sourcegraph/srcgraph/repo"
	"sourcegraph.com/sourcegraph/srcgraph/scan"
	"sourcegraph.com/sourcegraph/srcgraph/task2"
)

type JobContext struct {
	RootDir  string             // Root directory containing repository being analyzed
	VCS      vcs.VCS            // VCS type (git or hg)
	CommitID string             // CommitID of current working directory
	Repo     *config.Repository // Repository-level configuration, read from .sourcegraph-data/config.json
}

func NewJobContext(dir string, x *task2.Context) (*JobContext, error) {
	if !isDir(dir) {
		return nil, fmt.Errorf("directory not exist: %q", dir)
	}

	// VCS and root directory
	jc := new(JobContext)
	for _, v := range vcs.VCSByName {
		if d, err := getVCSRootDir(v, dir); err == nil {
			jc.VCS = v
			jc.RootDir = d
			break
		}
	}
	if jc.RootDir == "" {
		return nil, fmt.Errorf("warning: failed to detect repository root dir for %q", dir)
	}

	// Clone URL and Repo
	cloneURL, err := getVCSCloneURL(jc.VCS, dir)
	if err != nil {
		return nil, err
	}
	uri := repo.MakeURI(cloneURL)

	// CommitID
	repo, err := vcs.Open(jc.VCS, dir)
	if err != nil {
		return nil, err
	}
	jc.CommitID, err = repo.CurrentCommitID()
	if err != nil {
		return nil, err
	}

	// updateVCSIgnore("." + jc.VCSTypeName + "ignore") // TODO: desirable?

	jc.Repo, err = ReadOrComputeRepositoryConfig(jc.RootDir, jc.CommitID, uri, x)
	if err != nil {
		return nil, err
	}

	return jc, nil
}

func ReadOrComputeRepositoryConfig(rootDir string, commitID string, repoURI repo.URI, x *task2.Context) (*config.Repository, error) {
	if rootDir == "" {
		return nil, fmt.Errorf("no repository root directory")
	}
	repoStore, err := buildstore.NewRepositoryStore(rootDir)
	if err != nil {
		return nil, err
	}
	rootDataDir, err := buildstore.RootDir(repoStore)
	if err != nil {
		return nil, err
	}

	configFile := filepath.Join(rootDataDir, repoStore.CommitPath(commitID), buildstore.CachedRepositoryConfigFilename)
	if isFile(configFile) {
		// Read
		f, err := os.Open(configFile)
		if err != nil {
			return nil, err
		}
		var c config.Repository
		err = json.NewDecoder(f).Decode(&c)
		if err != nil {
			return nil, err
		}
		return &c, nil
	} else {
		// Compute
		return scan.ReadDirConfigAndScan(rootDir, repoURI, x)
	}
}

func getVCSRootDir(v vcs.VCS, dir string) (string, error) {
	var cmd *exec.Cmd
	switch v {
	case vcs.Git:
		cmd = exec.Command("git", "rev-parse", "--show-toplevel")
	case vcs.Hg:
		exec.Command("hg", "root")
	}
	if cmd == nil {
		return "", fmt.Errorf("unrecognized VCS %v", v)
	}
	out, err := cmd.Output()
	if err != nil {
		return "", err
	}
	return strings.TrimSpace(string(out)), nil
}

func getVCSCloneURL(v vcs.VCS, dir string) (string, error) {
	var cmd *exec.Cmd
	switch v {
	case vcs.Git:
		cmd = exec.Command("git", "config", "remote.origin.url")
	case vcs.Hg:
		cmd = exec.Command("hg", "paths", "default")
	}
	if cmd == nil {
		return "", fmt.Errorf("unrecognized VCS %v", v)
	}
	cmd.Dir = dir
	out, err := cmd.Output()
	if err != nil {
		return "", err
	}

	cloneURL := strings.TrimSpace(string(out))
	if v == vcs.Git {
		cloneURL = strings.Replace(cloneURL, "git@github.com", "git://github.com/", 1)
	}
	return cloneURL, nil
}

//////////////////////////////////////////////////////////////////////////

// repository represents a VCS repository on the filesystem. It can be
// autodetected (using detectRepository) or overridden using
// command-line flags (defined by AddRepositoryFlags).
type repository struct {
	CloneURL    string
	CommitID    string
	vcsTypeName string
	RootDir     string
}

func detectRepository2(dir string) *repository {
	r := detectRepository(dir)
	return &r
}

// detects respository from filesystem layout
func detectRepository(dir string) (dr repository) {
	if !isDir(dir) {
		log.Fatal("dir does not exist: ", dir)
	}

	rootDirCmds := map[string]*exec.Cmd{
		"git": exec.Command("git", "rev-parse", "--show-toplevel"),
		"hg":  exec.Command("hg", "root"),
	}
	for tn, cmd := range rootDirCmds {
		cmd.Dir = dir
		out, err := cmd.Output()
		if err == nil {
			dr.RootDir = strings.TrimSpace(string(out))
			dr.vcsTypeName = tn
			break
		}
	}

	if dr.RootDir == "" {
		if *Verbose {
			log.Printf("warning: failed to detect repository root dir for %q", dir)
		}
		return
	}

	updateVCSIgnore("." + dr.vcsTypeName + "ignore")

	cloneURLCmd := map[string]*exec.Cmd{
		"git": exec.Command("git", "config", "remote.origin.url"),
		"hg":  exec.Command("hg", "paths", "default"),
	}[dr.vcsTypeName]

	vcsType := vcs.VCSByName[dr.vcsTypeName]
	repo, err := vcs.Open(vcsType, dr.RootDir)
	if err != nil {
		if *Verbose {
			log.Printf("warning: failed to open repository at %s: %s", dr.RootDir, err)
		}
		return
	}

	dr.CommitID, err = repo.CurrentCommitID()
	if err != nil {
		return
	}

	cloneURLCmd.Dir = dir
	cloneURL, err := cloneURLCmd.Output()
	if err != nil {
		return
	}
	dr.CloneURL = strings.TrimSpace(string(cloneURL))

	if dr.vcsTypeName == "git" {
		dr.CloneURL = strings.Replace(dr.CloneURL, "git@github.com:", "git://github.com/", 1)
	}

	return
}

func AddRepositoryFlags(fs *flag.FlagSet) *repository {
	r := detectRepository(*dir)
	fs.StringVar(&r.CloneURL, "cloneurl", r.CloneURL, "clone URL of repository")
	fs.StringVar(&r.CommitID, "commit", r.CommitID, "commit ID of current working tree")
	fs.StringVar(&r.vcsTypeName, "vcs", r.vcsTypeName, `VCS type ("git" or "hg")`)
	fs.StringVar(&r.RootDir, "root", r.RootDir, `root directory of repository`)
	return &r
}

func AddRepositoryFlags2(fs *flag.FlagSet, r *repository) {
	fs.StringVar(&r.CloneURL, "cloneurl", r.CloneURL, "clone URL of repository")
	fs.StringVar(&r.CommitID, "commit", r.CommitID, "commit ID of current working tree")
	fs.StringVar(&r.vcsTypeName, "vcs", r.vcsTypeName, `VCS type ("git" or "hg")`)
	fs.StringVar(&r.RootDir, "root", r.RootDir, `root directory of repository`)
}

func AddRepositoryConfigFlags(fs *flag.FlagSet, r *repository) *repositoryConfigurator {
	rc := &repositoryConfigurator{Repository: r}

	var defaultFile string
	if f, err := findCachedRepoConfigFile(r); err == nil && isFile(f) {
		defaultFile = f
	}

	fs.StringVar(&rc.ConfigFile, "conf.cached", defaultFile, "cached repository config to use (if blank, scans repository for source units and reads .sourcegraph in root dir)")
	fs.BoolVar(&rc.cacheConfig, "conf.cache", true, "cache generated config for repository (saves time on subsequent runs)")
	return rc
}

func AddRepositoryConfigFlags2(fs *flag.FlagSet, rc *repositoryConfigurator) {
	fs.StringVar(&rc.ConfigFile, "conf.cached", "", "cached repository config to use (if blank, scans repository for source units and reads .sourcegraph in root dir)")
	fs.BoolVar(&rc.cacheConfig, "conf.cache", true, "cache generated config for repository (saves time on subsequent runs)")
}

func InitializeConfigurator(rc *repositoryConfigurator) {
	if rc.ConfigFile == "" {
		if f, err := findCachedRepoConfigFile(rc.Repository); err == nil && isFile(f) {
			rc.ConfigFile = f
		}
	}
}

// findCachedRepoConfigFile determines the filename where the cached
// repository config will be stored, if config caching is enabled.
func findCachedRepoConfigFile(r *repository) (string, error) {
	if r.RootDir == "" {
		return "", fmt.Errorf("no repository root directory")
	}

	repoStore, err := buildstore.NewRepositoryStore(r.RootDir)
	if err != nil {
		return "", err
	}

	rootDataDir, err := buildstore.RootDir(repoStore)
	if err != nil {
		return "", err
	}

	return filepath.Join(rootDataDir, repoStore.CommitPath(r.CommitID), buildstore.CachedRepositoryConfigFilename), nil
}

// repositoryConfigurator gets the *config.Repository for a
// *repository. It uses a cached config if one exists (and cacheConfig
// is true), and otherwise runs the config and scan steps to obtain
// the *config.Repository.
type repositoryConfigurator struct {
	Repository  *repository
	ConfigFile  string
	cacheConfig bool
}

// GetRepositoryConfig gets the .sourcegraph repository config file
// (parsed into *config.Repository) for the repository on the
// filesystem set in rc.Repository. If a cached repository config
// exists, and cacheConfig is true, that is used; otherwise, it runs
// the config and scan steps to obtain the *config.Repository.
func (rc *repositoryConfigurator) GetRepositoryConfig(x *task2.Context) *config.Repository {
	if rc.ConfigFile != "" {
		f, err := os.Open(rc.ConfigFile)
		if err != nil {
			log.Fatal("error opening repository config file:", err)
		}
		var c *config.Repository
		err = json.NewDecoder(f).Decode(&c)
		if err != nil {
			log.Fatalf("error decoding repository config file %q: %s", rc.ConfigFile, err)
		}
		return c
	}

	c, err := scan.ReadDirConfigAndScan(rc.Repository.RootDir, repo.MakeURI(rc.Repository.CloneURL), x)
	if err != nil {
		log.Fatal(err)
	}

	if rc.cacheConfig {
		configFile, err := findCachedRepoConfigFile(rc.Repository)
		if err != nil {
			log.Fatalf("can't determine filename for repository configuration cache file: %s", err)
		}
		err = os.MkdirAll(filepath.Dir(configFile), 0700)
		if err != nil {
			log.Fatal(err)
		}
		data, err := json.MarshalIndent(c, "", "  ")
		if err != nil {
			log.Fatal(err)
		}
		err = ioutil.WriteFile(configFile, data, 0700)
		if err != nil {
			log.Fatal(err)
		}
	}

	return c
}
<|endoftext|>


<commit_before>package rhymer_test

import (
    "testing"
    "os"
    "github.com/juanchel/rhymer"
)

var r = rhymer.NewRhymer()

var rhymeTests = []struct {
    a string     // first input
    b string     // second input
    expected int // expected result
} {
    {"cat",    "cat",     1},
    {"cat",    "bat",     1},
    {"cat",    "acrobat", 1},
    {"over",   "clover",  1},
    {"master", "raster",  1},
    {"masTER", "RaStEr",  1},
    {"aunt",   "rant",    1},
    {"aunt",   "want",    1},
    {"rant",   "want",    0},
    {"cat",    "dog",     0},
    {"over",   "ever",    0},
    {"ever",   "clover",  0},
    {"kanye",  "cat",     -1},
    {"kanye",  "yeezy",   -1},
    {"",       "cat",     -1},
    {"",       "",        -1},
    {"^cat",   "&bat",    -1},
}

func TestMain(m *testing.M) {
    os.Exit(m.Run())
}

func TestRhymes(m *testing.T) {
    for _, i := range rhymeTests {
        actual := r.Rhymes(i.a, i.b)
        if actual != i.expected {
            m.Errorf("Rhymes(%s, %s): expected %d but got %d", i.a, i.b, i.expected, actual)
        }
    }
}
<commit_msg>Add more tests (find rhymes)<commit_after>package rhymer_test

import (
    "testing"
    "os"
    "github.com/juanchel/rhymer"
)

var r = rhymer.NewRhymer()

var rhymeTests = []struct {
    a string     // first input
    b string     // second input
    expected int // expected result
} {
    {"cat",    "cat",     1},
    {"cat",    "bat",     1},
    {"cat",    "acrobat", 1},
    {"over",   "clover",  1},
    {"master", "raster",  1},
    {"masTER", "RaStEr",  1},
    {"aunt",   "rant",    1},
    {"aunt",   "want",    1},
    {"rant",   "want",    0},
    {"cat",    "dog",     0},
    {"over",   "ever",    0},
    {"ever",   "clover",  0},
    {"kanye",  "cat",     -1},
    {"kanye",  "yeezy",   -1},
    {"",       "cat",     -1},
    {"",       "",        -1},
    {"^cat",   "&bat",    -1},
}

var pronounceTests = []struct {
    a string          // input
    expected []string // expected result
} {
    {"cat", }
}

func TestMain(m *testing.M) {
    os.Exit(m.Run())
}

func TestRhymes(m *testing.T) {
    for _, i := range rhymeTests {
        actual := r.Rhymes(i.a, i.b)
        if actual != i.expected {
            m.Errorf("Rhymes(%s, %s): expected %d but got %d", i.a, i.b, i.expected, actual)
        }
    }
}

func TestPronounce(m *testing.T) {

}

func <|endoftext|>


<commit_before>package riot

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"strings"

	"github.com/coltiebaby/g-law/ratelimit"
	"github.com/coltiebaby/g-law/riot/errors"
)

type RiotClient struct {
	region           Region
	rateLimitEnabled bool
	limiter          ratelimit.Limiter
}

func NewRiotClient(region Region, enabled bool) *RiotClient {
	var rateLimiter ratelimit.Limiter
	if enabled {
		rateLimiter = SetupRateLimiter(enabled)
	}

	return &RiotClient{
		rateLimitEnabled: enabled,
		region:           region,
		limiter:          rateLimiter,
	}
}

func (rc *RiotClient) NewRequest(uri string) (req ApiRequest) {
	req = &RiotRequest{
		uri: uri,
	}

	if rc.rateLimitEnabled {
		rc.limiter.Take(int(rc.region))
	}

	return req
}

func (rc *RiotClient) Get(req ApiRequest) (resp *http.Response, err error) {
	platform := RegionsPlatform[rc.region]
	host := fmt.Sprintf("%s.api.riotgames.com", strings.ToLower(platform))

	u := &url.URL{
		Scheme:   "https",
		Host:     host,
		Path:     fmt.Sprintf("lol/%s", req.Uri()),
		RawQuery: req.Encode(),
	}

	var e error
	if resp, e = get(u); e != nil {
		err = errors.NewErrorFromString(e.Error())
	}

	fmt.Println("Get", err)
	return resp, err

}

func (rc *RiotClient) ChangeRegion(region Region) {
	rc.region = region
}

type RiotRequest struct {
	uri    string
	params url.Values
}

func (rr *RiotRequest) Encode() string {
	return rr.params.Encode()
}

func (rr *RiotRequest) Uri() string {
	return rr.uri
}

func (rr *RiotRequest) AddParameter(key, value string) {
	rr.params.Add(key, value)
}

func (rr *RiotRequest) SetParameters(params url.Values) {
	rr.params = params
}

func GetResultFromResp(resp *http.Response, v interface{}) (err error) {
	if e := json.NewDecoder(resp.Body).Decode(v); e != nil {
		return errors.NewErrorFromString(e.Error())
	}

	if isBad(resp.StatusCode) {
		err = errors.NewRequestError(resp)
	}

	return err
}

func isBad(code int) bool {
	return (code >= 200 && code < 300) != true
}

func get(u *url.URL) (resp *http.Response, err error) {
	req, err := http.NewRequest("GET", u.String(), nil)
	if err != nil {
		return resp, err
	}

	req.Header.Add("X-Riot-Token", c.Token)
	resp, err = http.DefaultClient.Do(req)
	if err != nil {
		return resp, err
	}

	fmt.Println("get", err)
	return resp, err
}
<commit_msg>ratelimiting on actual request not when we make a new one<commit_after>package riot

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"strings"

	"github.com/coltiebaby/g-law/ratelimit"
	"github.com/coltiebaby/g-law/riot/errors"
)

type RiotClient struct {
	region           Region
	rateLimitEnabled bool
	limiter          ratelimit.Limiter
}

func NewRiotClient(region Region, enabled bool) *RiotClient {
	var rateLimiter ratelimit.Limiter
	if enabled {
		rateLimiter = SetupRateLimiter(enabled)
	}

	return &RiotClient{
		rateLimitEnabled: enabled,
		region:           region,
		limiter:          rateLimiter,
	}
}

func (rc *RiotClient) NewRequest(uri string) (req ApiRequest) {
	req = &RiotRequest{
		uri: uri,
	}

	return req
}

func (rc *RiotClient) Get(req ApiRequest) (resp *http.Response, err error) {
	platform := RegionsPlatform[rc.region]
	host := fmt.Sprintf("%s.api.riotgames.com", strings.ToLower(platform))

	u := &url.URL{
		Scheme:   "https",
		Host:     host,
		Path:     fmt.Sprintf("lol/%s", req.Uri()),
		RawQuery: req.Encode(),
	}

	if rc.rateLimitEnabled {
		rc.limiter.Take(int(rc.region))
	}

	if resp, err = get(u); err != nil {
		err = errors.NewErrorFromString(err.Error())
	}

	return resp, err

}

func (rc *RiotClient) ChangeRegion(region Region) {
	rc.region = region
}

type RiotRequest struct {
	uri    string
	params url.Values
}

func (rr *RiotRequest) Encode() string {
	return rr.params.Encode()
}

func (rr *RiotRequest) Uri() string {
	return rr.uri
}

func (rr *RiotRequest) AddParameter(key, value string) {
	rr.params.Add(key, value)
}

func (rr *RiotRequest) SetParameters(params url.Values) {
	rr.params = params
}

func GetResultFromResp(resp *http.Response, v interface{}) (err error) {
	if e := json.NewDecoder(resp.Body).Decode(v); e != nil {
		return errors.NewErrorFromString(e.Error())
	}

	if isBad(resp.StatusCode) {
		err = errors.NewRequestError(resp)
	}

	return err
}

func isBad(code int) bool {
	return (code >= 200 && code < 300) != true
}

func get(u *url.URL) (resp *http.Response, err error) {
	req, err := http.NewRequest("GET", u.String(), nil)
	if err != nil {
		return resp, err
	}

	req.Header.Add("X-Riot-Token", c.Token)
	resp, err = http.DefaultClient.Do(req)
	if err != nil {
		return resp, err
	}

	return resp, err
}
<|endoftext|>


<commit_before>// Copyright 2017 The go-hep Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package rootio

import (
	"fmt"
	"reflect"
)

type taxis struct {
	rvers int16
	tnamed
	attaxis attaxis
	nbins   int        // number of bins
	xmin    float64    // low edge of first bin
	xmax    float64    // upper edge of last bin
	xbins   ArrayD     // bin edges array in X
	first   int        // first bin to display
	last    int        // last bin to display
	bits2   uint16     // second bit status word
	time    bool       // on/off displaying time values instead of numerics
	tfmt    string     // date&time format
	labels  *thashList // list of labels
	modlabs *tlist     // list of modified labels
}

func (a *taxis) Class() string {
	return "TAxis"
}

func (a *taxis) XMin() float64 {
	return a.xmin
}

func (a *taxis) XMax() float64 {
	return a.xmax
}

func (a *taxis) NBins() int {
	return a.nbins
}

func (a *taxis) XBins() []float64 {
	return a.xbins.Data
}

func (a *taxis) BinCenter(i int) float64 {
	if len(a.xbins.Data) == 0 || i < 1 || i > a.nbins {
		width := (a.xmax - a.xmin) / float64(a.nbins)
		return a.xmin + (float64(i)-0.5)*width
	}
	width := a.xbins.Data[i] - a.xbins.Data[i-1]
	return a.xbins.Data[i-1] + 0.5*width
}

func (a *taxis) BinLowEdge(i int) float64 {
	if len(a.xbins.Data) == 0 || i < 1 || i > a.nbins {
		width := (a.xmax - a.xmin) / float64(a.nbins)
		return a.xmin + float64(i-1)*width
	}
	return a.xbins.Data[i-1]
}

func (a *taxis) BinWidth(i int) float64 {
	if a.nbins <= 0 {
		return 0
	}
	if len(a.xbins.Data) <= 0 {
		return (a.xmax - a.xmin) / float64(a.nbins)
	}
	if i > a.nbins {
		i = a.nbins
	}
	if i < 1 {
		i = 1
	}
	return a.xbins.Data[i] - a.xbins.Data[i-1]
}

func (a *taxis) UnmarshalROOT(r *RBuffer) error {
	if r.err != nil {
		return r.err
	}

	beg := r.Pos()
	vers, pos, bcnt := r.ReadVersion()
	a.rvers = vers
	if vers < 9 {
		return fmt.Errorf("rootio: TAxis version too old (%d<9)", vers)
	}

	for _, v := range []ROOTUnmarshaler{
		&a.tnamed,
		&a.attaxis,
	} {
		if err := v.UnmarshalROOT(r); err != nil {
			r.err = err
			return r.err
		}
	}

	a.nbins = int(r.ReadI32())
	a.xmin = r.ReadF64()
	a.xmax = r.ReadF64()

	if err := a.xbins.UnmarshalROOT(r); err != nil {
		r.err = err
		return r.err
	}

	a.first = int(r.ReadI32())
	a.last = int(r.ReadI32())
	a.bits2 = r.ReadU16()
	a.time = r.ReadBool()
	a.tfmt = r.ReadString()

	a.labels = nil
	labels := r.ReadObjectAny()
	if labels != nil {
		a.labels = labels.(*thashList)
	}

	a.modlabs = nil
	if vers >= 10 {
		modlabs := r.ReadObjectAny()
		if modlabs != nil {
			a.modlabs = modlabs.(*tlist)
		}
	}

	r.CheckByteCount(pos, bcnt, beg, "TAxis")
	return r.err
}

func init() {
	{
		f := func() reflect.Value {
			o := &taxis{}
			return reflect.ValueOf(o)
		}
		Factory.add("TAxis", f)
		Factory.add("*rootio.taxis", f)
	}
}

var _ Object = (*taxis)(nil)
var _ Named = (*taxis)(nil)
var _ Axis = (*taxis)(nil)
var _ ROOTUnmarshaler = (*taxis)(nil)
<commit_msg>rootio: implement ROOTMarshaler for TAxis<commit_after>// Copyright 2017 The go-hep Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package rootio

import (
	"fmt"
	"reflect"
)

type taxis struct {
	rvers int16
	tnamed
	attaxis attaxis
	nbins   int        // number of bins
	xmin    float64    // low edge of first bin
	xmax    float64    // upper edge of last bin
	xbins   ArrayD     // bin edges array in X
	first   int        // first bin to display
	last    int        // last bin to display
	bits2   uint16     // second bit status word
	time    bool       // on/off displaying time values instead of numerics
	tfmt    string     // date&time format
	labels  *thashList // list of labels
	modlabs *tlist     // list of modified labels
}

func (a *taxis) Class() string {
	return "TAxis"
}

func (a *taxis) XMin() float64 {
	return a.xmin
}

func (a *taxis) XMax() float64 {
	return a.xmax
}

func (a *taxis) NBins() int {
	return a.nbins
}

func (a *taxis) XBins() []float64 {
	return a.xbins.Data
}

func (a *taxis) BinCenter(i int) float64 {
	if len(a.xbins.Data) == 0 || i < 1 || i > a.nbins {
		width := (a.xmax - a.xmin) / float64(a.nbins)
		return a.xmin + (float64(i)-0.5)*width
	}
	width := a.xbins.Data[i] - a.xbins.Data[i-1]
	return a.xbins.Data[i-1] + 0.5*width
}

func (a *taxis) BinLowEdge(i int) float64 {
	if len(a.xbins.Data) == 0 || i < 1 || i > a.nbins {
		width := (a.xmax - a.xmin) / float64(a.nbins)
		return a.xmin + float64(i-1)*width
	}
	return a.xbins.Data[i-1]
}

func (a *taxis) BinWidth(i int) float64 {
	if a.nbins <= 0 {
		return 0
	}
	if len(a.xbins.Data) <= 0 {
		return (a.xmax - a.xmin) / float64(a.nbins)
	}
	if i > a.nbins {
		i = a.nbins
	}
	if i < 1 {
		i = 1
	}
	return a.xbins.Data[i] - a.xbins.Data[i-1]
}

func (a *taxis) MarshalROOT(w *WBuffer) (int, error) {
	if w.err != nil {
		return 0, w.err
	}

	pos := w.Pos()
	w.WriteVersion(a.rvers)

	for _, v := range []ROOTMarshaler{
		&a.tnamed,
		&a.attaxis,
	} {
		if _, err := v.MarshalROOT(w); err != nil {
			w.err = err
			return 0, w.err
		}
	}

	w.WriteI32(int32(a.nbins))
	w.WriteF64(a.xmin)
	w.WriteF64(a.xmax)

	if _, err := a.xbins.MarshalROOT(w); err != nil {
		w.err = err
		return 0, w.err
	}

	w.WriteI32(int32(a.first))
	w.WriteI32(int32(a.last))
	w.WriteU16(a.bits2)
	w.WriteBool(a.time)
	w.WriteString(a.tfmt)

	for _, obj := range []Object{a.labels, a.modlabs} {
		if err := w.WriteObjectAny(obj); err != nil {
			w.err = err
			return 0, w.err
		}
	}

	return w.SetByteCount(pos, "TAxis")
}

func (a *taxis) UnmarshalROOT(r *RBuffer) error {
	if r.err != nil {
		return r.err
	}

	beg := r.Pos()
	vers, pos, bcnt := r.ReadVersion()
	a.rvers = vers
	if vers < 9 {
		return fmt.Errorf("rootio: TAxis version too old (%d<9)", vers)
	}

	for _, v := range []ROOTUnmarshaler{
		&a.tnamed,
		&a.attaxis,
	} {
		if err := v.UnmarshalROOT(r); err != nil {
			r.err = err
			return r.err
		}
	}

	a.nbins = int(r.ReadI32())
	a.xmin = r.ReadF64()
	a.xmax = r.ReadF64()

	if err := a.xbins.UnmarshalROOT(r); err != nil {
		r.err = err
		return r.err
	}

	a.first = int(r.ReadI32())
	a.last = int(r.ReadI32())
	a.bits2 = r.ReadU16()
	a.time = r.ReadBool()
	a.tfmt = r.ReadString()

	a.labels = nil
	labels := r.ReadObjectAny()
	if labels != nil {
		a.labels = labels.(*thashList)
	}

	a.modlabs = nil
	if vers >= 10 {
		modlabs := r.ReadObjectAny()
		if modlabs != nil {
			a.modlabs = modlabs.(*tlist)
		}
	}

	r.CheckByteCount(pos, bcnt, beg, "TAxis")
	return r.err
}

func init() {
	{
		f := func() reflect.Value {
			o := &taxis{}
			return reflect.ValueOf(o)
		}
		Factory.add("TAxis", f)
		Factory.add("*rootio.taxis", f)
	}
}

var (
	_ Object          = (*taxis)(nil)
	_ Named           = (*taxis)(nil)
	_ Axis            = (*taxis)(nil)
	_ ROOTMarshaler   = (*taxis)(nil)
	_ ROOTUnmarshaler = (*taxis)(nil)
)
<|endoftext|>


<commit_before>package route

import (
	"fmt"
	"log"
	"net/url"
	"reflect"
	"sort"
	"strings"

	"github.com/eBay/fabio/metrics"
)

// Route maps a path prefix to one or more target URLs.
// routes can have a share value which describes the
// amount of traffic this route should get. You can specify
// that a route should get a fixed percentage of the traffic
// independent of how many instances are running.
type Route struct {
	// Host contains the host of the route.
	// not used for routing but for config generation
	// Table has a map with the host as key
	// for faster lookup and smaller search space.
	Host string

	// Path is the path prefix from a request uri
	Path string

	// Opts is the raw route options
	Opts map[string]string

	// Targets contains the list of URLs
	Targets []*Target

	// wTargets contains 100 targets distributed
	// according to their weight and ordered RR in the
	// same order as targets
	wTargets []*Target

	// total contains the total number of requests for this route.
	// Used by the RRPicker
	total uint64
}

func (r *Route) addTarget(service string, targetURL *url.URL, fixedWeight float64, tags []string) {
	if fixedWeight < 0 {
		fixedWeight = 0
	}

	// de-dup existing target
	for _, t := range r.Targets {
		if t.Service == service && t.URL.String() == targetURL.String() && t.FixedWeight == fixedWeight && reflect.DeepEqual(t.Tags, tags) {
			return
		}
	}

	name, err := metrics.TargetName(service, r.Host, r.Path, targetURL)
	if err != nil {
		log.Printf("[ERROR] Invalid metrics name: %s", err)
		name = "unknown"
	}
	timer := ServiceRegistry.GetTimer(name)

	t := &Target{Service: service, Tags: tags, URL: targetURL, FixedWeight: fixedWeight, Timer: timer, timerName: name}
	r.Targets = append(r.Targets, t)
	r.weighTargets()
}

func (r *Route) filter(skip func(t *Target) bool) {
	var clone []*Target
	for _, t := range r.Targets {
		if skip(t) {
			continue
		}
		clone = append(clone, t)
	}
	r.Targets = clone
	r.weighTargets()
}

func (r *Route) setWeight(service string, weight float64, tags []string) int {
	loop := func(w float64) int {
		n := 0
		for _, t := range r.Targets {
			if service != "" && t.Service != service {
				continue
			}
			if len(tags) > 0 && !contains(t.Tags, tags) {
				continue
			}
			n++
			t.FixedWeight = w
		}
		return n
	}

	// if we have multiple matching targets
	// then we need to distribute the total
	// weight across all of them since the rule
	// states to assign only that percentage
	// of traffic to all matching routes combined.
	n := loop(0)
	w := weight / float64(n)
	loop(w)

	if n > 0 {
		r.weighTargets()
	}
	return n
}

func contains(src, dst []string) bool {
	for _, d := range dst {
		found := false
		for _, s := range src {
			if s == d {
				found = true
				break
			}
		}
		if !found {
			return false
		}
	}
	return true
}

func (r *Route) TargetConfig(t *Target, addWeight bool) string {
	s := fmt.Sprintf("route add %s %s %s", t.Service, r.Host+r.Path, t.URL)
	if addWeight {
		s += fmt.Sprintf(" weight %2.4f", t.Weight)
	} else if t.FixedWeight > 0 {
		s += fmt.Sprintf(" weight %.4f", t.FixedWeight)
	}
	if len(t.Tags) > 0 {
		s += fmt.Sprintf(" tags %q", strings.Join(t.Tags, ","))
	}
	return s
}

// config returns the route configuration in the config language.
// with the weights specified by the user.
func (r *Route) config(addWeight bool) []string {
	var cfg []string
	for _, t := range r.Targets {
		if t.Weight <= 0 {
			continue
		}
		cfg = append(cfg, r.TargetConfig(t, addWeight))
	}
	return cfg
}

// maxSlots defines the maximum number of slots on the ring for
// weighted round-robin distribution for a single route. Consequently,
// this then defines the maximum number of separate instances that can
// serve a single route. maxSlots must be a power of ten.
const maxSlots = 1e4 // 10000

// weighTargets computes the share of traffic each target receives based
// on its weight and the weight of the other targets.
//
// Traffic is first distributed to targets with a fixed weight. If the sum of
// all fixed weights exceeds 100% then they are normalized to 100%.
//
// Targets with a dynamic weight will receive an equal share of the remaining
// traffic if there is any left.
func (r *Route) weighTargets() {
	// how big is the fixed weighted traffic?
	var nFixed int
	var sumFixed float64
	for _, t := range r.Targets {
		if t.FixedWeight > 0 {
			nFixed++
			sumFixed += t.FixedWeight
		}
	}

	// if there are no targets with fixed weight then each target simply gets
	// an equal amount of traffic
	if nFixed == 0 {
		w := 1.0 / float64(len(r.Targets))
		for _, t := range r.Targets {
			t.Weight = w
		}
		r.wTargets = r.Targets
		return
	}

	// normalize fixed weights up (sumFixed < 1) or down (sumFixed > 1)
	scale := 1.0
	if sumFixed > 1 || (nFixed == len(r.Targets) && sumFixed < 1) {
		scale = 1 / sumFixed
	}

	// compute the weight for the targets with dynamic weights
	dynamic := (1 - sumFixed) / float64(len(r.Targets)-nFixed)
	if dynamic < 0 {
		dynamic = 0
	}

	// assign the actual weight to each target
	for _, t := range r.Targets {
		if t.FixedWeight > 0 {
			t.Weight = t.FixedWeight * scale
		} else {
			t.Weight = dynamic
		}
	}

	// distribute the targets on a ring suitable for weighted round-robin
	// distribution
	//
	// This is done in two steps:
	//
	// Step one determines the necessary ring size to distribute the targets
	// according to their weight with reasonable accuracy. For example, two
	// targets with 50% weight fit in a ring of size 2 whereas two targets with
	// 10% and 90% weight require a ring of size 10.
	//
	// To keep it simple we allocate 10000 slots which provides slots to all
	// targets with at least a weight of 0.01%. In addition, we guarantee that
	// every target with a weight > 0 gets at least one slot. The case where
	// all targets get an equal share of traffic is handled earlier so this is
	// for situations with some fixed weight.
	//
	// Step two distributes the targets onto the ring spacing them out evenly
	// so that iterating over the ring performs the weighted rr distribution.
	// For example, a 50/50 distribution on a ring of size 10 should be
	// 0101010101 instead of 0000011111.
	//
	// To ensure that targets with smaller weights are properly placed we place
	// them on the ring first by sorting the targets by slot count.
	//
	// TODO(fs): I assume that this is some sort of mathematical problem
	// (coloring, optimizing, ...) but I don't know which. Happy to make this
	// more formal, if possible.
	//
	slots := make(byN, len(r.Targets))
	usedSlots := 0
	for i, t := range r.Targets {
		n := int(float64(maxSlots) * t.Weight)
		if n == 0 && t.Weight > 0 {
			n = 1
		}
		slots[i].i = i
		slots[i].n = n
		usedSlots += n
	}

	sort.Sort(slots)
	targets := make([]*Target, usedSlots)
	for _, s := range slots {
		if s.n <= 0 {
			continue
		}

		next, step := 0, usedSlots/s.n
		for k := 0; k < s.n; k++ {
			// find the next empty slot
			for targets[next] != nil {
				next = (next + 1) % usedSlots
			}

			// use slot and move to next one
			targets[next] = r.Targets[s.i]
			next = (next + step) % usedSlots
		}
	}

	r.wTargets = targets
}

type byN []struct{ i, n int }

func (r byN) Len() int           { return len(r) }
func (r byN) Swap(i, j int)      { r[i], r[j] = r[j], r[i] }
func (r byN) Less(i, j int) bool { return r[i].n < r[j].n }
<commit_msg>Issue #111: Print opts on generated routes<commit_after>package route

import (
	"fmt"
	"log"
	"net/url"
	"reflect"
	"sort"
	"strings"

	"github.com/eBay/fabio/metrics"
)

// Route maps a path prefix to one or more target URLs.
// routes can have a share value which describes the
// amount of traffic this route should get. You can specify
// that a route should get a fixed percentage of the traffic
// independent of how many instances are running.
type Route struct {
	// Host contains the host of the route.
	// not used for routing but for config generation
	// Table has a map with the host as key
	// for faster lookup and smaller search space.
	Host string

	// Path is the path prefix from a request uri
	Path string

	// Opts is the raw route options
	Opts map[string]string

	// Targets contains the list of URLs
	Targets []*Target

	// wTargets contains 100 targets distributed
	// according to their weight and ordered RR in the
	// same order as targets
	wTargets []*Target

	// total contains the total number of requests for this route.
	// Used by the RRPicker
	total uint64
}

func (r *Route) addTarget(service string, targetURL *url.URL, fixedWeight float64, tags []string) {
	if fixedWeight < 0 {
		fixedWeight = 0
	}

	// de-dup existing target
	for _, t := range r.Targets {
		if t.Service == service && t.URL.String() == targetURL.String() && t.FixedWeight == fixedWeight && reflect.DeepEqual(t.Tags, tags) {
			return
		}
	}

	name, err := metrics.TargetName(service, r.Host, r.Path, targetURL)
	if err != nil {
		log.Printf("[ERROR] Invalid metrics name: %s", err)
		name = "unknown"
	}
	timer := ServiceRegistry.GetTimer(name)

	t := &Target{Service: service, Tags: tags, URL: targetURL, FixedWeight: fixedWeight, Timer: timer, timerName: name}
	r.Targets = append(r.Targets, t)
	r.weighTargets()
}

func (r *Route) filter(skip func(t *Target) bool) {
	var clone []*Target
	for _, t := range r.Targets {
		if skip(t) {
			continue
		}
		clone = append(clone, t)
	}
	r.Targets = clone
	r.weighTargets()
}

func (r *Route) setWeight(service string, weight float64, tags []string) int {
	loop := func(w float64) int {
		n := 0
		for _, t := range r.Targets {
			if service != "" && t.Service != service {
				continue
			}
			if len(tags) > 0 && !contains(t.Tags, tags) {
				continue
			}
			n++
			t.FixedWeight = w
		}
		return n
	}

	// if we have multiple matching targets
	// then we need to distribute the total
	// weight across all of them since the rule
	// states to assign only that percentage
	// of traffic to all matching routes combined.
	n := loop(0)
	w := weight / float64(n)
	loop(w)

	if n > 0 {
		r.weighTargets()
	}
	return n
}

func contains(src, dst []string) bool {
	for _, d := range dst {
		found := false
		for _, s := range src {
			if s == d {
				found = true
				break
			}
		}
		if !found {
			return false
		}
	}
	return true
}

func (r *Route) TargetConfig(t *Target, addWeight bool) string {
	s := fmt.Sprintf("route add %s %s %s", t.Service, r.Host+r.Path, t.URL)
	if addWeight {
		s += fmt.Sprintf(" weight %2.4f", t.Weight)
	} else if t.FixedWeight > 0 {
		s += fmt.Sprintf(" weight %.4f", t.FixedWeight)
	}
	if len(t.Tags) > 0 {
		s += fmt.Sprintf(" tags %q", strings.Join(t.Tags, ","))
	}
	if len(r.Opts) > 0 {
		var keys []string
		for k := range r.Opts {
			keys = append(keys, k)
		}
		sort.Strings(keys)

		var vals []string
		for _, k := range keys {
			vals = append(vals, k+"="+r.Opts[k])
		}
		s += fmt.Sprintf(" opts \"%s\"", strings.Join(vals, " "))
	}
	return s
}

// config returns the route configuration in the config language.
// with the weights specified by the user.
func (r *Route) config(addWeight bool) []string {
	var cfg []string
	for _, t := range r.Targets {
		if t.Weight <= 0 {
			continue
		}
		cfg = append(cfg, r.TargetConfig(t, addWeight))
	}
	return cfg
}

// maxSlots defines the maximum number of slots on the ring for
// weighted round-robin distribution for a single route. Consequently,
// this then defines the maximum number of separate instances that can
// serve a single route. maxSlots must be a power of ten.
const maxSlots = 1e4 // 10000

// weighTargets computes the share of traffic each target receives based
// on its weight and the weight of the other targets.
//
// Traffic is first distributed to targets with a fixed weight. If the sum of
// all fixed weights exceeds 100% then they are normalized to 100%.
//
// Targets with a dynamic weight will receive an equal share of the remaining
// traffic if there is any left.
func (r *Route) weighTargets() {
	// how big is the fixed weighted traffic?
	var nFixed int
	var sumFixed float64
	for _, t := range r.Targets {
		if t.FixedWeight > 0 {
			nFixed++
			sumFixed += t.FixedWeight
		}
	}

	// if there are no targets with fixed weight then each target simply gets
	// an equal amount of traffic
	if nFixed == 0 {
		w := 1.0 / float64(len(r.Targets))
		for _, t := range r.Targets {
			t.Weight = w
		}
		r.wTargets = r.Targets
		return
	}

	// normalize fixed weights up (sumFixed < 1) or down (sumFixed > 1)
	scale := 1.0
	if sumFixed > 1 || (nFixed == len(r.Targets) && sumFixed < 1) {
		scale = 1 / sumFixed
	}

	// compute the weight for the targets with dynamic weights
	dynamic := (1 - sumFixed) / float64(len(r.Targets)-nFixed)
	if dynamic < 0 {
		dynamic = 0
	}

	// assign the actual weight to each target
	for _, t := range r.Targets {
		if t.FixedWeight > 0 {
			t.Weight = t.FixedWeight * scale
		} else {
			t.Weight = dynamic
		}
	}

	// distribute the targets on a ring suitable for weighted round-robin
	// distribution
	//
	// This is done in two steps:
	//
	// Step one determines the necessary ring size to distribute the targets
	// according to their weight with reasonable accuracy. For example, two
	// targets with 50% weight fit in a ring of size 2 whereas two targets with
	// 10% and 90% weight require a ring of size 10.
	//
	// To keep it simple we allocate 10000 slots which provides slots to all
	// targets with at least a weight of 0.01%. In addition, we guarantee that
	// every target with a weight > 0 gets at least one slot. The case where
	// all targets get an equal share of traffic is handled earlier so this is
	// for situations with some fixed weight.
	//
	// Step two distributes the targets onto the ring spacing them out evenly
	// so that iterating over the ring performs the weighted rr distribution.
	// For example, a 50/50 distribution on a ring of size 10 should be
	// 0101010101 instead of 0000011111.
	//
	// To ensure that targets with smaller weights are properly placed we place
	// them on the ring first by sorting the targets by slot count.
	//
	// TODO(fs): I assume that this is some sort of mathematical problem
	// (coloring, optimizing, ...) but I don't know which. Happy to make this
	// more formal, if possible.
	//
	slots := make(byN, len(r.Targets))
	usedSlots := 0
	for i, t := range r.Targets {
		n := int(float64(maxSlots) * t.Weight)
		if n == 0 && t.Weight > 0 {
			n = 1
		}
		slots[i].i = i
		slots[i].n = n
		usedSlots += n
	}

	sort.Sort(slots)
	targets := make([]*Target, usedSlots)
	for _, s := range slots {
		if s.n <= 0 {
			continue
		}

		next, step := 0, usedSlots/s.n
		for k := 0; k < s.n; k++ {
			// find the next empty slot
			for targets[next] != nil {
				next = (next + 1) % usedSlots
			}

			// use slot and move to next one
			targets[next] = r.Targets[s.i]
			next = (next + step) % usedSlots
		}
	}

	r.wTargets = targets
}

type byN []struct{ i, n int }

func (r byN) Len() int           { return len(r) }
func (r byN) Swap(i, j int)      { r[i], r[j] = r[j], r[i] }
func (r byN) Less(i, j int) bool { return r[i].n < r[j].n }
<|endoftext|>


<commit_before>package main

import (
	"github.com/GeoNet/app/web"
	"github.com/GeoNet/app/web/webtest"
	"net/http"
	"testing"
)

func TestRoutes(t *testing.T) {
	setup()
	defer teardown()

	// GeoJSON routes
	r := webtest.Route{
		Accept:     web.V1GeoJSON,
		Content:    web.V1GeoJSON,
		Cache:      web.MaxAge300,
		Surrogate:  web.MaxAge300,
		Response:   http.StatusOK,
		Vary:       "Accept",
		TestAccept: false,
	}
	r.Add("/site?typeID=t1")
	r.Add("/site")
	r.Add("/site?siteID=TEST1&networkID=TN1")

	r.Test(ts, t)

	// CSV routes
	r = webtest.Route{
		Accept:     web.V1CSV,
		Content:    web.V1CSV,
		Cache:      web.MaxAge300,
		Surrogate:  web.MaxAge300,
		Response:   http.StatusOK,
		Vary:       "Accept",
		TestAccept: false,
	}
	r.Add("/observation?typeID=t1&siteID=TEST1&networkID=TN1")
	r.Add("/observation?typeID=t1&siteID=TEST1&networkID=TN1&methodID=m1")
	r.Add("/observation?typeID=t1&siteID=TEST1&networkID=TN1&methodID=m1&days=400")
	r.Add("/observation?typeID=t1&siteID=TEST1&networkID=TN1&days=400")
	r.Add("/observation?typeID=t1&start=2010-11-24T00:00:00Z&days=2")
	r.Add("/observation?typeID=t1&start=2010-11-24T00:00:00Z&days=2&methodID=m1")
	r.Add("/observation?typeID=t1&start=2010-11-24T00:00:00Z&days=2&within=POLYGON((170.18+-37.52,177.19+-47.52,177.20+-37.53,170.18+-37.52))")
	r.Add("/observation?typeID=t1&start=2010-11-24T00:00:00Z&days=2&within=POLYGON((170.18+-37.52,177.19+-47.52,177.20+-37.53,170.18+-37.52))&methodID=m1")

	r.Test(ts, t)

	// JSON routes
	r = webtest.Route{
		Accept:     web.V1JSON,
		Content:    web.V1JSON,
		Cache:      web.MaxAge300,
		Surrogate:  web.MaxAge300,
		Response:   http.StatusOK,
		Vary:       "Accept",
		TestAccept: false,
	}
	r.Add("/type")
	r.Add("/method?typeID=t1")
	r.Add("/method")

	r.Test(ts, t)

	// plot routes
	r = webtest.Route{
		Accept:     "",
		Content:    "image/svg+xml",
		Cache:      web.MaxAge300,
		Surrogate:  web.MaxAge300,
		Response:   http.StatusOK,
		Vary:       "Accept",
		TestAccept: false,
	}
	r.Add("/plot?typeID=t1&siteID=TEST1&networkID=TN1")
	r.Add("/plot?typeID=t1&siteID=TEST1&networkID=TN1&yrange=12.2")
	r.Add("/plot?typeID=t1&siteID=TEST1&networkID=TN1&days=10000")
	r.Add("/plot?typeID=t1&siteID=TEST1&networkID=TN1&days=10000&yrange=12.2")

	r.Test(ts, t)

	// Plot routes that should bad request
	r = webtest.Route{
		Accept:     "",
		Content:    web.ErrContent,
		Cache:      web.MaxAge10,
		Surrogate:  web.MaxAge86400,
		Response:   http.StatusBadRequest,
		Vary:       "Accept",
		TestAccept: false,
	}
	r.Add("/plot?typeID=t1&siteID=TEST1")
	r.Add("/plot?typeID=t1")
	r.Add("/plot?typeID=t1&siteID=TEST1&networkID=TN1&days=nan")
	r.Add("/plot?typeID=t1&siteID=TEST1&networkID=TN1&days=1000000000000")
	r.Add("/plot?typeID=t1&siteID=TEST1&networkID=TN1&start=")
	r.Add("/plot?typeID=t1&siteID=TEST1&networkID=TN1&yrange=-12.2")
	r.Add("/plot?typeID=t1&siteID=TEST1&networkID=TN1&yrange=0")

	r.Test(ts, t)

	// Routes that should 404
	r = webtest.Route{
		Accept:     web.V1JSON,
		Content:    web.ErrContent,
		Cache:      web.MaxAge10,
		Surrogate:  web.MaxAge10,
		Response:   http.StatusNotFound,
		Vary:       "Accept",
		TestAccept: false,
	}
	r.Add("/observation?typeID=t1&NO=TEST1&networkID=TN1")
	r.Add("/observation?typeID=t1&siteID=NO&networkID=TN1")
	r.Add("/observation?typeID=t1&siteID=TEST1&networkID=NO")
	r.Add("/observation?typeID=t1&siteID=TEST1&networkID=TN1&methodID=m100")
	r.Add("/observation?typeID=t1&siteID=TEST1&networkID=TN1&methodID=m100&days=100")
	r.Add("/observation?typeID=t1&start=2010-11-24T00:00:00Z&days=2&methodID=m100")
	r.Add("/observation?typeID=t1&start=2010-11-24T00:00:00Z&days=2&methodID=m100&within=POLYGON((170.18+-37.52,177.19+-47.52,177.20+-37.53,170.18+-37.52))")

	// r.Test(ts, t)

	// GeoJSON routes that should bad request
	r = webtest.Route{
		Accept:     web.V1GeoJSON,
		Content:    web.ErrContent,
		Cache:      web.MaxAge10,
		Surrogate:  web.MaxAge86400,
		Response:   http.StatusBadRequest,
		Vary:       "Accept",
		TestAccept: false,
	}
	r.Add("/")
	r.Add("/bob")

	r.Test(ts, t)

	// JSON routes that should bad request
	r = webtest.Route{
		Accept:     web.V1JSON,
		Content:    web.ErrContent,
		Cache:      web.MaxAge10,
		Surrogate:  web.MaxAge86400,
		Response:   http.StatusBadRequest,
		Vary:       "Accept",
		TestAccept: false,
	}
	r.Add("/")
	r.Add("/bob")

	r.Test(ts, t)

	// CSV routes that should bad request
	r = webtest.Route{
		Accept:     web.V1CSV,
		Content:    web.ErrContent,
		Cache:      web.MaxAge10,
		Surrogate:  web.MaxAge86400,
		Response:   http.StatusBadRequest,
		Vary:       "Accept",
		TestAccept: false,
	}
	r.Add("/")
	r.Add("/bob")
	r.Add("/observation?typeID=t1&start=2010-11-24T00:00:00Z&days=0")
	r.Add("/observation?typeID=t1&start=2010-11-24T00:00:00Z&days=8")
	r.Add("/observation?typeID=t1&start=2010-11-24T00:00:00Z&days=2&srsName=EPSG:999999")
	r.Add("/observation?typeID=t1&start=2010-11-24T00:00:00Z&days=2&within=POLYGON((177.18+-37.52,177.19+-37.52,177.20+-37.53))")             // no enough points
	r.Add("/observation?typeID=t1&start=2010-11-24T00:00:00Z&days=2&within=POLYGON((177.18+-37.52,177.19+-37.52,177.20+-37.53,178.0+-34.5))") // doesn't close

	r.Test(ts, t)
}

func TestGeoJSON(t *testing.T) {
	setup()
	defer teardown()

	// GeoJSON routes
	r := webtest.Route{
		Accept:     web.V1GeoJSON,
		Content:    web.V1GeoJSON,
		Cache:      web.MaxAge300,
		Surrogate:  web.MaxAge300,
		Response:   http.StatusOK,
		Vary:       "Accept",
		TestAccept: false,
	}
	r.Add("/site?typeID=t1")

	r.GeoJSON(ts, t)
}
<commit_msg>Correct 404 CSV tests.<commit_after>package main

import (
	"github.com/GeoNet/app/web"
	"github.com/GeoNet/app/web/webtest"
	"net/http"
	"testing"
)

func TestRoutes(t *testing.T) {
	setup()
	defer teardown()

	// GeoJSON routes
	r := webtest.Route{
		Accept:     web.V1GeoJSON,
		Content:    web.V1GeoJSON,
		Cache:      web.MaxAge300,
		Surrogate:  web.MaxAge300,
		Response:   http.StatusOK,
		Vary:       "Accept",
		TestAccept: false,
	}
	r.Add("/site?typeID=t1")
	r.Add("/site")
	r.Add("/site?siteID=TEST1&networkID=TN1")

	r.Test(ts, t)

	// CSV routes
	r = webtest.Route{
		Accept:     web.V1CSV,
		Content:    web.V1CSV,
		Cache:      web.MaxAge300,
		Surrogate:  web.MaxAge300,
		Response:   http.StatusOK,
		Vary:       "Accept",
		TestAccept: false,
	}
	r.Add("/observation?typeID=t1&siteID=TEST1&networkID=TN1")
	r.Add("/observation?typeID=t1&siteID=TEST1&networkID=TN1&methodID=m1")
	r.Add("/observation?typeID=t1&siteID=TEST1&networkID=TN1&methodID=m1&days=400")
	r.Add("/observation?typeID=t1&siteID=TEST1&networkID=TN1&days=400")
	r.Add("/observation?typeID=t1&start=2010-11-24T00:00:00Z&days=2")
	r.Add("/observation?typeID=t1&start=2010-11-24T00:00:00Z&days=2&methodID=m1")
	r.Add("/observation?typeID=t1&start=2010-11-24T00:00:00Z&days=2&within=POLYGON((170.18+-37.52,177.19+-47.52,177.20+-37.53,170.18+-37.52))")
	r.Add("/observation?typeID=t1&start=2010-11-24T00:00:00Z&days=2&within=POLYGON((170.18+-37.52,177.19+-47.52,177.20+-37.53,170.18+-37.52))&methodID=m1")

	r.Test(ts, t)

	// JSON routes
	r = webtest.Route{
		Accept:     web.V1JSON,
		Content:    web.V1JSON,
		Cache:      web.MaxAge300,
		Surrogate:  web.MaxAge300,
		Response:   http.StatusOK,
		Vary:       "Accept",
		TestAccept: false,
	}
	r.Add("/type")
	r.Add("/method?typeID=t1")
	r.Add("/method")

	r.Test(ts, t)

	// plot routes
	r = webtest.Route{
		Accept:     "",
		Content:    "image/svg+xml",
		Cache:      web.MaxAge300,
		Surrogate:  web.MaxAge300,
		Response:   http.StatusOK,
		Vary:       "Accept",
		TestAccept: false,
	}
	r.Add("/plot?typeID=t1&siteID=TEST1&networkID=TN1")
	r.Add("/plot?typeID=t1&siteID=TEST1&networkID=TN1&yrange=12.2")
	r.Add("/plot?typeID=t1&siteID=TEST1&networkID=TN1&days=10000")
	r.Add("/plot?typeID=t1&siteID=TEST1&networkID=TN1&days=10000&yrange=12.2")

	r.Test(ts, t)

	// Plot routes that should bad request
	r = webtest.Route{
		Accept:     "",
		Content:    web.ErrContent,
		Cache:      web.MaxAge10,
		Surrogate:  web.MaxAge86400,
		Response:   http.StatusBadRequest,
		Vary:       "Accept",
		TestAccept: false,
	}
	r.Add("/plot?typeID=t1&siteID=TEST1")
	r.Add("/plot?typeID=t1")
	r.Add("/plot?typeID=t1&siteID=TEST1&networkID=TN1&days=nan")
	r.Add("/plot?typeID=t1&siteID=TEST1&networkID=TN1&days=1000000000000")
	r.Add("/plot?typeID=t1&siteID=TEST1&networkID=TN1&start=")
	r.Add("/plot?typeID=t1&siteID=TEST1&networkID=TN1&yrange=-12.2")
	r.Add("/plot?typeID=t1&siteID=TEST1&networkID=TN1&yrange=0")

	r.Test(ts, t)

	// CSV routes that should 404
	r = webtest.Route{
		Accept:     web.V1CSV,
		Content:    web.ErrContent,
		Cache:      web.MaxAge10,
		Surrogate:  web.MaxAge10,
		Response:   http.StatusNotFound,
		Vary:       "Accept",
		TestAccept: false,
	}
	r.Add("/observation?typeID=t1&NO=TEST1&networkID=TN1")
	r.Add("/observation?typeID=t1&siteID=NO&networkID=TN1")
	r.Add("/observation?typeID=t1&siteID=TEST1&networkID=NO")
	r.Add("/observation?typeID=t1&siteID=TEST1&networkID=TN1&methodID=m100")
	r.Add("/observation?typeID=t1&siteID=TEST1&networkID=TN1&methodID=m100&days=100")
	r.Add("/observation?typeID=t1&start=2010-11-24T00:00:00Z&days=2&methodID=m100")
	r.Add("/observation?typeID=t1&start=2010-11-24T00:00:00Z&days=2&methodID=m100&within=POLYGON((170.18+-37.52,177.19+-47.52,177.20+-37.53,170.18+-37.52))")

	// r.Test(ts, t)

	// GeoJSON routes that should bad request
	r = webtest.Route{
		Accept:     web.V1GeoJSON,
		Content:    web.ErrContent,
		Cache:      web.MaxAge10,
		Surrogate:  web.MaxAge86400,
		Response:   http.StatusBadRequest,
		Vary:       "Accept",
		TestAccept: false,
	}
	r.Add("/")
	r.Add("/bob")

	r.Test(ts, t)

	// JSON routes that should bad request
	r = webtest.Route{
		Accept:     web.V1JSON,
		Content:    web.ErrContent,
		Cache:      web.MaxAge10,
		Surrogate:  web.MaxAge86400,
		Response:   http.StatusBadRequest,
		Vary:       "Accept",
		TestAccept: false,
	}
	r.Add("/")
	r.Add("/bob")

	r.Test(ts, t)

	// CSV routes that should bad request
	r = webtest.Route{
		Accept:     web.V1CSV,
		Content:    web.ErrContent,
		Cache:      web.MaxAge10,
		Surrogate:  web.MaxAge86400,
		Response:   http.StatusBadRequest,
		Vary:       "Accept",
		TestAccept: false,
	}
	r.Add("/")
	r.Add("/bob")
	r.Add("/observation?typeID=t1&start=2010-11-24T00:00:00Z&days=0")
	r.Add("/observation?typeID=t1&start=2010-11-24T00:00:00Z&days=8")
	r.Add("/observation?typeID=t1&start=2010-11-24T00:00:00Z&days=2&srsName=EPSG:999999")
	r.Add("/observation?typeID=t1&start=2010-11-24T00:00:00Z&days=2&within=POLYGON((177.18+-37.52,177.19+-37.52,177.20+-37.53))")             // no enough points
	r.Add("/observation?typeID=t1&start=2010-11-24T00:00:00Z&days=2&within=POLYGON((177.18+-37.52,177.19+-37.52,177.20+-37.53,178.0+-34.5))") // doesn't close

	r.Test(ts, t)
}

func TestGeoJSON(t *testing.T) {
	setup()
	defer teardown()

	// GeoJSON routes
	r := webtest.Route{
		Accept:     web.V1GeoJSON,
		Content:    web.V1GeoJSON,
		Cache:      web.MaxAge300,
		Surrogate:  web.MaxAge300,
		Response:   http.StatusOK,
		Vary:       "Accept",
		TestAccept: false,
	}
	r.Add("/site?typeID=t1")

	r.GeoJSON(ts, t)
}
<|endoftext|>


<commit_before>package sched

import (
	"bytes"
	"fmt"
	"log"
	"math"
	"time"

	"github.com/StackExchange/bosun/_third_party/github.com/StackExchange/scollector/collect"
	"github.com/StackExchange/bosun/_third_party/github.com/StackExchange/scollector/opentsdb"
	"github.com/StackExchange/bosun/conf"
	"github.com/StackExchange/bosun/expr"
)

func (s *Schedule) Status(ak AlertKey) *State {
	s.Lock()
	state := s.status[ak]
	if state == nil {
		g := ak.Group()
		state = &State{
			Alert: ak.Name(),
			Tags:  g.Tags(),
			Group: g,
		}
		s.status[ak] = state
	}
	state.Touch()
	s.Unlock()
	return state
}

type RunHistory map[AlertKey]*Event

// Check evaluates all critical and warning alert rules.
func (s *Schedule) Check() {
	r := make(RunHistory)
	s.CheckStart = time.Now().UTC()
	s.cache = opentsdb.NewCache(s.Conf.TsdbHost, s.Conf.ResponseLimit)
	for _, a := range s.Conf.Alerts {
		s.CheckAlert(r, a)
	}
	s.RunHistory(r)
}

// RunHistory processes an event history and triggers notifications if needed.
func (s *Schedule) RunHistory(r RunHistory) {
	checkNotify := false
	silenced := s.Silenced()
	s.Lock()
	defer s.Unlock()
	for ak, event := range r {
		state := s.status[ak]
		last := state.Append(event)
		a := s.Conf.Alerts[ak.Name()]
		if event.Status > StNormal {
			var subject = new(bytes.Buffer)
			if event.Status != StUnknown {
				if err := s.ExecuteSubject(subject, a, state); err != nil {
					log.Println(err)
				}
			}
			state.Subject = subject.String()
			state.Open = true
		}
		// On state increase, clear old notifications and notify current.
		// On state decrease, and if the old alert was already acknowledged, notify current.
		// If the old alert was not acknowledged, do nothing.
		// Do nothing if state did not change.
		notify := func(notifications map[string]*conf.Notification) {
			for _, n := range notifications {
				s.Notify(state, n)
				checkNotify = true
			}
		}
		notifyCurrent := func() {
			state.NeedAck = true
			if _, present := silenced[ak]; present {
				return
			}
			switch event.Status {
			case StCritical, StUnknown:
				notify(a.CritNotification)
			case StWarning:
				notify(a.WarnNotification)
			}
		}
		clearOld := func() {
			state.NeedAck = false
			delete(s.Notifications, ak)
		}
		if event.Status > last {
			clearOld()
			notifyCurrent()
		} else if event.Status < last {
			if _, hasOld := s.Notifications[ak]; hasOld {
				notifyCurrent()
			}
		}
	}
	if checkNotify {
		s.nc <- true
	}
	s.Save()
}

// CheckUnknown checks for unknown alerts.
func (s *Schedule) CheckUnknown() {
	for _ = range time.Tick(s.Conf.CheckFrequency / 4) {
		log.Println("checkUnknown")
		r := make(RunHistory)
		s.Lock()
		for ak, st := range s.status {
			if st.Forgotten {
				continue
			}
			t := s.Conf.Alerts[ak.Name()].Unknown
			if t == 0 {
				t = s.Conf.CheckFrequency * 2
			}
			if t == 0 {
				continue
			}
			if time.Since(st.Touched) < t {
				continue
			}
			r[ak] = &Event{Status: StUnknown}
		}
		s.Unlock()
		s.RunHistory(r)
	}
}

func (s *Schedule) CheckAlert(r RunHistory, a *conf.Alert) {
	log.Printf("checking alert %v", a.Name)
	start := time.Now()
	var warns AlertKeys
	crits, err := s.CheckExpr(r, a, a.Crit, StCritical, nil)
	if err == nil {
		warns, _ = s.CheckExpr(r, a, a.Warn, StWarning, crits)
	}
	log.Printf("done checking alert %v (%s): %v crits, %v warns", a.Name, time.Since(start), len(crits), len(warns))
}

func (s *Schedule) CheckExpr(rh RunHistory, a *conf.Alert, e *expr.Expr, checkStatus Status, ignore AlertKeys) (alerts AlertKeys, err error) {
	if e == nil {
		return
	}
	defer func() {
		if err == nil {
			return
		}
		collect.Add("check.errs", opentsdb.TagSet{"metric": a.Name}, 1)
		log.Println(err)
	}()
	results, _, err := e.ExecuteOpts(s.cache, nil, s.CheckStart, 0)
	if err != nil {
		ak := NewAlertKey(a.Name, nil)
		state := s.Status(ak)
		state.Result = &Result{
			Result: &expr.Result{
				Computations: []expr.Computation{
					{
						Text:  e.String(),
						Value: err.Error(),
					},
				},
			},
		}
		rh[ak] = &Event{
			Status: StError,
		}
		return
	}
Loop:
	for _, r := range results {
		if s.Conf.Squelched(a, r.Group) {
			continue
		}
		ak := NewAlertKey(a.Name, r.Group)
		for _, v := range ignore {
			if ak == v {
				continue Loop
			}
		}
		state := s.Status(ak)
		status := checkStatus
		var n float64
		switch v := r.Value.(type) {
		case expr.Number:
			n = float64(v)
		case expr.Scalar:
			n = float64(v)
		default:
			err = fmt.Errorf("expected number or scalar")
			return
		}
		event := rh[ak]
		if event == nil {
			event = new(Event)
			rh[ak] = event
		}
		result := Result{
			Result: r,
			Expr:   e.String(),
		}
		switch checkStatus {
		case StWarning:
			event.Warn = &result
		case StCritical:
			event.Crit = &result
		}
		if math.IsNaN(n) {
			status = StError
		} else if n == 0 {
			status = StNormal
		}
		if status != StNormal {
			alerts = append(alerts, ak)
		}
		if status > rh[ak].Status {
			event.Status = status
			state.Result = &result
		}
	}
	return
}
<commit_msg>collect evaluation time for each alert<commit_after>package sched

import (
	"bytes"
	"fmt"
	"log"
	"math"
	"time"

	"github.com/StackExchange/bosun/_third_party/github.com/StackExchange/scollector/collect"
	"github.com/StackExchange/bosun/_third_party/github.com/StackExchange/scollector/opentsdb"
	"github.com/StackExchange/bosun/conf"
	"github.com/StackExchange/bosun/expr"
)

func (s *Schedule) Status(ak AlertKey) *State {
	s.Lock()
	state := s.status[ak]
	if state == nil {
		g := ak.Group()
		state = &State{
			Alert: ak.Name(),
			Tags:  g.Tags(),
			Group: g,
		}
		s.status[ak] = state
	}
	state.Touch()
	s.Unlock()
	return state
}

type RunHistory map[AlertKey]*Event

// Check evaluates all critical and warning alert rules.
func (s *Schedule) Check() {
	r := make(RunHistory)
	s.CheckStart = time.Now().UTC()
	s.cache = opentsdb.NewCache(s.Conf.TsdbHost, s.Conf.ResponseLimit)
	for _, a := range s.Conf.Alerts {
		s.CheckAlert(r, a)
	}
	s.RunHistory(r)
}

// RunHistory processes an event history and triggers notifications if needed.
func (s *Schedule) RunHistory(r RunHistory) {
	checkNotify := false
	silenced := s.Silenced()
	s.Lock()
	defer s.Unlock()
	for ak, event := range r {
		state := s.status[ak]
		last := state.Append(event)
		a := s.Conf.Alerts[ak.Name()]
		if event.Status > StNormal {
			var subject = new(bytes.Buffer)
			if event.Status != StUnknown {
				if err := s.ExecuteSubject(subject, a, state); err != nil {
					log.Println(err)
				}
			}
			state.Subject = subject.String()
			state.Open = true
		}
		// On state increase, clear old notifications and notify current.
		// On state decrease, and if the old alert was already acknowledged, notify current.
		// If the old alert was not acknowledged, do nothing.
		// Do nothing if state did not change.
		notify := func(notifications map[string]*conf.Notification) {
			for _, n := range notifications {
				s.Notify(state, n)
				checkNotify = true
			}
		}
		notifyCurrent := func() {
			state.NeedAck = true
			if _, present := silenced[ak]; present {
				return
			}
			switch event.Status {
			case StCritical, StUnknown:
				notify(a.CritNotification)
			case StWarning:
				notify(a.WarnNotification)
			}
		}
		clearOld := func() {
			state.NeedAck = false
			delete(s.Notifications, ak)
		}
		if event.Status > last {
			clearOld()
			notifyCurrent()
		} else if event.Status < last {
			if _, hasOld := s.Notifications[ak]; hasOld {
				notifyCurrent()
			}
		}
	}
	if checkNotify {
		s.nc <- true
	}
	s.Save()
}

// CheckUnknown checks for unknown alerts.
func (s *Schedule) CheckUnknown() {
	for _ = range time.Tick(s.Conf.CheckFrequency / 4) {
		log.Println("checkUnknown")
		r := make(RunHistory)
		s.Lock()
		for ak, st := range s.status {
			if st.Forgotten {
				continue
			}
			t := s.Conf.Alerts[ak.Name()].Unknown
			if t == 0 {
				t = s.Conf.CheckFrequency * 2
			}
			if t == 0 {
				continue
			}
			if time.Since(st.Touched) < t {
				continue
			}
			r[ak] = &Event{Status: StUnknown}
		}
		s.Unlock()
		s.RunHistory(r)
	}
}

func (s *Schedule) CheckAlert(r RunHistory, a *conf.Alert) {
	log.Printf("checking alert %v", a.Name)
	start := time.Now()
	var warns AlertKeys
	crits, err := s.CheckExpr(r, a, a.Crit, StCritical, nil)
	if err == nil {
		warns, _ = s.CheckExpr(r, a, a.Warn, StWarning, crits)
	}
	collect.Put("check.duration", opentsdb.TagSet{"name": a.Name}, time.Since(start).Seconds())
	log.Printf("done checking alert %v (%s): %v crits, %v warns", a.Name, time.Since(start), len(crits), len(warns))
}

func (s *Schedule) CheckExpr(rh RunHistory, a *conf.Alert, e *expr.Expr, checkStatus Status, ignore AlertKeys) (alerts AlertKeys, err error) {
	if e == nil {
		return
	}
	defer func() {
		if err == nil {
			return
		}
		collect.Add("check.errs", opentsdb.TagSet{"metric": a.Name}, 1)
		log.Println(err)
	}()
	results, _, err := e.ExecuteOpts(s.cache, nil, s.CheckStart, 0)
	if err != nil {
		ak := NewAlertKey(a.Name, nil)
		state := s.Status(ak)
		state.Result = &Result{
			Result: &expr.Result{
				Computations: []expr.Computation{
					{
						Text:  e.String(),
						Value: err.Error(),
					},
				},
			},
		}
		rh[ak] = &Event{
			Status: StError,
		}
		return
	}
Loop:
	for _, r := range results {
		if s.Conf.Squelched(a, r.Group) {
			continue
		}
		ak := NewAlertKey(a.Name, r.Group)
		for _, v := range ignore {
			if ak == v {
				continue Loop
			}
		}
		state := s.Status(ak)
		status := checkStatus
		var n float64
		switch v := r.Value.(type) {
		case expr.Number:
			n = float64(v)
		case expr.Scalar:
			n = float64(v)
		default:
			err = fmt.Errorf("expected number or scalar")
			return
		}
		event := rh[ak]
		if event == nil {
			event = new(Event)
			rh[ak] = event
		}
		result := Result{
			Result: r,
			Expr:   e.String(),
		}
		switch checkStatus {
		case StWarning:
			event.Warn = &result
		case StCritical:
			event.Crit = &result
		}
		if math.IsNaN(n) {
			status = StError
		} else if n == 0 {
			status = StNormal
		}
		if status != StNormal {
			alerts = append(alerts, ak)
		}
		if status > rh[ak].Status {
			event.Status = status
			state.Result = &result
		}
	}
	return
}
<|endoftext|>


<commit_before>package sched

import (
	"bytes"
	"fmt"
	"log"
	"math"
	"time"

	"github.com/bosun-monitor/bosun/_third_party/github.com/MiniProfiler/go/miniprofiler"
	"github.com/bosun-monitor/bosun/_third_party/github.com/bosun-monitor/collect"
	"github.com/bosun-monitor/bosun/_third_party/github.com/bosun-monitor/opentsdb"
	"github.com/bosun-monitor/bosun/conf"
	"github.com/bosun-monitor/bosun/expr"
)

func (s *Schedule) Status(ak expr.AlertKey) *State {
	s.Lock()
	state := s.status[ak]
	if state == nil {
		g := ak.Group()
		state = &State{
			Alert: ak.Name(),
			Tags:  g.Tags(),
			Group: g,
		}
		s.status[ak] = state
	}
	s.Unlock()
	return state
}

type RunHistory struct {
	Start   time.Time
	Context opentsdb.Context
	Events  map[expr.AlertKey]*Event
}

func (s *Schedule) NewRunHistory(start time.Time) *RunHistory {
	return &RunHistory{
		Start:   start,
		Context: opentsdb.NewCache(s.Conf.TsdbHost, s.Conf.ResponseLimit),
		Events:  make(map[expr.AlertKey]*Event),
	}
}

// Check evaluates all critical and warning alert rules. An error is returned if
// the check could not be performed.
func (s *Schedule) Check(T miniprofiler.Timer, now time.Time) (time.Duration, error) {
	select {
	case s.checkRunning <- true:
		// Good, we've got the lock.
	default:
		return 0, fmt.Errorf("check already running")
	}
	r := s.NewRunHistory(now)
	start := time.Now()
	for _, a := range s.Conf.Alerts {
		s.CheckAlert(T, r, a)
	}
	d := time.Since(start)
	s.RunHistory(r)
	<-s.checkRunning
	return d, nil
}

// RunHistory processes an event history and trisggers notifications if needed.
func (s *Schedule) RunHistory(r *RunHistory) {
	checkNotify := false
	silenced := s.Silenced()
	s.Lock()
	defer s.Unlock()
	for ak, event := range r.Events {
		state := s.status[ak]
		last := state.Append(event)
		a := s.Conf.Alerts[ak.Name()]
		if event.Status > StNormal {
			var subject = new(bytes.Buffer)
			if event.Status != StUnknown {
				if err := s.ExecuteSubject(subject, r, a, state); err != nil {
					log.Println(err)
				}
			}
			state.Subject = subject.String()
			state.Open = true
		}
		// On state increase, clear old notifications and notify current.
		// On state decrease, and if the old alert was already acknowledged, notify current.
		// If the old alert was not acknowledged, do nothing.
		// Do nothing if state did not change.
		notify := func(ns *conf.Notifications) {
			nots := ns.Get(s.Conf, state.Group)
			for _, n := range nots {
				s.Notify(state, n)
				checkNotify = true
			}
		}
		notifyCurrent := func() {
			// Auto close ignoreUnknowns.
			if a.IgnoreUnknown && event.Status == StUnknown {
				state.Open = false
				state.NeedAck = false
				state.Action("bosun", "Auto close because alert has ignoreUnknown.", ActionClose)
				log.Printf("auto close %s because alert has ignoreUnknown", ak)
				return
			}
			state.NeedAck = true
			switch event.Status {
			case StCritical, StUnknown:
				notify(a.CritNotification)
			case StWarning:
				notify(a.WarnNotification)
			}
		}
		clearOld := func() {
			state.NeedAck = false
			delete(s.Notifications, ak)
		}
		if event.Status > last {
			clearOld()
			notifyCurrent()
		} else if event.Status < last {
			if _, hasOld := s.Notifications[ak]; hasOld {
				notifyCurrent()
			}
			// Auto close silenced alerts.
			if _, ok := silenced[ak]; ok && event.Status == StNormal {
				go func(ak expr.AlertKey) {
					log.Printf("auto close %s because was silenced", ak)
					err := s.Action("bosun", "Auto close because was silenced.", ActionClose, ak)
					if err != nil {
						log.Println(err)
					}
				}(ak)
			}
		}
	}
	if checkNotify && s.nc != nil {
		s.nc <- true
	}
	s.Save()
}

// CheckUnknown checks for unknown alerts.
func (s *Schedule) CheckUnknown() {
	for _ = range time.Tick(s.Conf.CheckFrequency / 4) {
		log.Println("checkUnknown")
		r := s.NewRunHistory(time.Now())
		s.Lock()
		for ak, st := range s.status {
			if st.Forgotten {
				continue
			}
			a := s.Conf.Alerts[ak.Name()]
			t := a.Unknown
			if t == 0 {
				t = s.Conf.CheckFrequency * 2
			}
			if t == 0 {
				continue
			}
			if time.Since(st.Touched) < t {
				continue
			}
			r.Events[ak] = &Event{Status: StUnknown}
		}
		s.Unlock()
		s.RunHistory(r)
	}
}

func (s *Schedule) CheckAlert(T miniprofiler.Timer, r *RunHistory, a *conf.Alert) {
	log.Printf("checking alert %v", a.Name)
	start := time.Now()
	var warns expr.AlertKeys
	crits, err := s.CheckExpr(T, r, a, a.Crit, StCritical, nil)
	if err == nil {
		warns, _ = s.CheckExpr(T, r, a, a.Warn, StWarning, crits)
	}
	collect.Put("check.duration", opentsdb.TagSet{"name": a.Name}, time.Since(start).Seconds())
	log.Printf("done checking alert %v (%s): %v crits, %v warns", a.Name, time.Since(start), len(crits), len(warns))
}

func (s *Schedule) CheckExpr(T miniprofiler.Timer, rh *RunHistory, a *conf.Alert, e *expr.Expr, checkStatus Status, ignore expr.AlertKeys) (alerts expr.AlertKeys, err error) {
	if e == nil {
		return
	}
	defer func() {
		if err == nil {
			return
		}
		collect.Add("check.errs", opentsdb.TagSet{"metric": a.Name}, 1)
		log.Println(err)
	}()
	results, _, err := e.Execute(rh.Context, T, rh.Start, 0, a.UnjoinedOK, s.Search, s.Conf.GetLookups(), s.Conf.AlertSquelched(a))
	if err != nil {
		ak := expr.NewAlertKey(a.Name, nil)
		state := s.Status(ak)
		state.Touch()
		state.Result = &Result{
			Result: &expr.Result{
				Computations: []expr.Computation{
					{
						Text:  e.String(),
						Value: err.Error(),
					},
				},
			},
		}
		rh.Events[ak] = &Event{
			Status: StError,
		}
		return
	}
Loop:
	for _, r := range results.Results {
		if s.Conf.Squelched(a, r.Group) {
			continue
		}
		ak := expr.NewAlertKey(a.Name, r.Group)
		for _, v := range ignore {
			if ak == v {
				continue Loop
			}
		}
		state := s.Status(ak)
		state.Touch()
		status := checkStatus
		var n float64
		switch v := r.Value.(type) {
		case expr.Number:
			n = float64(v)
		case expr.Scalar:
			n = float64(v)
		default:
			err = fmt.Errorf("expected number or scalar")
			return
		}
		event := rh.Events[ak]
		if event == nil {
			event = new(Event)
			rh.Events[ak] = event
		}
		result := Result{
			Result: r,
			Expr:   e.String(),
		}
		switch checkStatus {
		case StWarning:
			event.Warn = &result
		case StCritical:
			event.Crit = &result
		}
		if math.IsNaN(n) {
			status = StError
		} else if n == 0 {
			status = StNormal
		}
		if status != StNormal {
			alerts = append(alerts, ak)
		}
		if status > rh.Events[ak].Status {
			event.Status = status
			state.Result = &result
		}
	}
	return
}
<commit_msg>Also forget ignoreUnknowns<commit_after>package sched

import (
	"bytes"
	"fmt"
	"log"
	"math"
	"time"

	"github.com/bosun-monitor/bosun/_third_party/github.com/MiniProfiler/go/miniprofiler"
	"github.com/bosun-monitor/bosun/_third_party/github.com/bosun-monitor/collect"
	"github.com/bosun-monitor/bosun/_third_party/github.com/bosun-monitor/opentsdb"
	"github.com/bosun-monitor/bosun/conf"
	"github.com/bosun-monitor/bosun/expr"
)

func (s *Schedule) Status(ak expr.AlertKey) *State {
	s.Lock()
	state := s.status[ak]
	if state == nil {
		g := ak.Group()
		state = &State{
			Alert: ak.Name(),
			Tags:  g.Tags(),
			Group: g,
		}
		s.status[ak] = state
	}
	s.Unlock()
	return state
}

type RunHistory struct {
	Start   time.Time
	Context opentsdb.Context
	Events  map[expr.AlertKey]*Event
}

func (s *Schedule) NewRunHistory(start time.Time) *RunHistory {
	return &RunHistory{
		Start:   start,
		Context: opentsdb.NewCache(s.Conf.TsdbHost, s.Conf.ResponseLimit),
		Events:  make(map[expr.AlertKey]*Event),
	}
}

// Check evaluates all critical and warning alert rules. An error is returned if
// the check could not be performed.
func (s *Schedule) Check(T miniprofiler.Timer, now time.Time) (time.Duration, error) {
	select {
	case s.checkRunning <- true:
		// Good, we've got the lock.
	default:
		return 0, fmt.Errorf("check already running")
	}
	r := s.NewRunHistory(now)
	start := time.Now()
	for _, a := range s.Conf.Alerts {
		s.CheckAlert(T, r, a)
	}
	d := time.Since(start)
	s.RunHistory(r)
	<-s.checkRunning
	return d, nil
}

// RunHistory processes an event history and trisggers notifications if needed.
func (s *Schedule) RunHistory(r *RunHistory) {
	checkNotify := false
	silenced := s.Silenced()
	s.Lock()
	defer s.Unlock()
	for ak, event := range r.Events {
		state := s.status[ak]
		last := state.Append(event)
		a := s.Conf.Alerts[ak.Name()]
		if event.Status > StNormal {
			var subject = new(bytes.Buffer)
			if event.Status != StUnknown {
				if err := s.ExecuteSubject(subject, r, a, state); err != nil {
					log.Println(err)
				}
			}
			state.Subject = subject.String()
			state.Open = true
		}
		// On state increase, clear old notifications and notify current.
		// On state decrease, and if the old alert was already acknowledged, notify current.
		// If the old alert was not acknowledged, do nothing.
		// Do nothing if state did not change.
		notify := func(ns *conf.Notifications) {
			nots := ns.Get(s.Conf, state.Group)
			for _, n := range nots {
				s.Notify(state, n)
				checkNotify = true
			}
		}
		notifyCurrent := func() {
			// Auto close ignoreUnknowns.
			if a.IgnoreUnknown && event.Status == StUnknown {
				state.Open = false
				state.Forgotten = true
				state.NeedAck = false
				state.Action("bosun", "Auto close because alert has ignoreUnknown.", ActionClose)
				log.Printf("auto close %s because alert has ignoreUnknown", ak)
				return
			}
			state.NeedAck = true
			switch event.Status {
			case StCritical, StUnknown:
				notify(a.CritNotification)
			case StWarning:
				notify(a.WarnNotification)
			}
		}
		clearOld := func() {
			state.NeedAck = false
			delete(s.Notifications, ak)
		}
		if event.Status > last {
			clearOld()
			notifyCurrent()
		} else if event.Status < last {
			if _, hasOld := s.Notifications[ak]; hasOld {
				notifyCurrent()
			}
			// Auto close silenced alerts.
			if _, ok := silenced[ak]; ok && event.Status == StNormal {
				go func(ak expr.AlertKey) {
					log.Printf("auto close %s because was silenced", ak)
					err := s.Action("bosun", "Auto close because was silenced.", ActionClose, ak)
					if err != nil {
						log.Println(err)
					}
				}(ak)
			}
		}
	}
	if checkNotify && s.nc != nil {
		s.nc <- true
	}
	s.Save()
}

// CheckUnknown checks for unknown alerts.
func (s *Schedule) CheckUnknown() {
	for _ = range time.Tick(s.Conf.CheckFrequency / 4) {
		log.Println("checkUnknown")
		r := s.NewRunHistory(time.Now())
		s.Lock()
		for ak, st := range s.status {
			if st.Forgotten {
				continue
			}
			a := s.Conf.Alerts[ak.Name()]
			t := a.Unknown
			if t == 0 {
				t = s.Conf.CheckFrequency * 2
			}
			if t == 0 {
				continue
			}
			if time.Since(st.Touched) < t {
				continue
			}
			r.Events[ak] = &Event{Status: StUnknown}
		}
		s.Unlock()
		s.RunHistory(r)
	}
}

func (s *Schedule) CheckAlert(T miniprofiler.Timer, r *RunHistory, a *conf.Alert) {
	log.Printf("checking alert %v", a.Name)
	start := time.Now()
	var warns expr.AlertKeys
	crits, err := s.CheckExpr(T, r, a, a.Crit, StCritical, nil)
	if err == nil {
		warns, _ = s.CheckExpr(T, r, a, a.Warn, StWarning, crits)
	}
	collect.Put("check.duration", opentsdb.TagSet{"name": a.Name}, time.Since(start).Seconds())
	log.Printf("done checking alert %v (%s): %v crits, %v warns", a.Name, time.Since(start), len(crits), len(warns))
}

func (s *Schedule) CheckExpr(T miniprofiler.Timer, rh *RunHistory, a *conf.Alert, e *expr.Expr, checkStatus Status, ignore expr.AlertKeys) (alerts expr.AlertKeys, err error) {
	if e == nil {
		return
	}
	defer func() {
		if err == nil {
			return
		}
		collect.Add("check.errs", opentsdb.TagSet{"metric": a.Name}, 1)
		log.Println(err)
	}()
	results, _, err := e.Execute(rh.Context, T, rh.Start, 0, a.UnjoinedOK, s.Search, s.Conf.GetLookups(), s.Conf.AlertSquelched(a))
	if err != nil {
		ak := expr.NewAlertKey(a.Name, nil)
		state := s.Status(ak)
		state.Touch()
		state.Result = &Result{
			Result: &expr.Result{
				Computations: []expr.Computation{
					{
						Text:  e.String(),
						Value: err.Error(),
					},
				},
			},
		}
		rh.Events[ak] = &Event{
			Status: StError,
		}
		return
	}
Loop:
	for _, r := range results.Results {
		if s.Conf.Squelched(a, r.Group) {
			continue
		}
		ak := expr.NewAlertKey(a.Name, r.Group)
		for _, v := range ignore {
			if ak == v {
				continue Loop
			}
		}
		state := s.Status(ak)
		state.Touch()
		status := checkStatus
		var n float64
		switch v := r.Value.(type) {
		case expr.Number:
			n = float64(v)
		case expr.Scalar:
			n = float64(v)
		default:
			err = fmt.Errorf("expected number or scalar")
			return
		}
		event := rh.Events[ak]
		if event == nil {
			event = new(Event)
			rh.Events[ak] = event
		}
		result := Result{
			Result: r,
			Expr:   e.String(),
		}
		switch checkStatus {
		case StWarning:
			event.Warn = &result
		case StCritical:
			event.Crit = &result
		}
		if math.IsNaN(n) {
			status = StError
		} else if n == 0 {
			status = StNormal
		}
		if status != StNormal {
			alerts = append(alerts, ak)
		}
		if status > rh.Events[ak].Status {
			event.Status = status
			state.Result = &result
		}
	}
	return
}
<|endoftext|>


<commit_before>package db_test

import (
	"os"
	"testing"
	"time"

	"github.com/ariejan/firedragon/server/db"
	"github.com/ariejan/firedragon/server/model"
)

var database = &db.DB{}

func TestGetItem(t *testing.T) {
	item, err := database.GetItem("url")

	if err != nil {
		t.Error(err)
	}

	if item.Code != "url" {
		t.Error("GetItem() did not return an Item with the proper ID.")
	}
}

func TestItemExists(t *testing.T) {
	if !database.DoesItemExist("url") {
		t.Error("Expected item 'url' to exist.")
	}

	if database.DoesItemExist("not-found") {
		t.Error("Expected item 'not found' not to exist.")
	}
}

func TestGetItems(t *testing.T) {
	items, err := database.GetItems()

	if err != nil {
		t.Error(err)
	}

	if len(items) != 2 {
		t.Errorf("Expected to find 2 items, got %d instead.", len(items))
	}
}

func TestGetItemNotFound(t *testing.T) {
	item, err := database.GetItem("nope")

	if item != nil {
		t.Error("Expected item to be `nil` when it does not exist.")
	}

	if err == nil {
		t.Error("Expected an error when unable to find an Item by ID.")
	}
}

func TestSaveItem(t *testing.T) {
	var err error
	var actual *model.Item

	item := &model.Item{
		Code:      "bam",
		Content:   "Lorem Ipsum",
		Type:      model.TextItemType,
		CreatedAt: time.Now(),
	}

	_, err = database.SaveItem(item)

	if err != nil {
		t.Error("Did not expect an error saving Item", err)
	}

	actual, err = database.GetItem("bam")
	if actual == nil || err != nil {
		t.Error("Expected to be able to retrieve just saved Item.")
	}

	database.DeleteItem(item.Code)
}

func TestSaveItemAutoShortCode(t *testing.T) {
	var err error
	var actual *model.Item

	database.SaveItem(&model.Item{
		Code:      "1",
		Content:   "Original 1",
		Type:      model.TextItemType,
		CreatedAt: time.Now(),
	})
	database.SaveItem(&model.Item{
		Code:      "2",
		Content:   "Original 2",
		Type:      model.TextItemType,
		CreatedAt: time.Now(),
	})

	item1 := &model.Item{
		Code:      "",
		Content:   "Lorem Ipsum",
		Type:      model.TextItemType,
		CreatedAt: time.Now(),
	}

	item2 := &model.Item{
		Content:   "Lorem Ipsum",
		Type:      model.TextItemType,
		CreatedAt: time.Now(),
	}

	actual, err = database.SaveItem(item1)

	if err != nil {
		t.Error("Did not expect an error saving Item", err)
	}

	if actual.Code != "3" {
		t.Error("Expected saveItem to auto-generate shortcode '3', but got '%s' instead.", err, actual.Code)
	}

	actual, err = database.SaveItem(item2)

	if err != nil {
		t.Error("Did not expect an error saving Item", err)
	}

	if actual.Code != "4" {
		t.Error("Expected saveItem to auto-generate shortcode '4', but got '%s' instead.", err, actual.Code)
	}
}

func TestDeleteItem(t *testing.T) {
	database.SaveItem(&model.Item{
		Code:      "bam",
		Content:   "Lorem Ipsum",
		Type:      model.TextItemType,
		CreatedAt: time.Now(),
	})

	if !database.DoesItemExist("bam") {
		t.Error("Expected 'bam' to be saved to database.")
	}

	err := database.DeleteItem("bam")

	if err != nil {
		t.Error("failed deleting item")
		t.Error(err)
	}

	if database.DoesItemExist("bam") {
		t.Error("Expected 'bam' to be deleted.")
	}
}

func setup() {
	database.Open("testing.db")

	items := []*model.Item{
		&model.Item{Code: "url", Type: model.URLItemType, Content: "https://ariejan.net", CreatedAt: time.Now()},
		&model.Item{Code: "txt", Type: model.TextItemType, Content: "Lorem ipsum", CreatedAt: time.Now()},
	}

	for _, item := range items {
		database.SaveItem(item)
	}
}

func teardown() {
	os.Remove(database.Path())
	database.Close()
}

func TestMain(m *testing.M) {
	setup()

	retCode := m.Run()

	teardown()

	os.Exit(retCode)
}
<commit_msg>Add comments to tests<commit_after>package db_test

import (
	"os"
	"testing"
	"time"

	"github.com/ariejan/firedragon/server/db"
	"github.com/ariejan/firedragon/server/model"
)

var database = &db.DB{}

func TestGetItem(t *testing.T) {
	item, err := database.GetItem("url")

	if err != nil {
		t.Error(err)
	}

	if item.Code != "url" {
		t.Error("GetItem() did not return an Item with the proper ID.")
	}
}

func TestItemExists(t *testing.T) {
	if !database.DoesItemExist("url") {
		t.Error("Expected item 'url' to exist.")
	}

	if database.DoesItemExist("not-found") {
		t.Error("Expected item 'not found' not to exist.")
	}
}

func TestGetItems(t *testing.T) {
	items, err := database.GetItems()

	if err != nil {
		t.Error(err)
	}

	if len(items) != 2 {
		t.Errorf("Expected to find 2 items, got %d instead.", len(items))
	}
}

func TestGetItemNotFound(t *testing.T) {
	item, err := database.GetItem("nope")

	if item != nil {
		t.Error("Expected item to be `nil` when it does not exist.")
	}

	if err == nil {
		t.Error("Expected an error when unable to find an Item by ID.")
	}
}

func TestSaveItem(t *testing.T) {
	var err error
	var actual *model.Item

	item := &model.Item{
		Code:      "bam",
		Content:   "Lorem Ipsum",
		Type:      model.TextItemType,
		CreatedAt: time.Now(),
	}

	_, err = database.SaveItem(item)

	if err != nil {
		t.Error("Did not expect an error saving Item", err)
	}

	actual, err = database.GetItem("bam")
	if actual == nil || err != nil {
		t.Error("Expected to be able to retrieve just saved Item.")
	}

	database.DeleteItem(item.Code)
}

func TestSaveItemAutoShortCode(t *testing.T) {
	var err error
	var actual *model.Item

	// Bolt's NextSequence starts at 1 by default, let's assume "1" and "2"
	// have already been taken.
	database.SaveItem(&model.Item{
		Code:      "1",
		Content:   "Original 1",
		Type:      model.TextItemType,
		CreatedAt: time.Now(),
	})
	database.SaveItem(&model.Item{
		Code:      "2",
		Content:   "Original 2",
		Type:      model.TextItemType,
		CreatedAt: time.Now(),
	})

	// Test items with explicit and implicit blank codes
	item1 := &model.Item{
		Code:      "",
		Content:   "Lorem Ipsum",
		Type:      model.TextItemType,
		CreatedAt: time.Now(),
	}

	item2 := &model.Item{
		Content:   "Lorem Ipsum",
		Type:      model.TextItemType,
		CreatedAt: time.Now(),
	}

	// Test with explicit blank value
	actual, err = database.SaveItem(item1)
	if err != nil {
		t.Error("Did not expect an error saving Item", err)
	}
	if actual.Code != "3" {
		t.Error("Expected saveItem to auto-generate shortcode '3', but got '%s' instead.", err, actual.Code)
	}

	// TEst with implicit blank value
	actual, err = database.SaveItem(item2)
	if err != nil {
		t.Error("Did not expect an error saving Item", err)
	}
	if actual.Code != "4" {
		t.Error("Expected saveItem to auto-generate shortcode '4', but got '%s' instead.", err, actual.Code)
	}
}

func TestDeleteItem(t *testing.T) {
	database.SaveItem(&model.Item{
		Code:      "bam",
		Content:   "Lorem Ipsum",
		Type:      model.TextItemType,
		CreatedAt: time.Now(),
	})

	if !database.DoesItemExist("bam") {
		t.Error("Expected 'bam' to be saved to database.")
	}

	err := database.DeleteItem("bam")

	if err != nil {
		t.Error("failed deleting item")
		t.Error(err)
	}

	if database.DoesItemExist("bam") {
		t.Error("Expected 'bam' to be deleted.")
	}
}

func setup() {
	database.Open("testing.db")

	items := []*model.Item{
		&model.Item{Code: "url", Type: model.URLItemType, Content: "https://ariejan.net", CreatedAt: time.Now()},
		&model.Item{Code: "txt", Type: model.TextItemType, Content: "Lorem ipsum", CreatedAt: time.Now()},
	}

	for _, item := range items {
		database.SaveItem(item)
	}
}

func teardown() {
	os.Remove(database.Path())
	database.Close()
}

func TestMain(m *testing.M) {
	setup()

	retCode := m.Run()

	teardown()

	os.Exit(retCode)
}
<|endoftext|>


<commit_before>package lumberjack

import (
  "os" // for File and friends
  "fmt"
  "bytes"
  "io"
  "bufio"
  proto "code.google.com/p/goprotobuf/proto"
  "time"
)

type Harvester struct {
  Path string /* the file path to harvest */

  file os.File /* the file being watched */
}

func (h *Harvester) Harvest(output chan *FileEvent) {
  // TODO(sissel): Read the file
  // TODO(sissel): Emit FileEvent for each line to 'output'
  // TODO(sissel): Handle rotation
  // TODO(sissel): Sleep when there's nothing to do
  // TODO(sissel): Quit if we think the file is dead (file dev/inode changed, no data in X seconds)

  fmt.Printf("Starting harvester: %s\n", h.Path)

  file := h.open()
  defer file.Close()
  info := file.Stat()

  // TODO(sissel): Ask the registrar for the start position?
  // TODO(sissel): record the current file inode/device/etc

  var line uint64 = 0 // Ask registrar about the line number

  // get current offset in file
  offset, _ := file.Seek(0, os.SEEK_CUR)

  // TODO(sissel): Make the buffer size tunable at start-time
  reader := bufio.NewReaderSize(file, 16<<10) // 16kb buffer by default

  var read_timeout = 10 * time.Second
  last_read_time = time.Now()
  for {
    text, err := h.readline(reader, read_timeout)

    if err != nil {
      if err == io.EOF {
        // timed out waiting for data, got eof.
        // TODO(sissel): Check to see if the file was truncated
        // TODO(sissel): if last_read_time was more than 24 hours ago
        if age = time.Since(last_read_time); age > (24 * time.Hour) {
          // This file is idle for more than 24 hours. Give up and stop harvesting.
          fmt.Printf("Stopping harvest of %s; last change was %d seconds ago\n", h.Path, age.Seconds())
          return
        }
        continue
      } else {
        fmt.Printf("Unexpected state reading from %s; error: %s\n", h.Path, err)
        return
      }
    }
    last_read_time = time.Now()

    line++
    event := &FileEvent{
      Source: proto.String(h.Path),
      Offset: proto.Uint64(uint64(offset)),
      Line: proto.Uint64(line),
      Text: text,
    }
    offset += int64(len(*event.Text)) + 1  // +1 because of the line terminator

    output <- event // ship the new event downstream
  } /* forever */
}

func (h *Harvester) open() *os.File {
  var file *os.File

  // Special handling that "-" means to read from standard input
  if h.Path == "-" {
    return os.Stdin
  }

  for {
    var err error
    file, err = os.Open(h.Path)

    if err != nil {
      // retry on failure.
      fmt.Printf("Failed opening %s: %s\n", h.Path, err)
      time.Sleep(5 * time.Second)
    } else {
      break
    }
  }

  // TODO(sissel): In the future, use the registrary to determine where to seek.
  // TODO(sissel): Only seek if the file is a file, not a pipe or socket.
  file.Seek(0, os.SEEK_END)

  return file
}

func (h *Harvester) readline(reader *bufio.Reader, eof_timeout time.Duration) (*string, error) {
  var buffer bytes.Buffer
  start_time = time.Now()
  for {
    segment, is_partial, err := reader.ReadLine()

    if err != nil {
      // TODO(sissel): if eof and line_complete is false, don't check rotation unless a very long time has passed
      if err == io.EOF {
        time.Sleep(1 * time.Second) // TODO(sissel): Implement backoff

        // Give up waiting for data after a certain amount of time.
        // If we time out, return the error (eof)
        if time.Since(start_time) > eof_timeout {
          return nil, err
        }
        continue
      } else {
        fmt.Println(err)
        return nil, err // TODO(sissel): don't do this?
      }
    }

    // TODO(sissel): if buffer exceeds a certain length, maybe report an error condition? chop it?
    buffer.Write(segment)

    if !is_partial {
      // If we got a full line, return the whole line.
      str := new(string)
      *str = buffer.String()
      return str, nil
    }
  } /* forever read chunks */

  return nil, nil
}
<commit_msg>Fix compile errors<commit_after>package lumberjack

import (
  "os" // for File and friends
  "fmt"
  "bytes"
  "io"
  "bufio"
  proto "code.google.com/p/goprotobuf/proto"
  "time"
)

type Harvester struct {
  Path string /* the file path to harvest */

  file os.File /* the file being watched */
}

func (h *Harvester) Harvest(output chan *FileEvent) {
  // TODO(sissel): Read the file
  // TODO(sissel): Emit FileEvent for each line to 'output'
  // TODO(sissel): Handle rotation
  // TODO(sissel): Sleep when there's nothing to do
  // TODO(sissel): Quit if we think the file is dead (file dev/inode changed, no data in X seconds)

  fmt.Printf("Starting harvester: %s\n", h.Path)

  file := h.open()
  defer file.Close()
  //info, _ := file.Stat()

  // TODO(sissel): Ask the registrar for the start position?
  // TODO(sissel): record the current file inode/device/etc

  var line uint64 = 0 // Ask registrar about the line number

  // get current offset in file
  offset, _ := file.Seek(0, os.SEEK_CUR)

  // TODO(sissel): Make the buffer size tunable at start-time
  reader := bufio.NewReaderSize(file, 16<<10) // 16kb buffer by default

  var read_timeout = 10 * time.Second
  last_read_time := time.Now()
  for {
    text, err := h.readline(reader, read_timeout)

    if err != nil {
      if err == io.EOF {
        // timed out waiting for data, got eof.
        // TODO(sissel): Check to see if the file was truncated
        // TODO(sissel): if last_read_time was more than 24 hours ago
        if age := time.Since(last_read_time); age > (24 * time.Hour) {
          // This file is idle for more than 24 hours. Give up and stop harvesting.
          fmt.Printf("Stopping harvest of %s; last change was %d seconds ago\n", h.Path, age.Seconds())
          return
        }
        continue
      } else {
        fmt.Printf("Unexpected state reading from %s; error: %s\n", h.Path, err)
        return
      }
    }
    last_read_time = time.Now()

    line++
    event := &FileEvent{
      Source: proto.String(h.Path),
      Offset: proto.Uint64(uint64(offset)),
      Line: proto.Uint64(line),
      Text: text,
    }
    offset += int64(len(*event.Text)) + 1  // +1 because of the line terminator

    output <- event // ship the new event downstream
  } /* forever */
}

func (h *Harvester) open() *os.File {
  var file *os.File

  // Special handling that "-" means to read from standard input
  if h.Path == "-" {
    return os.Stdin
  }

  for {
    var err error
    file, err = os.Open(h.Path)

    if err != nil {
      // retry on failure.
      fmt.Printf("Failed opening %s: %s\n", h.Path, err)
      time.Sleep(5 * time.Second)
    } else {
      break
    }
  }

  // TODO(sissel): In the future, use the registrary to determine where to seek.
  // TODO(sissel): Only seek if the file is a file, not a pipe or socket.
  file.Seek(0, os.SEEK_END)

  return file
}

func (h *Harvester) readline(reader *bufio.Reader, eof_timeout time.Duration) (*string, error) {
  var buffer bytes.Buffer
  start_time := time.Now()
  for {
    segment, is_partial, err := reader.ReadLine()

    if err != nil {
      // TODO(sissel): if eof and line_complete is false, don't check rotation unless a very long time has passed
      if err == io.EOF {
        time.Sleep(1 * time.Second) // TODO(sissel): Implement backoff

        // Give up waiting for data after a certain amount of time.
        // If we time out, return the error (eof)
        if time.Since(start_time) > eof_timeout {
          return nil, err
        }
        continue
      } else {
        fmt.Println(err)
        return nil, err // TODO(sissel): don't do this?
      }
    }

    // TODO(sissel): if buffer exceeds a certain length, maybe report an error condition? chop it?
    buffer.Write(segment)

    if !is_partial {
      // If we got a full line, return the whole line.
      str := new(string)
      *str = buffer.String()
      return str, nil
    }
  } /* forever read chunks */

  return nil, nil
}
<|endoftext|>