<commit_before>package main

import (
	"bufio"
	"context"
	. "fmt"
	"github.com/google/go-github/github"
	"golang.org/x/oauth2"
	"os"
	"strconv"
	"time"
)

var userName = "ccqpein"

// Define types
type repoDetail struct {
	Name   string
	Detail []*github.WeeklyStats
}

type repoWeekDetail struct {
	Name       string
	weeklyData [][]int
}

type ChartFile struct {
	ChartType, Title, SubTitle, ValueSuffix, YAxisText string
	XAxisNumbers                                       []int
	Data                                               []repoWeekDetail
}

type intArray1 []int
type intArray2 [][]int

// Authentication and collect repos information
// Codes come from Go official document
func Authentication(userName string) *github.Client {
	fi, err := os.Open("./token")
	check(err)
	ffi := bufio.NewReader(fi)

	str, _, _ := ffi.ReadLine()

	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: string(str)},
	)

	tc := oauth2.NewClient(oauth2.NoContext, ts)
	client := github.NewClient(tc)

	return client
}

func GetAllRepos(userName string, client *github.Client) []*github.Repository {
	ctx := context.Background()
	ReOption := &github.RepositoryListOptions{Type: "owner"}
	repos, _, err2 := client.Repositories.List(ctx, userName, ReOption)
	if err2 != nil {
		Println(err2)
	}

	//Println(repos)
	return repos
}

func GetWeeklyStats(userName string, repos []*github.Repository, rD chan repoDetail, client *github.Client) {
	ctx := context.Background()
	for _, repo := range repos {
		var A repoDetail
		name := repo.Name
		reposs, _, _ := client.Repositories.ListCodeFrequency(ctx, userName, *name)
		A.Name = *name
		A.Detail = reposs
		rD <- A
	}
}

// Handle the information
func DoWeeklyStats(repoD chan repoDetail, repos []*github.Repository) []repoWeekDetail {
	now := time.Now()
	OneYearAgo := now.AddDate(-1, 0, 0)
	//Println((now.Sub(OneYearAgo).Hours() / 24))
	var repoWeekDetailList []repoWeekDetail

	for i := 0; i < len(repos); i++ {
		var sumAdd, sumDel int
		var weeklyData [][]int

		A := <-repoD
		for _, codeStatues := range A.Detail {
			we := *codeStatues.Week
			//Println(A.Name, we)
			if we == *A.Detail[0].Week && we.After(OneYearAgo) {
				da := int(we.Sub(OneYearAgo).Hours() / (24 * 7))
				for daa := 0; daa < da; daa++ {
					weeklyData = append(weeklyData, []int{0, 0})
				}
			}
			if we.After(OneYearAgo) {
				ad := *codeStatues.Additions
				de := *codeStatues.Deletions
				var temp = []int{ad, de}
				weeklyData = append(weeklyData, temp)
				sumAdd += ad
				sumDel += de
			}
		}

		var tempDetail = repoWeekDetail{Name: A.Name, weeklyData: weeklyData}
		repoWeekDetailList = append(repoWeekDetailList, tempDetail)
		//Println(len(weeklyData))
		Println(A.Name, sumAdd, sumDel)
	}
	return repoWeekDetailList
}

//// Make chart file below
//------------------------------------------------------------------------------------
func check(e error) {
	if e != nil {
		panic(e)
	}
}

func MakeChartFile(dataInput *[]repoWeekDetail) ChartFile {
	var chartTemp = ChartFile{
		ChartType:    "column",
		Title:        "LineNumbers",
		SubTitle:     " ",
		ValueSuffix:  "",
		XAxisNumbers: []int{5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70},
		YAxisText:    "Line ",
	}

	for _, i := range *dataInput {
		chartTemp.Data = append(chartTemp.Data, i)

	}
	//Println(chartTemp)
	return chartTemp
}

func (dd intArray1) changeToString() string {
	ss := ""
	for _, num := range dd {
		ss = ss + strconv.Itoa(num) + ", "
	}
	return ss
}

func (dd intArray2) changeToString(index int) string {
	ss := ""
	for _, num := range dd {
		ss = ss + strconv.Itoa(num[index]) + ", "
	}
	return ss
}

func WriteChartFileIn(dataInput ChartFile) error {
	var stringToWrite string

	// Write gochart file
	stringToWrite = Sprintf("ChartType = %s \nTitle = %s \nSubTitle = %s \nValueSuffix = %s \nXAxisNumbers = %s \nYAxisText = %s \n \n# The data and the name of the lines \n",
		dataInput.ChartType,
		dataInput.Title,
		dataInput.SubTitle,
		dataInput.ValueSuffix,
		intArray1(dataInput.XAxisNumbers).changeToString(),
		dataInput.YAxisText)

	stringToWrite = stringToWrite +
		func(d []repoWeekDetail) string {
			stringTemp := ""
			for _, i := range d {
				stringTemp = stringTemp + Sprintf("Data|%s = %s \n",
					i.Name, intArray2(i.weeklyData).changeToString(0))
			}
			return stringTemp
		}(dataInput.Data)

	// Save file in folder
	if _, err := os.Stat("./tmp"); err != nil {
		if os.IsNotExist(err) {
			Print("Create new folder store data")
			os.MkdirAll("./tmp", 0777)
		}
	}

	f, err := os.Create("./tmp/data.chart")
	check(err)
	defer f.Close()

	_, err = f.WriteString(stringToWrite)
	check(err)
	return err
}

func main() {
	client := Authentication(userName)

	allRepos := GetAllRepos(userName, client)
	rD := make(chan repoDetail)
	go GetWeeklyStats(userName, allRepos, rD, client)
	tempFileDat := DoWeeklyStats(rD, allRepos)

	fileData := MakeChartFile(&tempFileDat)
	WriteChartFileIn(fileData)

}
<commit_msg>a bit<commit_after>package main

import (
	"bufio"
	"context"
	. "fmt"
	"github.com/google/go-github/github"
	"golang.org/x/oauth2"
	"os"
	"strconv"
	"sync"
	"time"
)

var userName = "ccqpein"

// Define types
type repoDetail struct {
	Name   string
	Detail []*github.WeeklyStats
}

type repoWeekDetail struct {
	Name       string
	weeklyData [][]int
}

type ChartFile struct {
	ChartType, Title, SubTitle, ValueSuffix, YAxisText string
	XAxisNumbers                                       []int
	Data                                               []repoWeekDetail
}

type intArray1 []int
type intArray2 [][]int

// Authentication and collect repos information
// Codes come from Go official document
func Authentication(userName string) *github.Client {
	fi, err := os.Open("./token")
	check(err)
	ffi := bufio.NewReader(fi)

	str, _, _ := ffi.ReadLine()

	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: string(str)},
	)

	tc := oauth2.NewClient(oauth2.NoContext, ts)
	client := github.NewClient(tc)

	return client
}

func GetAllRepos(userName string, client *github.Client) []*github.Repository {
	ctx := context.Background()
	ReOption := &github.RepositoryListOptions{Type: "owner"}
	repos, _, err2 := client.Repositories.List(ctx, userName, ReOption)
	if err2 != nil {
		Println(err2)
	}

	return repos
}

func GetWeeklyStats(userName string, repo *github.Repository, rD chan repoDetail, client *github.Client, wg sync.WaitGroup) {
	defer wg.Done()
	ctx := context.Background()
	var A repoDetail
	name := repo.Name
	reposs, _, _ := client.Repositories.ListCodeFrequency(ctx, userName, *name)
	A.Name = *name
	A.Detail = reposs
	rD <- A
}

// Handle the information
func DoWeeklyStats(repoD chan repoDetail, repos []*github.Repository) []repoWeekDetail {
	now := time.Now()
	OneYearAgo := now.AddDate(-1, 0, 0)
	//Println((now.Sub(OneYearAgo).Hours() / 24))
	var repoWeekDetailList []repoWeekDetail

	for i := 0; i < len(repos); i++ {
		var sumAdd, sumDel int
		var weeklyData [][]int

		A := <-repoD
		for _, codeStatues := range A.Detail {
			we := *codeStatues.Week
			//Println(A.Name, we)
			if we == *A.Detail[0].Week && we.After(OneYearAgo) {
				da := int(we.Sub(OneYearAgo).Hours() / (24 * 7))
				for daa := 0; daa < da; daa++ {
					weeklyData = append(weeklyData, []int{0, 0})
				}
			}
			if we.After(OneYearAgo) {
				ad := *codeStatues.Additions
				de := *codeStatues.Deletions
				var temp = []int{ad, de}
				weeklyData = append(weeklyData, temp)
				sumAdd += ad
				sumDel += de
			}
		}

		var tempDetail = repoWeekDetail{Name: A.Name, weeklyData: weeklyData}
		repoWeekDetailList = append(repoWeekDetailList, tempDetail)
		//Println(len(weeklyData))
		Println(A.Name, sumAdd, sumDel)
	}
	return repoWeekDetailList
}

//// Make chart file below
//------------------------------------------------------------------------------------
func check(e error) {
	if e != nil {
		panic(e)
	}
}

func MakeChartFile(dataInput *[]repoWeekDetail) ChartFile {
	var chartTemp = ChartFile{
		ChartType:    "column",
		Title:        "LineNumbers",
		SubTitle:     " ",
		ValueSuffix:  "",
		XAxisNumbers: []int{5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70},
		YAxisText:    "Line ",
	}

	for _, i := range *dataInput {
		chartTemp.Data = append(chartTemp.Data, i)

	}
	//Println(chartTemp)
	return chartTemp
}

func (dd intArray1) changeToString() string {
	ss := ""
	for _, num := range dd {
		ss = ss + strconv.Itoa(num) + ", "
	}
	return ss
}

func (dd intArray2) changeToString(index int) string {
	ss := ""
	for _, num := range dd {
		ss = ss + strconv.Itoa(num[index]) + ", "
	}
	return ss
}

func WriteChartFileIn(dataInput ChartFile) error {
	var stringToWrite string

	// Write gochart file
	stringToWrite = Sprintf("ChartType = %s \nTitle = %s \nSubTitle = %s \nValueSuffix = %s \nXAxisNumbers = %s \nYAxisText = %s \n \n# The data and the name of the lines \n",
		dataInput.ChartType,
		dataInput.Title,
		dataInput.SubTitle,
		dataInput.ValueSuffix,
		intArray1(dataInput.XAxisNumbers).changeToString(),
		dataInput.YAxisText)

	stringToWrite = stringToWrite +
		func(d []repoWeekDetail) string {
			stringTemp := ""
			for _, i := range d {
				stringTemp = stringTemp + Sprintf("Data|%s = %s \n",
					i.Name, intArray2(i.weeklyData).changeToString(0))
			}
			return stringTemp
		}(dataInput.Data)

	// Save file in folder
	if _, err := os.Stat("./tmp"); err != nil {
		if os.IsNotExist(err) {
			Print("Create new folder store data")
			os.MkdirAll("./tmp", 0777)
		}
	}

	f, err := os.Create("./tmp/data.chart")
	check(err)
	defer f.Close()

	_, err = f.WriteString(stringToWrite)
	check(err)
	return err
}

func main() {
	client := Authentication(userName)
	wg := sync.WaitGroup{}

	allRepos := GetAllRepos(userName, client)
	//	Println(allRepos)
	rD := make(chan repoDetail)
	wg.Add(len(allRepos))
	for _, repo := range allRepos {
		Println(*repo.Name)
		go GetWeeklyStats(userName, repo, rD, client, wg)
	}
	wg.Wait()
	tempFileDat := DoWeeklyStats(rD, allRepos)

	fileData := MakeChartFile(&tempFileDat)
	WriteChartFileIn(fileData)

}
<|endoftext|>


<commit_before>// Copyright 2017 The WPT Dashboard Project. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package webapp

import (
	"encoding/json"
	"fmt"
	"net/http"

	models "github.com/w3c/wptdashboard/shared"
)

// This handler is responsible for all pages that display test results.
// It fetches the latest TestRun for each browser then renders the HTML
// page with the TestRuns encoded as JSON. The Polymer app picks those up
// and loads the summary files based on each entity's TestRun.ResultsURL.
//
// The browsers initially displayed to the user are defined in browsers.json.
// The JSON property "initially_loaded" is what controls this.
func testHandler(w http.ResponseWriter, r *http.Request) {
	runSHA, err := ParseSHAParam(r)
	if err != nil {
		http.Error(w, "Invalid query params", http.StatusBadRequest)
		return
	}

	var testRunSources []string

	specBefore := r.URL.Query().Get("before")
	specAfter := r.URL.Query().Get("after")
	if specBefore != "" || specAfter != "" {
		if specBefore == "" {
			http.Error(w, "after param provided, but before param missing", http.StatusBadRequest)
			return
		} else if specAfter == "" {
			http.Error(w, "before param provided, but after param missing", http.StatusBadRequest)
			return
		}
		var before platformAtRevision
		var after platformAtRevision
		if before, err = parsePlatformAtRevisionSpec(specBefore); err != nil {
			http.Error(w, "invalid before param", http.StatusBadRequest)
			return
		} else if after, err = parsePlatformAtRevisionSpec(specAfter); err != nil {
			http.Error(w, "invalid after param", http.StatusBadRequest)
			return
		}
		const singleRunURL = `/api/run?sha=%s&browser=%s`
		testRunSources = []string{
			fmt.Sprintf(singleRunURL, before.Revision, before.Platform),
			fmt.Sprintf(singleRunURL, after.Revision, after.Platform),
		}
	} else {
		const sourceURL = `/api/runs?sha=%s`
		testRunSources = []string{fmt.Sprintf(sourceURL, runSHA)}
	}

	testRunSourcesBytes, err := json.Marshal(testRunSources)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	data := struct {
		TestRuns       string
		TestRunSources string
		SHA            string
	}{
		TestRunSources: string(testRunSourcesBytes),
		SHA:            runSHA,
	}

	if specBefore != "" || specAfter != "" {
		const diffRunURL = `/api/diff?before=%s&after=%s`
		diffRun := models.TestRun{
			Revision:    "diff",
			BrowserName: "Diff",
			ResultsURL:  fmt.Sprintf(diffRunURL, specBefore, specAfter),
		}
		var marshaled []byte
		if marshaled, err = json.Marshal([]models.TestRun{diffRun}); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		data.TestRuns = string(marshaled)
	}

	if err := templates.ExecuteTemplate(w, "index.html", data); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
}
<commit_msg>Fix capitalization<commit_after>// Copyright 2017 The WPT Dashboard Project. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package webapp

import (
	"encoding/json"
	"fmt"
	"net/http"

	models "github.com/w3c/wptdashboard/shared"
)

// This handler is responsible for all pages that display test results.
// It fetches the latest TestRun for each browser then renders the HTML
// page with the TestRuns encoded as JSON. The Polymer app picks those up
// and loads the summary files based on each entity's TestRun.ResultsURL.
//
// The browsers initially displayed to the user are defined in browsers.json.
// The JSON property "initially_loaded" is what controls this.
func testHandler(w http.ResponseWriter, r *http.Request) {
	runSHA, err := ParseSHAParam(r)
	if err != nil {
		http.Error(w, "Invalid query params", http.StatusBadRequest)
		return
	}

	var testRunSources []string

	specBefore := r.URL.Query().Get("before")
	specAfter := r.URL.Query().Get("after")
	if specBefore != "" || specAfter != "" {
		if specBefore == "" {
			http.Error(w, "after param provided, but before param missing", http.StatusBadRequest)
			return
		} else if specAfter == "" {
			http.Error(w, "before param provided, but after param missing", http.StatusBadRequest)
			return
		}
		var before platformAtRevision
		var after platformAtRevision
		if before, err = parsePlatformAtRevisionSpec(specBefore); err != nil {
			http.Error(w, "invalid before param", http.StatusBadRequest)
			return
		} else if after, err = parsePlatformAtRevisionSpec(specAfter); err != nil {
			http.Error(w, "invalid after param", http.StatusBadRequest)
			return
		}
		const singleRunURL = `/api/run?sha=%s&browser=%s`
		testRunSources = []string{
			fmt.Sprintf(singleRunURL, before.Revision, before.Platform),
			fmt.Sprintf(singleRunURL, after.Revision, after.Platform),
		}
	} else {
		const sourceURL = `/api/runs?sha=%s`
		testRunSources = []string{fmt.Sprintf(sourceURL, runSHA)}
	}

	testRunSourcesBytes, err := json.Marshal(testRunSources)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	data := struct {
		TestRuns       string
		TestRunSources string
		SHA            string
	}{
		TestRunSources: string(testRunSourcesBytes),
		SHA:            runSHA,
	}

	if specBefore != "" || specAfter != "" {
		const diffRunURL = `/api/diff?before=%s&after=%s`
		diffRun := models.TestRun{
			Revision:    "diff",
			BrowserName: "diff",
			ResultsURL:  fmt.Sprintf(diffRunURL, specBefore, specAfter),
		}
		var marshaled []byte
		if marshaled, err = json.Marshal([]models.TestRun{diffRun}); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		data.TestRuns = string(marshaled)
	}

	if err := templates.ExecuteTemplate(w, "index.html", data); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"sort"
)

// check aborts on non-nil errors. (In this program, all errors are generally fatal for simplicity.)
func check(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

type Command interface {
	Execute(args []string) error
	Help() string
}

var (
	commands = map[string]Command{
		"moveresize": &MoveResize{},
		"focus":      &Focus{},
	}
	commandNames []string
)

func init() {
	for name := range commands {
		commandNames = append(commandNames, name)
	}
	sort.Strings(commandNames)
}

func usage(status int) {
	fmt.Printf(`Usage:
    %s COMMAND [arg1] [arg2] ...
where COMMAND is one of %v
(Type '%[1]s help COMMAND' to see information about a specific command.)
`, os.Args[0], commandNames)
	os.Exit(status)
}

func main() {
	if len(os.Args) < 2 {
		usage(-1)
	}
	switch os.Args[1] {
	case "-h", "-help", "--help", "help":
		usage(0)
	}
	command, ok := commands[os.Args[1]]
	if !ok {
		usage(-1)
	}
	if len(os.Args) >= 3 {
		switch os.Args[2] {
		case "-h", "-help", "--help", "help":
			fmt.Println(command.Help())
			os.Exit(0)
		}
	}
	check(command.Execute(os.Args[2:]))
}
<commit_msg>Fix help directions<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"sort"
)

// check aborts on non-nil errors. (In this program, all errors are generally fatal for simplicity.)
func check(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

type Command interface {
	Execute(args []string) error
	Help() string
}

var (
	commands = map[string]Command{
		"moveresize": &MoveResize{},
		"focus":      &Focus{},
	}
	commandNames []string
)

func init() {
	for name := range commands {
		commandNames = append(commandNames, name)
	}
	sort.Strings(commandNames)
}

func usage(status int) {
	fmt.Printf(`Usage:
    %s COMMAND [arg1] [arg2] ...
where COMMAND is one of %v
(Type '%[1]s COMMAND help' to see information about a specific command.)
`, os.Args[0], commandNames)
	os.Exit(status)
}

func main() {
	if len(os.Args) < 2 {
		usage(-1)
	}
	switch os.Args[1] {
	case "-h", "-help", "--help", "help":
		usage(0)
	}
	command, ok := commands[os.Args[1]]
	if !ok {
		usage(-1)
	}
	if len(os.Args) >= 3 {
		switch os.Args[2] {
		case "-h", "-help", "--help", "help":
			fmt.Println(command.Help())
			os.Exit(0)
		}
	}
	check(command.Execute(os.Args[2:]))
}
<|endoftext|>


<commit_before>package caseconv

import (
	"regexp"
	"strings"
)

/*
CamelCase to snake_case
*/
func CamelToSnake(str string) string {
	snake1 := regexp.MustCompile("([A-Z])([A-Z][a-z])")
	snake2 := regexp.MustCompile("([a-z])([A-Z])")
	return strings.ToLower(snake2.ReplaceAllString(snake1.ReplaceAllString(str, "${1}_${2}"), "${1}_${2}"))
}
<commit_msg>Updated doc<commit_after>package caseconv

import (
	"regexp"
	"strings"
)

// CamelToSnake converts CamelCase to snake_case
func CamelToSnake(str string) string {
	snake1 := regexp.MustCompile("([A-Z])([A-Z][a-z])")
	snake2 := regexp.MustCompile("([a-z])([A-Z])")
	return strings.ToLower(snake2.ReplaceAllString(snake1.ReplaceAllString(str, "${1}_${2}"), "${1}_${2}"))
}
<|endoftext|>


<commit_before>// Copyright ©2016 The Gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package stat

import "sort"

// ROC returns paired false positive rate (FPR) and true positive rate
// (TPR) values corresponding to n cutoffs spanning the relative
// (or receiver) operator characteristic (ROC) curve obtained when y is
// treated as a binary classifier for classes with weights.
//
// Cutoffs are equally spaced from eps less than the minimum value of y
// to the maximum value of y, including both endpoints meaning that the
// resulting ROC curve will always begin at (0,0) and end at (1,1).
//
// The input y must be sorted, and SortWeightedLabeled can be used in
// order to sort y together with classes and weights.
//
// For a given cutoff value, observations corresponding to entries in y
// greater than the cutoff value are classified as false, while those
// below (or equal to) the cutoff value are classified as true. These
// assigned class labels are compared with the true values in the classes
// slice and used to calculate the FPR and TPR.
//
// If weights is nil, all weights are treated as 1.
//
// When n is zero all possible cutoffs are calculated, resulting
// in fpr and tpr having length one greater than the number of unique
// values in y. When n is greater than one fpr and tpr will be returned
// with length n. ROC will panic if n is equal to one or less than 0.
//
// More details about ROC curves are available at
// https://en.wikipedia.org/wiki/Receiver_operating_characteristic
func ROC(n int, y []float64, classes []bool, weights []float64) (tpr, fpr []float64) {
	if len(y) != len(classes) {
		panic("stat: slice length mismatch")
	}
	if weights != nil && len(y) != len(weights) {
		panic("stat: slice length mismatch")
	}
	if !sort.Float64sAreSorted(y) {
		panic("stat: input must be sorted")
	}

	var incWidth, tol float64
	if n == 0 {
		if len(y) == 0 {
			return nil, nil
		}
		tpr = make([]float64, len(y)+1)
		fpr = make([]float64, len(y)+1)
	} else {
		if n < 2 {
			panic("stat: cannot calculate fewer than 2 points on a ROC curve")
		}
		if len(y) == 0 {
			return nil, nil
		}
		tpr = make([]float64, n)
		fpr = make([]float64, n)
		incWidth = (y[len(y)-1] - y[0]) / float64(n-1)
		tol = y[0] + incWidth
		if incWidth == 0 {
			tpr[n-1] = 1
			fpr[n-1] = 1
			return
		}
	}

	var bin int = 1 // the initial bin is known to have 0 fpr and 0 tpr
	var nPos, nNeg float64
	for i, u := range classes {
		var posWeight, negWeight float64 = 0, 1
		if weights != nil {
			negWeight = weights[i]
		}
		if u {
			posWeight, negWeight = negWeight, posWeight
		}
		nPos += posWeight
		nNeg += negWeight
		tpr[bin] += posWeight
		fpr[bin] += negWeight

		// Assess if the bin needs to be updated. If n is zero,
		// the bin is always updated, unless consecutive y values
		// are equal. Otherwise, the bin must be updated until it
		// matches the next y value (skipping empty bins).
		if n == 0 {
			if i != (len(y)-1) && y[i] != y[i+1] {
				bin++
				tpr[bin] = tpr[bin-1]
				fpr[bin] = fpr[bin-1]
			}
		} else {
			for i != (len(y)-1) && y[i+1] > tol {
				tol += incWidth
				bin++
				tpr[bin] = tpr[bin-1]
				fpr[bin] = fpr[bin-1]
			}
		}
	}
	if n == 0 {
		tpr = tpr[:(bin + 1)]
		fpr = fpr[:(bin + 1)]
	}

	invNeg := 1 / nNeg
	invPos := 1 / nPos
	for i := range tpr {
		tpr[i] *= invPos
		fpr[i] *= invNeg
	}
	tpr[len(tpr)-1] = 1
	fpr[len(fpr)-1] = 1

	return tpr, fpr
}
<commit_msg>stat: fix ROC for uniform cases<commit_after>// Copyright ©2016 The Gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package stat

import "sort"

// ROC returns paired false positive rate (FPR) and true positive rate
// (TPR) values corresponding to n cutoffs spanning the relative
// (or receiver) operator characteristic (ROC) curve obtained when y is
// treated as a binary classifier for classes with weights.
//
// Cutoffs are equally spaced from eps less than the minimum value of y
// to the maximum value of y, including both endpoints meaning that the
// resulting ROC curve will always begin at (0,0) and end at (1,1).
//
// The input y must be sorted, and SortWeightedLabeled can be used in
// order to sort y together with classes and weights.
//
// For a given cutoff value, observations corresponding to entries in y
// greater than the cutoff value are classified as false, while those
// below (or equal to) the cutoff value are classified as true. These
// assigned class labels are compared with the true values in the classes
// slice and used to calculate the FPR and TPR.
//
// If weights is nil, all weights are treated as 1.
//
// When n is zero all possible cutoffs are calculated, resulting
// in fpr and tpr having length one greater than the number of unique
// values in y. When n is greater than one fpr and tpr will be returned
// with length n. ROC will panic if n is equal to one or less than 0.
//
// More details about ROC curves are available at
// https://en.wikipedia.org/wiki/Receiver_operating_characteristic
func ROC(n int, y []float64, classes []bool, weights []float64) (tpr, fpr []float64) {
	if len(y) != len(classes) {
		panic("stat: slice length mismatch")
	}
	if weights != nil && len(y) != len(weights) {
		panic("stat: slice length mismatch")
	}
	if !sort.Float64sAreSorted(y) {
		panic("stat: input must be sorted")
	}

	var incWidth, tol float64
	if n == 0 {
		if len(y) == 0 {
			return nil, nil
		}
		tpr = make([]float64, len(y)+1)
		fpr = make([]float64, len(y)+1)
	} else {
		if n < 2 {
			panic("stat: cannot calculate fewer than 2 points on a ROC curve")
		}
		if len(y) == 0 {
			return nil, nil
		}
		tpr = make([]float64, n)
		fpr = make([]float64, n)
		incWidth = (y[len(y)-1] - y[0]) / float64(n-1)
		tol = y[0] + incWidth
		if incWidth == 0 {
			tpr[n-1] = 1
			fpr[n-1] = 1
			return
		}
	}

	var bin int = 1 // the initial bin is known to have 0 fpr and 0 tpr
	var nPos, nNeg float64
	for i, u := range classes {
		var posWeight, negWeight float64 = 0, 1
		if weights != nil {
			negWeight = weights[i]
		}
		if u {
			posWeight, negWeight = negWeight, posWeight
		}
		nPos += posWeight
		nNeg += negWeight
		tpr[bin] += posWeight
		fpr[bin] += negWeight

		// Assess if the bin needs to be updated. If n is zero,
		// the bin is always updated, unless consecutive y values
		// are equal. Otherwise, the bin must be updated until it
		// matches the next y value (skipping empty bins).
		if n == 0 {
			if i != (len(y)-1) && y[i] != y[i+1] {
				bin++
				tpr[bin] = tpr[bin-1]
				fpr[bin] = fpr[bin-1]
			}
		} else {
			for i != (len(y)-1) && y[i+1] > tol {
				tol += incWidth
				bin++
				tpr[bin] = tpr[bin-1]
				fpr[bin] = fpr[bin-1]
			}
		}
	}
	if n == 0 {
		tpr = tpr[:(bin + 1)]
		fpr = fpr[:(bin + 1)]
	}

	var invNeg, invPos float64
	if nNeg != 0 {
		invNeg = 1 / nNeg
	}
	if nPos != 0 {
		invPos = 1 / nPos
	}
	for i := range tpr {
		tpr[i] *= invPos
		fpr[i] *= invNeg
	}
	tpr[len(tpr)-1] = 1
	fpr[len(fpr)-1] = 1

	return tpr, fpr
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"encoding/json"
	"fmt"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"os"

	"github.com/google/go-github/github"
)

type MirroredPR struct {
	DownstreamID int
	UpstreamID   int
}

type PRMirror struct {
	GitHubClient  *github.Client
	Context       *context.Context
	Configuration *Config
	Database      *Database
}

func (p PRMirror) HandlePREvent(prEvent *github.PullRequestEvent) {

	prAction := prEvent.GetAction()

	log.Debugf("%s\n", prEvent.PullRequest.GetURL())

	if prAction == "closed" {
		if prEvent.PullRequest.GetMerged() == true {
			prID, err := p.MirrorPR(prEvent.PullRequest)
			if err != nil {
				log.Errorf("Error while creating a new PR: %s\n", err.Error())
			} else {
				p.AddLabels(prID, []string{"Upstream PR Merged"})
				p.Database.StoreMirror(prID, prEvent.PullRequest.GetNumber())
			}
		}
	}
}

func (p PRMirror) isRatelimit(err error) bool {
	if _, ok := err.(*github.RateLimitError); ok {
		// TODO: Maybe add some context here
		log.Error("The github.com rate limit has been hit")
		return true
	}
	return false
}

func (p PRMirror) GetRepoEvents() ([]*github.Event, int64, error) {
	var allEvents []*github.Event
	var pollInterval = int64(0)

	opt := &github.ListOptions{
		PerPage: 100,
	}

	for {
		log.Debugf("Getting RepoEvents Page %d\n", opt.Page)

		events, resp, err := p.GitHubClient.Activity.ListRepositoryEvents(*p.Context, p.Configuration.UpstreamOwner, p.Configuration.UpstreamRepo, opt)
		if err != nil {
			log.Errorf("Error while listing repository events. %s", err.Error())
			return nil, 60, err
		}

		allEvents = append(allEvents, events...)
		if resp.NextPage == 0 {
			pollInterval, err = strconv.ParseInt(resp.Response.Header.Get("X-Poll-Interval"), 10, 64)
			if err != nil {
				panic(err)
			}
			break
		}
		opt.Page = resp.NextPage
	}

	return allEvents, pollInterval, nil
}

func (p PRMirror) GetOpenPRs() ([]*github.PullRequest, error) {
	var allPrs []*github.PullRequest

	opt := &github.PullRequestListOptions{
		ListOptions: github.ListOptions{PerPage: 100},
	}

	for {
		log.Debugf("Getting OpenPRs Page %d\n", opt.ListOptions.Page)

		prs, resp, err := p.GitHubClient.PullRequests.List(*p.Context, p.Configuration.UpstreamOwner, p.Configuration.UpstreamRepo, opt)
		if p.isRatelimit(err) {
			return nil, err
		}

		allPrs = append(allPrs, prs...)
		if resp.NextPage == 0 {
			break
		}
		opt.ListOptions.Page = resp.NextPage
	}

	return allPrs, nil
}

func (p PRMirror) InitialImport() {
	prs, err := p.GetOpenPRs()
	if p.isRatelimit(err) {
		return
	}

	for _, pr := range prs {
		prNum, err := p.Database.GetDownstreamID(pr.GetNumber())
		if err != nil {
			panic(err)
		}

		if prNum != 0 {
			log.Infof("DUP: [%d] - %s\n", pr.GetNumber(), pr.GetTitle())
		} else {
			log.Infof("NEW: [%d] - %s\n", pr.GetNumber(), pr.GetTitle())
			prID, err := p.MirrorPR(pr)
			if err != nil {
				log.Errorf("Error while creating a new PR: %s\n", err.Error())
			} else {
				p.Database.StoreMirror(prID, pr.GetNumber())
				p.AddLabels(prID, []string{"Upstream PR Open"})
			}
			time.Sleep(2 * time.Second)
		}
	}
}

func (p PRMirror) Run() {
	for {
		events, pollInterval, err := p.GetRepoEvents()
		if err == nil {
			for _, event := range events {
				seenEvent, _ := p.Database.SeenEvent(event.GetID())

				if !seenEvent {
					eventType := event.GetType()

					if eventType == "PullRequestEvent" {
						prEvent := github.PullRequestEvent{}
						err = json.Unmarshal(event.GetRawPayload(), &prEvent)
						if err != nil {
							panic(err)
						}

						p.HandlePREvent(&prEvent)
						p.Database.AddEvent(event.GetID())
					}
				}
			}
		}

		log.Debugf("Sleeping for %d as specified by GitHub\n", pollInterval)
		time.Sleep(time.Duration(pollInterval) * time.Second)
	}
}

func (p PRMirror) MirrorPR(pr *github.PullRequest) (int, error) {
	log.Infof("Mirroring PR [%d]: %s from %s\n", pr.GetNumber(), pr.GetTitle(), pr.User.GetLogin())

	cmd := exec.Command(fmt.Sprintf("%s%s", p.Configuration.RepoPath, p.Configuration.ToolPath), strconv.Itoa(pr.GetNumber()))
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err := cmd.Start()
	if err != nil {
		panic(err)
	}
	err = cmd.Wait()
	if err != nil {
		panic(err)
	}

	base := fmt.Sprintf("upstream-merge-%d", pr.GetNumber())
	head := "HippieStation/HippieStation"
	maintainerCanModify := false
	title := fmt.Sprintf("[MIRROR] %s", pr.GetTitle())
	body := fmt.Sprintf("Original PR: %s\n--------------------\n%s", pr.GetHTMLURL(), strings.Replace(pr.GetBody(), "@", "@ ", -1))

	newPR := github.NewPullRequest{}
	newPR.Title = &title
	newPR.Body = &body
	newPR.Base = &base
	newPR.Head = &head
	newPR.MaintainerCanModify = &maintainerCanModify

	pr, _, err = p.GitHubClient.PullRequests.Create(*p.Context, p.Configuration.DownstreamOwner, p.Configuration.DownstreamRepo, &newPR)
	if err != nil {
		return 0, err
	}

	return pr.GetNumber(), nil
}

func (p PRMirror) CreateLabel(labelText string, labelColour string) bool {
	label := github.Label{
		Name:  &labelText,
		Color: &labelColour,
	}

	_, _, err := p.GitHubClient.Issues.CreateLabel(*p.Context, p.Configuration.DownstreamOwner, p.Configuration.DownstreamRepo, &label)
	if err != nil {
		log.Errorf("Error while creating a label - %s", err.Error())
		return false
	}

	return true
}

func (p PRMirror) AddLabels(id int, labels []string) bool {
	_, _, err := p.GitHubClient.Issues.AddLabelsToIssue(*p.Context, p.Configuration.DownstreamOwner, p.Configuration.DownstreamRepo, id, labels)
	if err != nil {
		log.Errorf("Error while adding a label on issue#:%d - %s", id, err.Error())
		return false
	}

	return true
}

func (p PRMirror) RemoveLabel(id int, labels string) bool {
	_, err := p.GitHubClient.Issues.RemoveLabelForIssue(*p.Context, p.Configuration.DownstreamOwner, p.Configuration.DownstreamRepo, id, labels)
	if err != nil {
		log.Errorf("Error while removing a label on issue#:%d - %s", id, err.Error())
		return false
	}

	return true
}

func (p PRMirror) AddComment(id int, comment string) bool {
	issueComment := github.IssueComment{}
	issueComment.Body = &comment

	_, _, err := p.GitHubClient.Issues.CreateComment(*p.Context, p.Configuration.DownstreamOwner, p.Configuration.DownstreamRepo, id, &issueComment)
	if err != nil {
		log.Errorf("Error while adding a comment to issue#:%d - %s", id, err.Error())
		return false
	}

	return true
}
<commit_msg>This is important<commit_after>package main

import (
	"context"
	"encoding/json"
	"fmt"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"os"

	"github.com/google/go-github/github"
)

type MirroredPR struct {
	DownstreamID int
	UpstreamID   int
}

type PRMirror struct {
	GitHubClient  *github.Client
	Context       *context.Context
	Configuration *Config
	Database      *Database
}

func (p PRMirror) HandlePREvent(prEvent *github.PullRequestEvent) {

	prAction := prEvent.GetAction()

	log.Debugf("%s\n", prEvent.PullRequest.GetURL())

	if prAction == "closed" {
		if prEvent.PullRequest.GetMerged() == true {
			prID, err := p.MirrorPR(prEvent.PullRequest)
			if err != nil {
				log.Errorf("Error while creating a new PR: %s\n", err.Error())
			} else {
				p.AddLabels(prID, []string{"Upstream PR Merged"})
				p.Database.StoreMirror(prID, prEvent.PullRequest.GetNumber())
			}
		}
	}
}

func (p PRMirror) isRatelimit(err error) bool {
	if _, ok := err.(*github.RateLimitError); ok {
		// TODO: Maybe add some context here
		log.Error("The github.com rate limit has been hit")
		return true
	}
	return false
}

func (p PRMirror) GetRepoEvents() ([]*github.Event, int64, error) {
	var allEvents []*github.Event
	var pollInterval = int64(0)

	opt := &github.ListOptions{
		PerPage: 100,
	}

	for {
		log.Debugf("Getting RepoEvents Page %d\n", opt.Page)

		events, resp, err := p.GitHubClient.Activity.ListRepositoryEvents(*p.Context, p.Configuration.UpstreamOwner, p.Configuration.UpstreamRepo, opt)
		if err != nil {
			log.Errorf("Error while listing repository events. %s", err.Error())
			return nil, 60, err
		}

		allEvents = append(allEvents, events...)
		if resp.NextPage == 0 {
			pollInterval, err = strconv.ParseInt(resp.Response.Header.Get("X-Poll-Interval"), 10, 64)
			if err != nil {
				panic(err)
			}
			break
		}
		opt.Page = resp.NextPage
	}

	return allEvents, pollInterval, nil
}

func (p PRMirror) GetOpenPRs() ([]*github.PullRequest, error) {
	var allPrs []*github.PullRequest

	opt := &github.PullRequestListOptions{
		ListOptions: github.ListOptions{PerPage: 100},
	}

	for {
		log.Debugf("Getting OpenPRs Page %d\n", opt.ListOptions.Page)

		prs, resp, err := p.GitHubClient.PullRequests.List(*p.Context, p.Configuration.UpstreamOwner, p.Configuration.UpstreamRepo, opt)
		if p.isRatelimit(err) {
			return nil, err
		}

		allPrs = append(allPrs, prs...)
		if resp.NextPage == 0 {
			break
		}
		opt.ListOptions.Page = resp.NextPage
	}

	return allPrs, nil
}

func (p PRMirror) InitialImport() {
	prs, err := p.GetOpenPRs()
	if p.isRatelimit(err) {
		return
	}

	for _, pr := range prs {
		prNum, err := p.Database.GetDownstreamID(pr.GetNumber())
		if err != nil {
			panic(err)
		}

		if prNum != 0 {
			log.Infof("DUP: [%d] - %s\n", pr.GetNumber(), pr.GetTitle())
		} else {
			log.Infof("NEW: [%d] - %s\n", pr.GetNumber(), pr.GetTitle())
			prID, err := p.MirrorPR(pr)
			if err != nil {
				log.Errorf("Error while creating a new PR: %s\n", err.Error())
			} else {
				p.Database.StoreMirror(prID, pr.GetNumber())
				p.AddLabels(prID, []string{"Upstream PR Open"})
			}
			time.Sleep(2 * time.Second)
		}
	}
}

func (p PRMirror) Run() {
	for {
		events, pollInterval, err := p.GetRepoEvents()
		if err == nil {
			for _, event := range events {
				seenEvent, _ := p.Database.SeenEvent(event.GetID())

				if !seenEvent {
					eventType := event.GetType()

					if eventType == "PullRequestEvent" {
						prEvent := github.PullRequestEvent{}
						err = json.Unmarshal(event.GetRawPayload(), &prEvent)
						if err != nil {
							panic(err)
						}

						p.HandlePREvent(&prEvent)
						p.Database.AddEvent(event.GetID())
					}
				}
			}
		}

		log.Debugf("Sleeping for %d as specified by GitHub\n", pollInterval)
		time.Sleep(time.Duration(pollInterval) * time.Second)
	}
}

func (p PRMirror) MirrorPR(pr *github.PullRequest) (int, error) {
	log.Infof("Mirroring PR [%d]: %s from %s\n", pr.GetNumber(), pr.GetTitle(), pr.User.GetLogin())

	cmd := exec.Command(fmt.Sprintf("%s%s", p.Configuration.RepoPath, p.Configuration.ToolPath), strconv.Itoa(pr.GetNumber()))
	cmd.Dir = p.Configuration.RepoPath
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err := cmd.Start()
	if err != nil {
		panic(err)
	}
	err = cmd.Wait()
	if err != nil {
		panic(err)
	}

	base := fmt.Sprintf("upstream-merge-%d", pr.GetNumber())
	head := "HippieStation/HippieStation"
	maintainerCanModify := false
	title := fmt.Sprintf("[MIRROR] %s", pr.GetTitle())
	body := fmt.Sprintf("Original PR: %s\n--------------------\n%s", pr.GetHTMLURL(), strings.Replace(pr.GetBody(), "@", "@ ", -1))

	newPR := github.NewPullRequest{}
	newPR.Title = &title
	newPR.Body = &body
	newPR.Base = &base
	newPR.Head = &head
	newPR.MaintainerCanModify = &maintainerCanModify

	pr, _, err = p.GitHubClient.PullRequests.Create(*p.Context, p.Configuration.DownstreamOwner, p.Configuration.DownstreamRepo, &newPR)
	if err != nil {
		return 0, err
	}

	return pr.GetNumber(), nil
}

func (p PRMirror) CreateLabel(labelText string, labelColour string) bool {
	label := github.Label{
		Name:  &labelText,
		Color: &labelColour,
	}

	_, _, err := p.GitHubClient.Issues.CreateLabel(*p.Context, p.Configuration.DownstreamOwner, p.Configuration.DownstreamRepo, &label)
	if err != nil {
		log.Errorf("Error while creating a label - %s", err.Error())
		return false
	}

	return true
}

func (p PRMirror) AddLabels(id int, labels []string) bool {
	_, _, err := p.GitHubClient.Issues.AddLabelsToIssue(*p.Context, p.Configuration.DownstreamOwner, p.Configuration.DownstreamRepo, id, labels)
	if err != nil {
		log.Errorf("Error while adding a label on issue#:%d - %s", id, err.Error())
		return false
	}

	return true
}

func (p PRMirror) RemoveLabel(id int, labels string) bool {
	_, err := p.GitHubClient.Issues.RemoveLabelForIssue(*p.Context, p.Configuration.DownstreamOwner, p.Configuration.DownstreamRepo, id, labels)
	if err != nil {
		log.Errorf("Error while removing a label on issue#:%d - %s", id, err.Error())
		return false
	}

	return true
}

func (p PRMirror) AddComment(id int, comment string) bool {
	issueComment := github.IssueComment{}
	issueComment.Body = &comment

	_, _, err := p.GitHubClient.Issues.CreateComment(*p.Context, p.Configuration.DownstreamOwner, p.Configuration.DownstreamRepo, id, &issueComment)
	if err != nil {
		log.Errorf("Error while adding a comment to issue#:%d - %s", id, err.Error())
		return false
	}

	return true
}
<|endoftext|>


<commit_before>// Copyright 2013 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package metric

import (
	"bytes"
	"fmt"
	"strings"
	"time"

	"code.google.com/p/goprotobuf/proto"

	clientmodel "github.com/prometheus/client_golang/model"

	dto "github.com/prometheus/prometheus/model/generated"

	"github.com/prometheus/prometheus/coding"
	"github.com/prometheus/prometheus/storage"
	"github.com/prometheus/prometheus/storage/raw"
	"github.com/prometheus/prometheus/storage/raw/leveldb"
)

// CurationState contains high-level curation state information for the
// heads-up-display.
type CurationState struct {
	Active      bool
	Name        string
	Limit       time.Duration
	Fingerprint *clientmodel.Fingerprint
}

// curator is responsible for effectuating a given curation policy across the
// stored samples on-disk.  This is useful to compact sparse sample values into
// single sample entities to reduce keyspace load on the datastore.
type Curator struct {
	// Stop functions as a channel that when empty allows the curator to operate.
	// The moment a value is ingested inside of it, the curator goes into drain
	// mode.
	Stop chan bool
}

// watermarkScanner converts (dto.Fingerprint, dto.MetricHighWatermark) doubles
// into (model.Fingerprint, model.Watermark) doubles.
//
// watermarkScanner determines whether to include or exclude candidate
// values from the curation process by virtue of how old the high watermark is.
//
// watermarkScanner scans over the curator.samples table for metrics whose
// high watermark has been determined to be allowable for curation.  This type
// is individually responsible for compaction.
//
// The scanning starts from CurationRemark.LastCompletionTimestamp and goes
// forward until the stop point or end of the series is reached.
type watermarkScanner struct {
	// curationState is the data store for curation remarks.
	curationState CurationRemarker
	// diskFrontier models the available seekable ranges for the provided
	// sampleIterator.
	diskFrontier *diskFrontier
	// ignoreYoungerThan is passed into the curation remark for the given series.
	ignoreYoungerThan time.Duration
	// processor is responsible for executing a given stategy on the
	// to-be-operated-on series.
	processor Processor
	// sampleIterator is a snapshotted iterator for the time series.
	sampleIterator leveldb.Iterator
	// samples
	samples raw.Persistence
	// stopAt is a cue for when to stop mutating a given series.
	stopAt time.Time

	// stop functions as the global stop channel for all future operations.
	stop chan bool
	// status is the outbound channel for notifying the status page of its state.
	status chan CurationState
}

// run facilitates the curation lifecycle.
//
// recencyThreshold represents the most recent time up to which values will be
// curated.
// curationState is the on-disk store where the curation remarks are made for
// how much progress has been made.
func (c *Curator) Run(ignoreYoungerThan time.Duration, instant time.Time, processor Processor, curationState CurationRemarker, samples *leveldb.LevelDBPersistence, watermarks HighWatermarker, status chan CurationState) (err error) {
	defer func(t time.Time) {
		duration := float64(time.Since(t) / time.Millisecond)

		labels := map[string]string{
			cutOff:        fmt.Sprint(ignoreYoungerThan),
			processorName: processor.Name(),
			result:        success,
		}
		if err != nil {
			labels[result] = failure
		}

		curationDuration.IncrementBy(labels, duration)
		curationDurations.Add(labels, duration)
	}(time.Now())
	defer func() {
		select {
		case status <- CurationState{Active: false}:
		case <-status:
		default:
		}
	}()

	iterator := samples.NewIterator(true)
	defer iterator.Close()

	diskFrontier, present, err := newDiskFrontier(iterator)
	if err != nil {
		return
	}
	if !present {
		// No sample database exists; no work to do!
		return
	}

	scanner := &watermarkScanner{
		curationState:     curationState,
		ignoreYoungerThan: ignoreYoungerThan,
		processor:         processor,
		status:            status,
		stop:              c.Stop,
		stopAt:            instant.Add(-1 * ignoreYoungerThan),

		diskFrontier:   diskFrontier,
		sampleIterator: iterator,
		samples:        samples,
	}

	// Right now, the ability to stop a curation is limited to the beginning of
	// each fingerprint cycle.  It is impractical to cease the work once it has
	// begun for a given series.
	_, err = watermarks.ForEach(scanner, scanner, scanner)

	return
}

// drain instructs the curator to stop at the next convenient moment as to not
// introduce data inconsistencies.
func (c *Curator) Drain() {
	if len(c.Stop) == 0 {
		c.Stop <- true
	}
}

func (w *watermarkScanner) DecodeKey(in interface{}) (interface{}, error) {
	key := new(dto.Fingerprint)
	bytes := in.([]byte)

	if err := proto.Unmarshal(bytes, key); err != nil {
		return nil, err
	}

	fingerprint := new(clientmodel.Fingerprint)
	loadFingerprint(fingerprint, key)

	return fingerprint, nil
}

func (w *watermarkScanner) DecodeValue(in interface{}) (interface{}, error) {
	value := new(dto.MetricHighWatermark)
	bytes := in.([]byte)

	if err := proto.Unmarshal(bytes, value); err != nil {
		return nil, err
	}

	watermark := new(watermarks)
	watermark.load(value)

	return watermark, nil
}

func (w *watermarkScanner) shouldStop() bool {
	return len(w.stop) != 0
}

func (w *watermarkScanner) Filter(key, value interface{}) (r storage.FilterResult) {
	fingerprint := key.(*clientmodel.Fingerprint)

	defer func() {
		labels := map[string]string{
			cutOff:        fmt.Sprint(w.ignoreYoungerThan),
			result:        strings.ToLower(r.String()),
			processorName: w.processor.Name(),
		}

		curationFilterOperations.Increment(labels)

		select {
		case w.status <- CurationState{
			Active:      true,
			Name:        w.processor.Name(),
			Limit:       w.ignoreYoungerThan,
			Fingerprint: fingerprint,
		}:
		case <-w.status:
		default:
		}
	}()

	if w.shouldStop() {
		return storage.STOP
	}

	k := &curationKey{
		Fingerprint:              fingerprint,
		ProcessorMessageRaw:      w.processor.Signature(),
		ProcessorMessageTypeName: w.processor.Name(),
		IgnoreYoungerThan:        w.ignoreYoungerThan,
	}

	curationRemark, present, err := w.curationState.Get(k)
	if err != nil {
		return
	}
	if !present {
		return storage.ACCEPT
	}
	if !curationRemark.Before(w.stopAt) {
		return storage.SKIP
	}
	watermark := value.(*watermarks)
	if !curationRemark.Before(watermark.High) {
		return storage.SKIP
	}
	curationConsistent, err := w.curationConsistent(fingerprint, watermark)
	if err != nil {
		return
	}
	if curationConsistent {
		return storage.SKIP
	}

	return storage.ACCEPT
}

// curationConsistent determines whether the given metric is in a dirty state
// and needs curation.
func (w *watermarkScanner) curationConsistent(f *clientmodel.Fingerprint, watermark *watermarks) (bool, error) {
	k := &curationKey{
		Fingerprint:              f,
		ProcessorMessageRaw:      w.processor.Signature(),
		ProcessorMessageTypeName: w.processor.Name(),
		IgnoreYoungerThan:        w.ignoreYoungerThan,
	}
	curationRemark, present, err := w.curationState.Get(k)
	if err != nil {
		return false, err
	}
	if !present {
		return false, nil
	}
	if !curationRemark.Before(watermark.High) {
		return true, nil
	}

	return false, nil
}

func (w *watermarkScanner) Operate(key, _ interface{}) (oErr *storage.OperatorError) {
	fingerprint := key.(*clientmodel.Fingerprint)

	seriesFrontier, present, err := newSeriesFrontier(fingerprint, w.diskFrontier, w.sampleIterator)
	if err != nil || !present {
		// An anomaly with the series frontier is severe in the sense that some sort
		// of an illegal state condition exists in the storage layer, which would
		// probably signify an illegal disk frontier.
		return &storage.OperatorError{error: err, Continuable: false}
	}

	curationState, present, err := w.curationState.Get(&curationKey{
		Fingerprint:              fingerprint,
		ProcessorMessageRaw:      w.processor.Signature(),
		ProcessorMessageTypeName: w.processor.Name(),
		IgnoreYoungerThan:        w.ignoreYoungerThan,
	})

	if err != nil {
		// An anomaly with the curation remark is likely not fatal in the sense that
		// there was a decoding error with the entity and shouldn't be cause to stop
		// work.  The process will simply start from a pessimistic work time and
		// work forward.  With an idempotent processor, this is safe.
		return &storage.OperatorError{error: err, Continuable: true}
	}
	var firstSeek time.Time
	switch {
	case !present, seriesFrontier.After(curationState):
		firstSeek = seriesFrontier.firstSupertime
	case !seriesFrontier.InSafeSeekRange(curationState):
		firstSeek = seriesFrontier.lastSupertime
	default:
		firstSeek = curationState
	}

	startKey := &SampleKey{
		Fingerprint:    fingerprint,
		FirstTimestamp: firstSeek,
	}
	dto := new(dto.SampleKey)

	startKey.Dump(dto)
	prospectiveKey := coding.NewPBEncoder(dto).MustEncode()
	if !w.sampleIterator.Seek(prospectiveKey) {
		// LevelDB is picky about the seek ranges.  If an iterator was invalidated,
		// no work may occur, and the iterator cannot be recovered.
		return &storage.OperatorError{error: fmt.Errorf("Illegal Condition: Iterator invalidated due to seek range."), Continuable: false}
	}

	newestAllowedSample := w.stopAt
	if !newestAllowedSample.Before(seriesFrontier.lastSupertime) {
		newestAllowedSample = seriesFrontier.lastSupertime
	}

	lastTime, err := w.processor.Apply(w.sampleIterator, w.samples, newestAllowedSample, fingerprint)
	if err != nil {
		// We can't divine the severity of a processor error without refactoring the
		// interface.
		return &storage.OperatorError{error: err, Continuable: false}
	}

	err = w.curationState.Update(&curationKey{
		Fingerprint:              fingerprint,
		ProcessorMessageRaw:      w.processor.Signature(),
		ProcessorMessageTypeName: w.processor.Name(),
		IgnoreYoungerThan:        w.ignoreYoungerThan,
	},
		lastTime)
	if err != nil {
		// Under the assumption that the processors are idempotent, they can be
		// re-run; thusly, the commitment of the curation remark is no cause
		// to cease further progress.
		return &storage.OperatorError{error: err, Continuable: true}
	}

	return nil
}

// curationKey provides a representation of dto.CurationKey with associated
// business logic methods attached to it to enhance code readability.
type curationKey struct {
	Fingerprint              *clientmodel.Fingerprint
	ProcessorMessageRaw      []byte
	ProcessorMessageTypeName string
	IgnoreYoungerThan        time.Duration
}

// Equal answers whether the two curationKeys are equivalent.
func (c *curationKey) Equal(o *curationKey) bool {
	switch {
	case !c.Fingerprint.Equal(o.Fingerprint):
		return false
	case bytes.Compare(c.ProcessorMessageRaw, o.ProcessorMessageRaw) != 0:
		return false
	case c.ProcessorMessageTypeName != o.ProcessorMessageTypeName:
		return false
	case c.IgnoreYoungerThan != o.IgnoreYoungerThan:
		return false
	}

	return true
}

func (c *curationKey) dump(d *dto.CurationKey) {
	d.Reset()

	// BUG(matt): Avenue for simplification.
	fingerprintDTO := &dto.Fingerprint{}

	dumpFingerprint(fingerprintDTO, c.Fingerprint)

	d.Fingerprint = fingerprintDTO
	d.ProcessorMessageRaw = c.ProcessorMessageRaw
	d.ProcessorMessageTypeName = proto.String(c.ProcessorMessageTypeName)
	d.IgnoreYoungerThan = proto.Int64(int64(c.IgnoreYoungerThan))
}

func (c *curationKey) load(d *dto.CurationKey) {
	// BUG(matt): Avenue for simplification.
	c.Fingerprint = &clientmodel.Fingerprint{}

	loadFingerprint(c.Fingerprint, d.Fingerprint)

	c.ProcessorMessageRaw = d.ProcessorMessageRaw
	c.ProcessorMessageTypeName = d.GetProcessorMessageTypeName()
	c.IgnoreYoungerThan = time.Duration(d.GetIgnoreYoungerThan())
}
<commit_msg>Code Review: Manual re-alignment.<commit_after>// Copyright 2013 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package metric

import (
	"bytes"
	"fmt"
	"strings"
	"time"

	"code.google.com/p/goprotobuf/proto"

	clientmodel "github.com/prometheus/client_golang/model"

	dto "github.com/prometheus/prometheus/model/generated"

	"github.com/prometheus/prometheus/coding"
	"github.com/prometheus/prometheus/storage"
	"github.com/prometheus/prometheus/storage/raw"
	"github.com/prometheus/prometheus/storage/raw/leveldb"
)

// CurationState contains high-level curation state information for the
// heads-up-display.
type CurationState struct {
	Active      bool
	Name        string
	Limit       time.Duration
	Fingerprint *clientmodel.Fingerprint
}

// curator is responsible for effectuating a given curation policy across the
// stored samples on-disk.  This is useful to compact sparse sample values into
// single sample entities to reduce keyspace load on the datastore.
type Curator struct {
	// Stop functions as a channel that when empty allows the curator to operate.
	// The moment a value is ingested inside of it, the curator goes into drain
	// mode.
	Stop chan bool
}

// watermarkScanner converts (dto.Fingerprint, dto.MetricHighWatermark) doubles
// into (model.Fingerprint, model.Watermark) doubles.
//
// watermarkScanner determines whether to include or exclude candidate
// values from the curation process by virtue of how old the high watermark is.
//
// watermarkScanner scans over the curator.samples table for metrics whose
// high watermark has been determined to be allowable for curation.  This type
// is individually responsible for compaction.
//
// The scanning starts from CurationRemark.LastCompletionTimestamp and goes
// forward until the stop point or end of the series is reached.
type watermarkScanner struct {
	// curationState is the data store for curation remarks.
	curationState CurationRemarker
	// diskFrontier models the available seekable ranges for the provided
	// sampleIterator.
	diskFrontier *diskFrontier
	// ignoreYoungerThan is passed into the curation remark for the given series.
	ignoreYoungerThan time.Duration
	// processor is responsible for executing a given stategy on the
	// to-be-operated-on series.
	processor Processor
	// sampleIterator is a snapshotted iterator for the time series.
	sampleIterator leveldb.Iterator
	// samples
	samples raw.Persistence
	// stopAt is a cue for when to stop mutating a given series.
	stopAt time.Time

	// stop functions as the global stop channel for all future operations.
	stop chan bool
	// status is the outbound channel for notifying the status page of its state.
	status chan CurationState
}

// run facilitates the curation lifecycle.
//
// recencyThreshold represents the most recent time up to which values will be
// curated.
// curationState is the on-disk store where the curation remarks are made for
// how much progress has been made.
func (c *Curator) Run(ignoreYoungerThan time.Duration, instant time.Time, processor Processor, curationState CurationRemarker, samples *leveldb.LevelDBPersistence, watermarks HighWatermarker, status chan CurationState) (err error) {
	defer func(t time.Time) {
		duration := float64(time.Since(t) / time.Millisecond)

		labels := map[string]string{
			cutOff:        fmt.Sprint(ignoreYoungerThan),
			processorName: processor.Name(),
			result:        success,
		}
		if err != nil {
			labels[result] = failure
		}

		curationDuration.IncrementBy(labels, duration)
		curationDurations.Add(labels, duration)
	}(time.Now())
	defer func() {
		select {
		case status <- CurationState{Active: false}:
		case <-status:
		default:
		}
	}()

	iterator := samples.NewIterator(true)
	defer iterator.Close()

	diskFrontier, present, err := newDiskFrontier(iterator)
	if err != nil {
		return
	}
	if !present {
		// No sample database exists; no work to do!
		return
	}

	scanner := &watermarkScanner{
		curationState:     curationState,
		ignoreYoungerThan: ignoreYoungerThan,
		processor:         processor,
		status:            status,
		stop:              c.Stop,
		stopAt:            instant.Add(-1 * ignoreYoungerThan),

		diskFrontier:   diskFrontier,
		sampleIterator: iterator,
		samples:        samples,
	}

	// Right now, the ability to stop a curation is limited to the beginning of
	// each fingerprint cycle.  It is impractical to cease the work once it has
	// begun for a given series.
	_, err = watermarks.ForEach(scanner, scanner, scanner)

	return
}

// drain instructs the curator to stop at the next convenient moment as to not
// introduce data inconsistencies.
func (c *Curator) Drain() {
	if len(c.Stop) == 0 {
		c.Stop <- true
	}
}

func (w *watermarkScanner) DecodeKey(in interface{}) (interface{}, error) {
	key := new(dto.Fingerprint)
	bytes := in.([]byte)

	if err := proto.Unmarshal(bytes, key); err != nil {
		return nil, err
	}

	fingerprint := new(clientmodel.Fingerprint)
	loadFingerprint(fingerprint, key)

	return fingerprint, nil
}

func (w *watermarkScanner) DecodeValue(in interface{}) (interface{}, error) {
	value := new(dto.MetricHighWatermark)
	bytes := in.([]byte)

	if err := proto.Unmarshal(bytes, value); err != nil {
		return nil, err
	}

	watermark := new(watermarks)
	watermark.load(value)

	return watermark, nil
}

func (w *watermarkScanner) shouldStop() bool {
	return len(w.stop) != 0
}

func (w *watermarkScanner) Filter(key, value interface{}) (r storage.FilterResult) {
	fingerprint := key.(*clientmodel.Fingerprint)

	defer func() {
		labels := map[string]string{
			cutOff:        fmt.Sprint(w.ignoreYoungerThan),
			result:        strings.ToLower(r.String()),
			processorName: w.processor.Name(),
		}

		curationFilterOperations.Increment(labels)

		select {
		case w.status <- CurationState{
			Active:      true,
			Name:        w.processor.Name(),
			Limit:       w.ignoreYoungerThan,
			Fingerprint: fingerprint,
		}:
		case <-w.status:
		default:
		}
	}()

	if w.shouldStop() {
		return storage.STOP
	}

	k := &curationKey{
		Fingerprint:              fingerprint,
		ProcessorMessageRaw:      w.processor.Signature(),
		ProcessorMessageTypeName: w.processor.Name(),
		IgnoreYoungerThan:        w.ignoreYoungerThan,
	}

	curationRemark, present, err := w.curationState.Get(k)
	if err != nil {
		return
	}
	if !present {
		return storage.ACCEPT
	}
	if !curationRemark.Before(w.stopAt) {
		return storage.SKIP
	}
	watermark := value.(*watermarks)
	if !curationRemark.Before(watermark.High) {
		return storage.SKIP
	}
	curationConsistent, err := w.curationConsistent(fingerprint, watermark)
	if err != nil {
		return
	}
	if curationConsistent {
		return storage.SKIP
	}

	return storage.ACCEPT
}

// curationConsistent determines whether the given metric is in a dirty state
// and needs curation.
func (w *watermarkScanner) curationConsistent(f *clientmodel.Fingerprint, watermark *watermarks) (bool, error) {
	k := &curationKey{
		Fingerprint:              f,
		ProcessorMessageRaw:      w.processor.Signature(),
		ProcessorMessageTypeName: w.processor.Name(),
		IgnoreYoungerThan:        w.ignoreYoungerThan,
	}
	curationRemark, present, err := w.curationState.Get(k)
	if err != nil {
		return false, err
	}
	if !present {
		return false, nil
	}
	if !curationRemark.Before(watermark.High) {
		return true, nil
	}

	return false, nil
}

func (w *watermarkScanner) Operate(key, _ interface{}) (oErr *storage.OperatorError) {
	fingerprint := key.(*clientmodel.Fingerprint)

	seriesFrontier, present, err := newSeriesFrontier(fingerprint, w.diskFrontier, w.sampleIterator)
	if err != nil || !present {
		// An anomaly with the series frontier is severe in the sense that some sort
		// of an illegal state condition exists in the storage layer, which would
		// probably signify an illegal disk frontier.
		return &storage.OperatorError{error: err, Continuable: false}
	}

	curationState, present, err := w.curationState.Get(&curationKey{
		Fingerprint:              fingerprint,
		ProcessorMessageRaw:      w.processor.Signature(),
		ProcessorMessageTypeName: w.processor.Name(),
		IgnoreYoungerThan:        w.ignoreYoungerThan,
	})

	if err != nil {
		// An anomaly with the curation remark is likely not fatal in the sense that
		// there was a decoding error with the entity and shouldn't be cause to stop
		// work.  The process will simply start from a pessimistic work time and
		// work forward.  With an idempotent processor, this is safe.
		return &storage.OperatorError{error: err, Continuable: true}
	}
	var firstSeek time.Time
	switch {
	case !present, seriesFrontier.After(curationState):
		firstSeek = seriesFrontier.firstSupertime
	case !seriesFrontier.InSafeSeekRange(curationState):
		firstSeek = seriesFrontier.lastSupertime
	default:
		firstSeek = curationState
	}

	startKey := &SampleKey{
		Fingerprint:    fingerprint,
		FirstTimestamp: firstSeek,
	}
	dto := new(dto.SampleKey)

	startKey.Dump(dto)
	prospectiveKey := coding.NewPBEncoder(dto).MustEncode()
	if !w.sampleIterator.Seek(prospectiveKey) {
		// LevelDB is picky about the seek ranges.  If an iterator was invalidated,
		// no work may occur, and the iterator cannot be recovered.
		return &storage.OperatorError{error: fmt.Errorf("Illegal Condition: Iterator invalidated due to seek range."), Continuable: false}
	}

	newestAllowedSample := w.stopAt
	if !newestAllowedSample.Before(seriesFrontier.lastSupertime) {
		newestAllowedSample = seriesFrontier.lastSupertime
	}

	lastTime, err := w.processor.Apply(w.sampleIterator, w.samples, newestAllowedSample, fingerprint)
	if err != nil {
		// We can't divine the severity of a processor error without refactoring the
		// interface.
		return &storage.OperatorError{error: err, Continuable: false}
	}

	err = w.curationState.Update(&curationKey{
		Fingerprint:              fingerprint,
		ProcessorMessageRaw:      w.processor.Signature(),
		ProcessorMessageTypeName: w.processor.Name(),
		IgnoreYoungerThan:        w.ignoreYoungerThan,
	}, lastTime)
	if err != nil {
		// Under the assumption that the processors are idempotent, they can be
		// re-run; thusly, the commitment of the curation remark is no cause
		// to cease further progress.
		return &storage.OperatorError{error: err, Continuable: true}
	}

	return nil
}

// curationKey provides a representation of dto.CurationKey with associated
// business logic methods attached to it to enhance code readability.
type curationKey struct {
	Fingerprint              *clientmodel.Fingerprint
	ProcessorMessageRaw      []byte
	ProcessorMessageTypeName string
	IgnoreYoungerThan        time.Duration
}

// Equal answers whether the two curationKeys are equivalent.
func (c *curationKey) Equal(o *curationKey) bool {
	switch {
	case !c.Fingerprint.Equal(o.Fingerprint):
		return false
	case bytes.Compare(c.ProcessorMessageRaw, o.ProcessorMessageRaw) != 0:
		return false
	case c.ProcessorMessageTypeName != o.ProcessorMessageTypeName:
		return false
	case c.IgnoreYoungerThan != o.IgnoreYoungerThan:
		return false
	}

	return true
}

func (c *curationKey) dump(d *dto.CurationKey) {
	d.Reset()

	// BUG(matt): Avenue for simplification.
	fingerprintDTO := &dto.Fingerprint{}

	dumpFingerprint(fingerprintDTO, c.Fingerprint)

	d.Fingerprint = fingerprintDTO
	d.ProcessorMessageRaw = c.ProcessorMessageRaw
	d.ProcessorMessageTypeName = proto.String(c.ProcessorMessageTypeName)
	d.IgnoreYoungerThan = proto.Int64(int64(c.IgnoreYoungerThan))
}

func (c *curationKey) load(d *dto.CurationKey) {
	// BUG(matt): Avenue for simplification.
	c.Fingerprint = &clientmodel.Fingerprint{}

	loadFingerprint(c.Fingerprint, d.Fingerprint)

	c.ProcessorMessageRaw = d.ProcessorMessageRaw
	c.ProcessorMessageTypeName = d.GetProcessorMessageTypeName()
	c.IgnoreYoungerThan = time.Duration(d.GetIgnoreYoungerThan())
}
<|endoftext|>


<commit_before>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Tamir Duberstein (tamird@gmail.com)

package storage

import (
	"net"
	"time"

	"github.com/cenk/backoff"
	"github.com/coreos/etcd/raft/raftpb"
	"github.com/pkg/errors"
	"golang.org/x/net/context"
	"google.golang.org/grpc"

	"github.com/cockroachdb/cockroach/gossip"
	"github.com/cockroachdb/cockroach/roachpb"
	"github.com/cockroachdb/cockroach/rpc"
	"github.com/cockroachdb/cockroach/util/log"
	"github.com/cockroachdb/cockroach/util/syncutil"
	"github.com/cockroachdb/cockroach/util/timeutil"
	"github.com/rubyist/circuitbreaker"
)

const (
	// Outgoing messages are queued per-replica on a channel of this size.
	raftSendBufferSize = 100

	// When no message has been queued for this duration, the corresponding
	// instance of processQueue will shut down.
	//
	// TODO(tamird): make culling of outbound streams more evented, so that we
	// need not rely on this timeout to shut things down.
	raftIdleTimeout = time.Minute
)

type raftMessageHandler func(*RaftMessageRequest) error

// NodeAddressResolver is the function used by RaftTransport to map node IDs to
// network addresses.
type NodeAddressResolver func(roachpb.NodeID) (net.Addr, error)

// GossipAddressResolver is a thin wrapper around gossip's GetNodeIDAddress
// that allows its return value to be used as the net.Addr interface.
func GossipAddressResolver(gossip *gossip.Gossip) NodeAddressResolver {
	return func(nodeID roachpb.NodeID) (net.Addr, error) {
		return gossip.GetNodeIDAddress(nodeID)
	}
}

// RaftSnapshotStatus contains a MsgSnap message and its resulting
// error, for asynchronous notification of completion.
type RaftSnapshotStatus struct {
	Req *RaftMessageRequest
	Err error
}

// RaftTransport handles the rpc messages for raft.
//
// The raft transport is asynchronous with respect to the caller, and
// internally multiplexes outbound messages. Internally, each message is
// queued on a per-destination queue before being asynchronously delivered.
//
// Callers are required to construct a RaftSender before being able to
// dispatch messages, and must provide an error handler which will be invoked
// asynchronously in the event that the recipient of any message closes its
// inbound RPC stream. This callback is asynchronous with respect to the
// outbound message which caused the remote to hang up; all that is known is
// which remote hung up.
type RaftTransport struct {
	resolver           NodeAddressResolver
	rpcContext         *rpc.Context
	SnapshotStatusChan chan RaftSnapshotStatus

	mu struct {
		syncutil.Mutex
		handlers     map[roachpb.StoreID]raftMessageHandler
		queues       map[bool]map[roachpb.ReplicaDescriptor]chan *RaftMessageRequest
		connBreakers map[roachpb.NodeID]*circuit.Breaker
	}
}

// NewDummyRaftTransport returns a dummy raft transport for use in tests which
// need a non-nil raft transport that need not function.
func NewDummyRaftTransport() *RaftTransport {
	return NewRaftTransport(nil, nil, nil)
}

// NewRaftTransport creates a new RaftTransport with specified resolver and grpc server.
// Callers are responsible for monitoring RaftTransport.SnapshotStatusChan.
func NewRaftTransport(resolver NodeAddressResolver, grpcServer *grpc.Server, rpcContext *rpc.Context) *RaftTransport {
	t := &RaftTransport{
		resolver:           resolver,
		rpcContext:         rpcContext,
		SnapshotStatusChan: make(chan RaftSnapshotStatus),
	}
	t.mu.handlers = make(map[roachpb.StoreID]raftMessageHandler)
	t.mu.queues = make(map[bool]map[roachpb.ReplicaDescriptor]chan *RaftMessageRequest)
	t.mu.connBreakers = make(map[roachpb.NodeID]*circuit.Breaker)

	if grpcServer != nil {
		RegisterMultiRaftServer(grpcServer, t)
	}

	return t
}

// RaftMessage proxies the incoming request to the listening server interface.
func (t *RaftTransport) RaftMessage(stream MultiRaft_RaftMessageServer) (err error) {
	errCh := make(chan error, 1)

	// Node stopping error is caught below in the select.
	if err := t.rpcContext.Stopper.RunTask(func() {
		t.rpcContext.Stopper.RunWorker(func() {
			errCh <- func() error {
				for {
					req, err := stream.Recv()
					if err != nil {
						return err
					}

					t.mu.Lock()
					handler, ok := t.mu.handlers[req.ToReplica.StoreID]
					t.mu.Unlock()

					if !ok {
						return errors.Errorf(
							"unable to accept Raft message from %+v: no store registered for %+v",
							req.ToReplica, req.FromReplica)
					}

					if err := handler(req); err != nil {
						return err
					}
				}
			}()
		})
	}); err != nil {
		return err
	}

	select {
	case err := <-errCh:
		return err
	case <-t.rpcContext.Stopper.ShouldQuiesce():
		return stream.SendAndClose(new(RaftMessageResponse))
	}
}

// Listen registers a raftMessageHandler to receive proxied messages.
func (t *RaftTransport) Listen(storeID roachpb.StoreID, handler raftMessageHandler) {
	t.mu.Lock()
	t.mu.handlers[storeID] = handler
	t.mu.Unlock()
}

// Stop unregisters a raftMessageHandler.
func (t *RaftTransport) Stop(storeID roachpb.StoreID) {
	t.mu.Lock()
	delete(t.mu.handlers, storeID)
	t.mu.Unlock()
}

// GetCircuitBreaker returns the circuit breaker controlling
// connection attempts to the specified node.
// NOTE: For unittesting.
func (t *RaftTransport) GetCircuitBreaker(nodeID roachpb.NodeID) *circuit.Breaker {
	t.mu.Lock()
	defer t.mu.Unlock()
	return t.mu.connBreakers[nodeID]
}

// getNodeConn returns a shared instance of a GRPC connection to the
// node specified by nodeID. Returns null if the remote node is not
// available (e.g. because the node ID can't be resolved or the remote
// node is not responding or is timing out, or the network is
// partitioned, etc.).
func (t *RaftTransport) getNodeConn(nodeID roachpb.NodeID) *grpc.ClientConn {
	t.mu.Lock()
	cb, ok := t.mu.connBreakers[nodeID]
	if !ok {
		// This exponential backoff limits the circuit breaker to 1 second
		// intervals between successive attempts to resolve a node address
		// and connect via GRPC.
		expBO := backoff.NewExponentialBackOff()
		expBO.MaxInterval = 1 * time.Second
		expBO.MaxElapsedTime = 0 * time.Second

		cb = circuit.NewBreakerWithOptions(&circuit.Options{
			BackOff:    expBO,
			ShouldTrip: circuit.ThresholdTripFunc(1),
		})
		t.mu.connBreakers[nodeID] = cb
	}
	t.mu.Unlock()

	// The number of consecutive failures suffered by the circuit breaker
	// is used to log only state changes in our connection status to the
	// remote node.
	consecFailures := cb.ConsecFailures()
	var conn *grpc.ClientConn
	if err := cb.Call(func() error {
		addr, err := t.resolver(nodeID)
		if err != nil {
			return err
		}
		conn, err = t.rpcContext.GRPCDial(addr.String())
		if err != nil {
			return err
		}
		if consecFailures > 0 {
			log.Infof(context.TODO(), "connection succeeded to node %s", nodeID)
		}
		return nil
	}, 0); err != nil {
		if consecFailures == 0 {
			log.Warningf(context.TODO(), "failed to connect to node %s: %s", nodeID, err)
		}
		return nil
	}
	return conn
}

// processQueue creates a client and sends messages from its designated queue
// via that client, exiting when the client fails or when it idles out. All
// messages remaining in the queue at that point are lost and a new instance of
// processQueue should be started by the next message to be sent.
// TODO(tschottdorf) should let raft know if the node is down;
// need a feedback mechanism for that. Potentially easiest is to arrange for
// the next call to Send() to fail appropriately.
func (t *RaftTransport) processQueue(ch chan *RaftMessageRequest, conn *grpc.ClientConn) error {
	client := NewMultiRaftClient(conn)
	ctx, cancel := context.WithCancel(context.TODO())
	defer cancel()
	stream, err := client.RaftMessage(ctx)
	if err != nil {
		return err
	}

	errCh := make(chan error, 1)

	// Starting workers in a task prevents data races during shutdown.
	if err := t.rpcContext.Stopper.RunTask(func() {
		t.rpcContext.Stopper.RunWorker(func() {
			errCh <- stream.RecvMsg(new(RaftMessageResponse))
		})
	}); err != nil {
		return err
	}

	var raftIdleTimer timeutil.Timer
	defer raftIdleTimer.Stop()
	for {
		raftIdleTimer.Reset(raftIdleTimeout)
		select {
		case <-t.rpcContext.Stopper.ShouldStop():
			return nil
		case <-raftIdleTimer.C:
			raftIdleTimer.Read = true
			return nil
		case err := <-errCh:
			return err
		case req := <-ch:
			err := stream.Send(req)
			if req.Message.Type == raftpb.MsgSnap {
				select {
				case <-t.rpcContext.Stopper.ShouldStop():
					return nil
				case t.SnapshotStatusChan <- RaftSnapshotStatus{req, err}:
				}

			}
			if err != nil {
				return err
			}
		}
	}
}

type errHandler func(error, roachpb.ReplicaDescriptor)

// RaftSender is a wrapper around RaftTransport that provides an error
// handler.
type RaftSender struct {
	transport *RaftTransport
	onError   errHandler
}

// MakeSender constructs a RaftSender with the provided error handler.
func (t *RaftTransport) MakeSender(onError errHandler) RaftSender {
	return RaftSender{transport: t, onError: onError}
}

// SendAsync sends a message to the recipient specified in the request. It
// returns false if the outgoing queue is full and calls s.onError when the
// recipient closes the stream.
func (s RaftSender) SendAsync(req *RaftMessageRequest) bool {
	isHeartbeat := (req.Message.Type == raftpb.MsgHeartbeat ||
		req.Message.Type == raftpb.MsgHeartbeatResp)
	if req.RangeID == 0 && !isHeartbeat {
		// Coalesced heartbeats are addressed to range 0; everything else
		// needs an explicit range ID.
		panic("only heartbeat messages may be sent to range ID 0")
	}
	isSnap := req.Message.Type == raftpb.MsgSnap
	toReplica := req.ToReplica
	// Get a connection to the node specified by the replica's node
	// ID. If no connection can be made, return false to indicate caller
	// should drop the Raft message.
	conn := s.transport.getNodeConn(toReplica.NodeID)
	if conn == nil {
		return false
	}

	s.transport.mu.Lock()
	// We use two queues; one will be used for snapshots, the other for all other
	// traffic. This is done to prevent snapshots from blocking other traffic.
	queues, ok := s.transport.mu.queues[isSnap]
	if !ok {
		queues = make(map[roachpb.ReplicaDescriptor]chan *RaftMessageRequest)
		s.transport.mu.queues[isSnap] = queues
	}
	ch, ok := queues[toReplica]
	if !ok {
		ch = make(chan *RaftMessageRequest, raftSendBufferSize)
		queues[toReplica] = ch
	}
	s.transport.mu.Unlock()

	if !ok {
		// Starting workers in a task prevents data races during shutdown.
		if err := s.transport.rpcContext.Stopper.RunTask(func() {
			s.transport.rpcContext.Stopper.RunWorker(func() {
				s.onError(s.transport.processQueue(ch, conn), toReplica)

				s.transport.mu.Lock()
				delete(queues, toReplica)
				s.transport.mu.Unlock()
			})
		}); err != nil {
			s.onError(err, toReplica)
		}
	}

	select {
	case ch <- req:
		return true
	default:
		return false
	}
}
<commit_msg>storage: only get a grpc connection if we're creating a queue<commit_after>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Tamir Duberstein (tamird@gmail.com)

package storage

import (
	"net"
	"time"

	"github.com/cenk/backoff"
	"github.com/coreos/etcd/raft/raftpb"
	"github.com/pkg/errors"
	"golang.org/x/net/context"
	"google.golang.org/grpc"

	"github.com/cockroachdb/cockroach/gossip"
	"github.com/cockroachdb/cockroach/roachpb"
	"github.com/cockroachdb/cockroach/rpc"
	"github.com/cockroachdb/cockroach/util/log"
	"github.com/cockroachdb/cockroach/util/syncutil"
	"github.com/cockroachdb/cockroach/util/timeutil"
	"github.com/rubyist/circuitbreaker"
)

const (
	// Outgoing messages are queued per-replica on a channel of this size.
	raftSendBufferSize = 100

	// When no message has been queued for this duration, the corresponding
	// instance of processQueue will shut down.
	//
	// TODO(tamird): make culling of outbound streams more evented, so that we
	// need not rely on this timeout to shut things down.
	raftIdleTimeout = time.Minute
)

type raftMessageHandler func(*RaftMessageRequest) error

// NodeAddressResolver is the function used by RaftTransport to map node IDs to
// network addresses.
type NodeAddressResolver func(roachpb.NodeID) (net.Addr, error)

// GossipAddressResolver is a thin wrapper around gossip's GetNodeIDAddress
// that allows its return value to be used as the net.Addr interface.
func GossipAddressResolver(gossip *gossip.Gossip) NodeAddressResolver {
	return func(nodeID roachpb.NodeID) (net.Addr, error) {
		return gossip.GetNodeIDAddress(nodeID)
	}
}

// RaftSnapshotStatus contains a MsgSnap message and its resulting
// error, for asynchronous notification of completion.
type RaftSnapshotStatus struct {
	Req *RaftMessageRequest
	Err error
}

// RaftTransport handles the rpc messages for raft.
//
// The raft transport is asynchronous with respect to the caller, and
// internally multiplexes outbound messages. Internally, each message is
// queued on a per-destination queue before being asynchronously delivered.
//
// Callers are required to construct a RaftSender before being able to
// dispatch messages, and must provide an error handler which will be invoked
// asynchronously in the event that the recipient of any message closes its
// inbound RPC stream. This callback is asynchronous with respect to the
// outbound message which caused the remote to hang up; all that is known is
// which remote hung up.
type RaftTransport struct {
	resolver           NodeAddressResolver
	rpcContext         *rpc.Context
	SnapshotStatusChan chan RaftSnapshotStatus

	mu struct {
		syncutil.Mutex
		handlers     map[roachpb.StoreID]raftMessageHandler
		queues       map[bool]map[roachpb.ReplicaDescriptor]chan *RaftMessageRequest
		connBreakers map[roachpb.NodeID]*circuit.Breaker
	}
}

// NewDummyRaftTransport returns a dummy raft transport for use in tests which
// need a non-nil raft transport that need not function.
func NewDummyRaftTransport() *RaftTransport {
	return NewRaftTransport(nil, nil, nil)
}

// NewRaftTransport creates a new RaftTransport with specified resolver and grpc server.
// Callers are responsible for monitoring RaftTransport.SnapshotStatusChan.
func NewRaftTransport(resolver NodeAddressResolver, grpcServer *grpc.Server, rpcContext *rpc.Context) *RaftTransport {
	t := &RaftTransport{
		resolver:           resolver,
		rpcContext:         rpcContext,
		SnapshotStatusChan: make(chan RaftSnapshotStatus),
	}
	t.mu.handlers = make(map[roachpb.StoreID]raftMessageHandler)
	t.mu.queues = make(map[bool]map[roachpb.ReplicaDescriptor]chan *RaftMessageRequest)
	t.mu.connBreakers = make(map[roachpb.NodeID]*circuit.Breaker)

	if grpcServer != nil {
		RegisterMultiRaftServer(grpcServer, t)
	}

	return t
}

// RaftMessage proxies the incoming request to the listening server interface.
func (t *RaftTransport) RaftMessage(stream MultiRaft_RaftMessageServer) (err error) {
	errCh := make(chan error, 1)

	// Node stopping error is caught below in the select.
	if err := t.rpcContext.Stopper.RunTask(func() {
		t.rpcContext.Stopper.RunWorker(func() {
			errCh <- func() error {
				for {
					req, err := stream.Recv()
					if err != nil {
						return err
					}

					t.mu.Lock()
					handler, ok := t.mu.handlers[req.ToReplica.StoreID]
					t.mu.Unlock()

					if !ok {
						return errors.Errorf(
							"unable to accept Raft message from %+v: no store registered for %+v",
							req.ToReplica, req.FromReplica)
					}

					if err := handler(req); err != nil {
						return err
					}
				}
			}()
		})
	}); err != nil {
		return err
	}

	select {
	case err := <-errCh:
		return err
	case <-t.rpcContext.Stopper.ShouldQuiesce():
		return stream.SendAndClose(new(RaftMessageResponse))
	}
}

// Listen registers a raftMessageHandler to receive proxied messages.
func (t *RaftTransport) Listen(storeID roachpb.StoreID, handler raftMessageHandler) {
	t.mu.Lock()
	t.mu.handlers[storeID] = handler
	t.mu.Unlock()
}

// Stop unregisters a raftMessageHandler.
func (t *RaftTransport) Stop(storeID roachpb.StoreID) {
	t.mu.Lock()
	delete(t.mu.handlers, storeID)
	t.mu.Unlock()
}

// GetCircuitBreaker returns the circuit breaker controlling
// connection attempts to the specified node.
// NOTE: For unittesting.
func (t *RaftTransport) GetCircuitBreaker(nodeID roachpb.NodeID) *circuit.Breaker {
	t.mu.Lock()
	defer t.mu.Unlock()
	return t.mu.connBreakers[nodeID]
}

// getNodeConn returns a shared instance of a GRPC connection to the
// node specified by nodeID. Returns null if the remote node is not
// available (e.g. because the node ID can't be resolved or the remote
// node is not responding or is timing out, or the network is
// partitioned, etc.).
func (t *RaftTransport) getNodeConn(nodeID roachpb.NodeID) *grpc.ClientConn {
	t.mu.Lock()
	cb, ok := t.mu.connBreakers[nodeID]
	if !ok {
		// This exponential backoff limits the circuit breaker to 1 second
		// intervals between successive attempts to resolve a node address
		// and connect via GRPC.
		expBO := backoff.NewExponentialBackOff()
		expBO.MaxInterval = 1 * time.Second
		expBO.MaxElapsedTime = 0 * time.Second

		cb = circuit.NewBreakerWithOptions(&circuit.Options{
			BackOff:    expBO,
			ShouldTrip: circuit.ThresholdTripFunc(1),
		})
		t.mu.connBreakers[nodeID] = cb
	}
	t.mu.Unlock()

	// The number of consecutive failures suffered by the circuit breaker
	// is used to log only state changes in our connection status to the
	// remote node.
	consecFailures := cb.ConsecFailures()
	var conn *grpc.ClientConn
	if err := cb.Call(func() error {
		addr, err := t.resolver(nodeID)
		if err != nil {
			return err
		}
		conn, err = t.rpcContext.GRPCDial(addr.String())
		if err != nil {
			return err
		}
		if consecFailures > 0 {
			log.Infof(context.TODO(), "connection succeeded to node %s", nodeID)
		}
		return nil
	}, 0); err != nil {
		if consecFailures == 0 {
			log.Warningf(context.TODO(), "failed to connect to node %s: %s", nodeID, err)
		}
		return nil
	}
	return conn
}

// processQueue creates a client and sends messages from its designated queue
// via that client, exiting when the client fails or when it idles out. All
// messages remaining in the queue at that point are lost and a new instance of
// processQueue should be started by the next message to be sent.
// TODO(tschottdorf) should let raft know if the node is down;
// need a feedback mechanism for that. Potentially easiest is to arrange for
// the next call to Send() to fail appropriately.
func (t *RaftTransport) processQueue(ch chan *RaftMessageRequest, conn *grpc.ClientConn) error {
	client := NewMultiRaftClient(conn)
	ctx, cancel := context.WithCancel(context.TODO())
	defer cancel()
	stream, err := client.RaftMessage(ctx)
	if err != nil {
		return err
	}

	errCh := make(chan error, 1)

	// Starting workers in a task prevents data races during shutdown.
	if err := t.rpcContext.Stopper.RunTask(func() {
		t.rpcContext.Stopper.RunWorker(func() {
			errCh <- stream.RecvMsg(new(RaftMessageResponse))
		})
	}); err != nil {
		return err
	}

	var raftIdleTimer timeutil.Timer
	defer raftIdleTimer.Stop()
	for {
		raftIdleTimer.Reset(raftIdleTimeout)
		select {
		case <-t.rpcContext.Stopper.ShouldStop():
			return nil
		case <-raftIdleTimer.C:
			raftIdleTimer.Read = true
			return nil
		case err := <-errCh:
			return err
		case req := <-ch:
			err := stream.Send(req)
			if req.Message.Type == raftpb.MsgSnap {
				select {
				case <-t.rpcContext.Stopper.ShouldStop():
					return nil
				case t.SnapshotStatusChan <- RaftSnapshotStatus{req, err}:
				}

			}
			if err != nil {
				return err
			}
		}
	}
}

type errHandler func(error, roachpb.ReplicaDescriptor)

// RaftSender is a wrapper around RaftTransport that provides an error
// handler.
type RaftSender struct {
	transport *RaftTransport
	onError   errHandler
}

// MakeSender constructs a RaftSender with the provided error handler.
func (t *RaftTransport) MakeSender(onError errHandler) RaftSender {
	return RaftSender{transport: t, onError: onError}
}

// SendAsync sends a message to the recipient specified in the request. It
// returns false if the outgoing queue is full and calls s.onError when the
// recipient closes the stream.
func (s RaftSender) SendAsync(req *RaftMessageRequest) bool {
	isHeartbeat := (req.Message.Type == raftpb.MsgHeartbeat ||
		req.Message.Type == raftpb.MsgHeartbeatResp)
	if req.RangeID == 0 && !isHeartbeat {
		// Coalesced heartbeats are addressed to range 0; everything else
		// needs an explicit range ID.
		panic("only heartbeat messages may be sent to range ID 0")
	}
	isSnap := req.Message.Type == raftpb.MsgSnap
	toReplica := req.ToReplica

	s.transport.mu.Lock()
	// We use two queues; one will be used for snapshots, the other for all other
	// traffic. This is done to prevent snapshots from blocking other traffic.
	queues, ok := s.transport.mu.queues[isSnap]
	if !ok {
		queues = make(map[roachpb.ReplicaDescriptor]chan *RaftMessageRequest)
		s.transport.mu.queues[isSnap] = queues
	}
	ch, ok := queues[toReplica]
	if !ok {
		ch = make(chan *RaftMessageRequest, raftSendBufferSize)
		queues[toReplica] = ch
	}
	s.transport.mu.Unlock()

	if !ok {
		// Get a connection to the node specified by the replica's node
		// ID. If no connection can be made, return false to indicate caller
		// should drop the Raft message.
		conn := s.transport.getNodeConn(toReplica.NodeID)
		if conn == nil {
			s.transport.mu.Lock()
			delete(queues, toReplica)
			s.transport.mu.Unlock()
			return false
		}

		// Starting workers in a task prevents data races during shutdown.
		if err := s.transport.rpcContext.Stopper.RunTask(func() {
			s.transport.rpcContext.Stopper.RunWorker(func() {
				s.onError(s.transport.processQueue(ch, conn), toReplica)

				s.transport.mu.Lock()
				delete(queues, toReplica)
				s.transport.mu.Unlock()
			})
		}); err != nil {
			s.onError(err, toReplica)
		}
	}

	select {
	case ch <- req:
		return true
	default:
		return false
	}
}
<|endoftext|>


<commit_before>// +build linux

package storage

import (
	"io/ioutil"
	"os"
	"testing"
	"math/rand"
	"github.com/golang/glog"
	"github.com/stretchr/testify/assert"
)

func TestWAL(t *testing.T) {
	assert := assert.New(t)

	//Create temp directory
	dir, err := ioutil.TempDir("", "IosWALTests")
	if err != nil {
		glog.Fatal(err)
	}
	defer os.RemoveAll(dir) // clean up

	//create file
  testFile := dir + "/test.temp"
	wal := openWriteAheadFile(testFile, "fsync")
	actualBytes, err := ioutil.ReadFile(testFile)
	assert.Equal(64*1000*1000,len(actualBytes), "File is expected size")

	//verfiy that write ahead logging works
  expectedBytes := make([]byte, 100)
	rand.Read(expectedBytes)
  wal.writeAhead(expectedBytes)
  actualBytes, err = ioutil.ReadFile(testFile)
  assert.Nil(err)
  //assert.Equal(1001,len(actualBytes), "Number of bytes read is not same as bytes written")
  assert.Equal(expectedBytes,actualBytes[len(actualBytes)-100+1:], "Bytes read are not same as written")

}
<commit_msg>shifting write ahead test bytes by 1<commit_after>// +build linux

package storage

import (
	"io/ioutil"
	"os"
	"testing"
	"math/rand"
	"github.com/golang/glog"
	"github.com/stretchr/testify/assert"
)

func TestWAL(t *testing.T) {
	assert := assert.New(t)

	//Create temp directory
	dir, err := ioutil.TempDir("", "IosWALTests")
	if err != nil {
		glog.Fatal(err)
	}
	defer os.RemoveAll(dir) // clean up

	//create file
  testFile := dir + "/test.temp"
	wal := openWriteAheadFile(testFile, "fsync")
	actualBytes, err := ioutil.ReadFile(testFile)
	assert.Equal(64*1000*1000,len(actualBytes), "File is expected size")

	//verfiy that write ahead logging works
  expectedBytes := make([]byte, 100)
	rand.Read(expectedBytes)
  wal.writeAhead(expectedBytes)
  actualBytes, err = ioutil.ReadFile(testFile)
  assert.Nil(err)
  //assert.Equal(1001,len(actualBytes), "Number of bytes read is not same as bytes written")
  assert.Equal(expectedBytes,actualBytes[len(actualBytes)-100-1:], "Bytes read are not same as written")

}
<|endoftext|>