<commit_before>// Copyright 2015 ThoughtWorks, Inc.

// This file is part of getgauge/html-report.

// getgauge/html-report is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// getgauge/html-report is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with getgauge/html-report.  If not, see <http://www.gnu.org/licenses/>.
package generator

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"text/template"

	"path"

	"github.com/getgauge/common"
	"github.com/getgauge/html-report/env"
	"github.com/getgauge/html-report/theme"
	"github.com/microcosm-cc/bluemonday"
	"github.com/russross/blackfriday"
)

type summary struct {
	Total   int
	Failed  int
	Passed  int
	Skipped int
}

type overview struct {
	ProjectName   string
	Env           string
	Tags          string
	SuccessRate   float32
	ExecutionTime string
	Timestamp     string
	Summary       *summary
	BasePath      string
}

type specsMeta struct {
	SpecName      string
	ExecutionTime string
	Failed        bool
	Skipped       bool
	Tags          []string
	ReportFile    string
}

type sidebar struct {
	IsBeforeHookFailure bool
	Specs               []*specsMeta
}

type specHeader struct {
	SpecName      string
	ExecutionTime string
	FileName      string
	Tags          []string
	Summary       *summary
}

type errorType string
type tokenKind string
type status string

type buildError struct {
	ErrorType  errorType
	FileName   string
	LineNumber int
	Message    string
}

// SuiteResult holds the aggregated execution information for a run
type SuiteResult struct {
	ProjectName            string       `json:"ProjectName"`
	Timestamp              string       `json:"Timestamp"`
	SuccessRate            float32      `json:"SuccessRate"`
	Environment            string       `json:"Environment"`
	Tags                   string       `json:"Tags"`
	ExecutionTime          int64        `json:"ExecutionTime"`
	ExecutionStatus        status       `json:"ExecutionStatus"`
	SpecResults            []*spec      `json:"SpecResults"`
	BeforeSuiteHookFailure *hookFailure `json:"BeforeSuiteHookFailure"`
	AfterSuiteHookFailure  *hookFailure `json:"AfterSuiteHookFailure"`
	PassedSpecsCount       int          `json:"PassedSpecsCount"`
	FailedSpecsCount       int          `json:"FailedSpecsCount"`
	SkippedSpecsCount      int          `json:"SkippedSpecsCount"`
	BasePath               string       `json:"BasePath"`
}

type spec struct {
	CommentsBeforeDatatable []string       `json:"CommentsBeforeDatatable"`
	CommentsAfterDatatable  []string       `json:"CommentsAfterDatatable"`
	SpecHeading             string         `json:"SpecHeading"`
	FileName                string         `json:"FileName"`
	Tags                    []string       `json:"Tags"`
	ExecutionTime           int64          `json:"ExecutionTime"`
	ExecutionStatus         status         `json:"ExecutionStatus"`
	Scenarios               []*scenario    `json:"Scenarios"`
	IsTableDriven           bool           `json:"IsTableDriven"`
	Datatable               *table         `json:"Datatable"`
	BeforeSpecHookFailures  []*hookFailure `json:"BeforeSpecHookFailures"`
	AfterSpecHookFailures   []*hookFailure `json:"AfterSpecHookFailures"`
	PassedScenarioCount     int            `json:"PassedScenarioCount"`
	FailedScenarioCount     int            `json:"FailedScenarioCount"`
	SkippedScenarioCount    int            `json:"SkippedScenarioCount"`
	Errors                  []buildError   `json:"Errors"`
}

type scenario struct {
	Heading                   string       `json:"Heading"`
	Tags                      []string     `json:"Tags"`
	ExecutionTime             string       `json:"ExecutionTime"`
	ExecutionStatus           status       `json:"ExecutionStatus"`
	Contexts                  []item       `json:"Contexts"`
	Teardowns                 []item       `json:"Teardowns"`
	Items                     []item       `json:"Items"`
	BeforeScenarioHookFailure *hookFailure `json:"BeforeScenarioHookFailure"`
	AfterScenarioHookFailure  *hookFailure `json:"AfterScenarioHookFailure"`
	SkipErrors                []string     `json:"SkipErrors"`
	TableRowIndex             int          `json:"TableRowIndex"`
}

type step struct {
	Fragments             []*fragment  `json:"Fragments"`
	ItemType              tokenKind    `json:"ItemType"`
	StepText              string       `json:"StepText"`
	Table                 *table       `json:"Table"`
	BeforeStepHookFailure *hookFailure `json:"BeforeStepHookFailure"`
	AfterStepHookFailure  *hookFailure `json:"AfterStepHookFailure"`
	Result                *result      `json:"Result"`
}

func (s *step) Kind() tokenKind {
	return stepKind
}

type result struct {
	Status        status    `json:"Status"`
	StackTrace    string    `json:"StackTrace"`
	Screenshot    string    `json:"Screenshot"`
	ErrorMessage  string    `json:"ErrorMessage"`
	ExecutionTime string    `json:"ExecutionTime"`
	SkippedReason string    `json:"SkippedReason"`
	Messages      []string  `json:"Messages"`
	ErrorType     errorType `json:"ErrorType"`
}

type hookFailure struct {
	HookName      string `json:"HookName"`
	ErrMsg        string `json:"ErrMsg"`
	Screenshot    string `json:"Screenshot"`
	StackTrace    string `json:"StackTrace"`
	TableRowIndex int32  `json:"TableRowIndex"`
}

type concept struct {
	ItemType    tokenKind `json:"ItemType"`
	ConceptStep *step     `json:"ConceptStep"`
	Items       []item    `json:"Items"`
	Result      result    `json:"Result"`
}

func (s *concept) Kind() tokenKind {
	return conceptKind
}

type table struct {
	Headers []string `json:"Headers"`
	Rows    []*row   `json:"Rows"`
}

type row struct {
	Cells  []string `json:"Cells"`
	Result status   `json:"Status"`
}

func (e buildError) Error() string {
	if e.isParseError() {
		return "[Parse Error] " + e.Message
	}
	return "[Validation Error] " + e.Message
}

func (e buildError) isParseError() bool {
	return e.ErrorType == parseErrorType
}

type item struct {
	Kind    tokenKind
	Step    *step
	Concept *concept
	Comment *comment
}

type comment struct {
	Text string
}

func (c *comment) Kind() tokenKind {
	return commentKind
}

type searchIndex struct {
	Tags  map[string][]string `json:"Tags"`
	Specs map[string][]string `json:"Specs"`
}

const (
	pass                  status    = "pass"
	fail                  status    = "fail"
	skip                  status    = "skip"
	notExecuted           status    = "not executed"
	stepKind              tokenKind = "step"
	conceptKind           tokenKind = "concept"
	commentKind           tokenKind = "comment"
	assertionErrorType    errorType = "assertion"
	parseErrorType        errorType = "parse"
	verificationErrorType errorType = "verification"
	validationErrorType   errorType = "validation"
)

var parsedTemplates *template.Template

func readTemplates(themePath string) {
	var encodeNewLine = func(s string) string {
		return strings.Replace(s, "\n", "<br/>", -1)
	}
	var parseMarkdown = func(args ...interface{}) string {
		s := blackfriday.MarkdownCommon([]byte(fmt.Sprintf("%s", args...)))
		return string(s)
	}
	var sanitizeHTML = func(s string) string {
		var b bytes.Buffer
		var html = bluemonday.UGCPolicy().SanitizeBytes([]byte(s))
		b.Write(html)
		return b.String()
	}

	var funcs = template.FuncMap{
		"parseMarkdown":       parseMarkdown,
		"sanitize":            sanitizeHTML,
		"escapeHTML":          template.HTMLEscapeString,
		"encodeNewLine":       encodeNewLine,
		"containsParseErrors": containsParseErrors,
		"toSpecHeader":        toSpecHeader,
		"toSidebar":           toSidebar,
		"toOverview":          toOverview,
		"toPath":              path.Join,
	}
	f, err := ioutil.ReadFile(filepath.Join(getAbsThemePath(themePath), "views", "partials.tmpl"))
	if err != nil {
		log.Fatalf(err.Error())
	}
	parsedTemplates, err = template.New("Reports").Funcs(funcs).Parse(string(f))
	if err != nil {
		log.Fatalf(err.Error())
	}
}

func getAbsThemePath(themePath string) string {
	if filepath.IsAbs(themePath) {
		return themePath
	}
	return filepath.Join(projectRoot, themePath)
}

func execTemplate(tmplName string, w io.Writer, data interface{}) {
	err := parsedTemplates.ExecuteTemplate(w, tmplName, data)
	if err != nil {
		log.Fatalf(err.Error())
	}
}

// ProjectRoot is root dir of current project
var projectRoot string

// GenerateReports generates HTML report in the given report dir location
func GenerateReports(res *SuiteResult, reportsDir, themePath string) error {
	readTemplates(themePath)
	f, err := os.Create(filepath.Join(reportsDir, "index.html"))
	if err != nil {
		return err
	}
	defer f.Close()
	if res.BeforeSuiteHookFailure != nil {
		execTemplate("indexPageFailure", f, res)
	} else {
		var wg sync.WaitGroup
		wg.Add(1)
		res.BasePath = ""
		go generateIndexPage(res, f, &wg)
		if env.ShouldUseNestedSpecs() {
			go generateIndexPages(res, reportsDir, &wg)
		}
		specRes := res.SpecResults
		for _, r := range specRes {
			relPath, _ := filepath.Rel(projectRoot, r.FileName)
			env.CreateDirectory(filepath.Join(reportsDir, filepath.Dir(relPath)))
			sf, err := os.Create(filepath.Join(reportsDir, toHTMLFileName(r.FileName, projectRoot)))
			if err != nil {
				return err
			}
			defer sf.Close()
			wg.Add(1)
			go generateSpecPage(res, r, sf, &wg)
		}
		wg.Wait()
	}
	err = generateSearchIndex(res, reportsDir)
	if err != nil {
		return err
	}
	return nil
}

func GenerateReport(res *SuiteResult, reportDir, themePath string) {
	err := GenerateReports(res, reportDir, themePath)
	if err != nil {
		log.Fatalf("Failed to generate reports: %s\n", err.Error())
	}
	err = theme.CopyReportTemplateFiles(themePath, reportDir)
	if err != nil {
		log.Fatalf("Error copying template directory :%s\n", err.Error())
	}
	fmt.Printf("Successfully generated html-report to => %s\n", reportDir)
}

func newSearchIndex() *searchIndex {
	var i searchIndex
	i.Tags = make(map[string][]string)
	i.Specs = make(map[string][]string)
	return &i
}

func (i *searchIndex) hasValueForTag(tag string, spec string) bool {
	for _, s := range i.Tags[tag] {
		if s == spec {
			return true
		}
	}
	return false
}

func (i *searchIndex) hasSpec(specHeading string, specFileName string) bool {
	for _, s := range i.Specs[specHeading] {
		if s == specFileName {
			return true
		}
	}
	return false
}

func containsParseErrors(errors []buildError) bool {
	for _, e := range errors {
		if e.isParseError() {
			return true
		}
	}
	return false
}

func generateSearchIndex(suiteRes *SuiteResult, reportsDir string) error {
	env.CreateDirectory(filepath.Join(reportsDir, "js"))
	f, err := os.Create(filepath.Join(reportsDir, "js", "search_index.js"))
	if err != nil {
		return err
	}
	defer f.Close()
	index := newSearchIndex()
	for _, r := range suiteRes.SpecResults {
		specFileName := toHTMLFileName(r.FileName, projectRoot)
		for _, t := range r.Tags {
			if !index.hasValueForTag(t, specFileName) {
				index.Tags[t] = append(index.Tags[t], specFileName)
			}
		}
		for _, s := range r.Scenarios {
			for _, t := range s.Tags {
				if !index.hasValueForTag(t, specFileName) {
					index.Tags[t] = append(index.Tags[t], specFileName)
				}
			}
		}
		specHeading := r.SpecHeading
		if !index.hasSpec(specHeading, specFileName) {
			index.Specs[specHeading] = append(index.Specs[specHeading], specFileName)
		}
	}
	s, err := json.Marshal(index)
	if err != nil {
		return err
	}
	f.WriteString(fmt.Sprintf("var index = %s;", s))
	return nil
}

func generateIndexPage(suiteRes *SuiteResult, w io.Writer, wg *sync.WaitGroup) {
	defer wg.Done()
	execTemplate("indexPage", w, suiteRes)
}

func generateIndexPages(suiteRes *SuiteResult, reportsDir string, wg *sync.WaitGroup) {
	dirs := make(map[string]int)
	for _, s := range suiteRes.SpecResults {
		p, err := filepath.Rel(projectRoot, filepath.Dir(s.FileName))
		if err != nil {
			log.Fatal(err)
		}
		childDirs := filepath.SplitList(p)
		basePath := ""
		for _, d := range childDirs {
			if _, ok := dirs[d]; !ok {
				dirs[d] = 1
			}
			basePath = filepath.Join(basePath, d)
		}

	}
	delete(dirs, ".")
	for d := range dirs {
		dirPath := filepath.Join(reportsDir, d)
		os.MkdirAll(dirPath, common.NewDirectoryPermissions)
		p := filepath.Join(dirPath, "index.html")
		f, err := os.Create(p)
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()
		wg.Add(1)
		res := toNestedSuiteResult(d, suiteRes)
		generateIndexPage(res, f, wg)
	}
}

func generateSpecPage(suiteRes *SuiteResult, specRes *spec, w io.Writer, wg *sync.WaitGroup) {
	defer wg.Done()
	execTemplate("specPage", w, struct {
		SuiteRes *SuiteResult
		SpecRes  *spec
	}{suiteRes, specRes})
}
<commit_msg>Printing link to index file in console<commit_after>// Copyright 2015 ThoughtWorks, Inc.

// This file is part of getgauge/html-report.

// getgauge/html-report is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// getgauge/html-report is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with getgauge/html-report.  If not, see <http://www.gnu.org/licenses/>.
package generator

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"text/template"

	"path"

	"github.com/getgauge/common"
	"github.com/getgauge/html-report/env"
	"github.com/getgauge/html-report/theme"
	"github.com/microcosm-cc/bluemonday"
	"github.com/russross/blackfriday"
)

type summary struct {
	Total   int
	Failed  int
	Passed  int
	Skipped int
}

type overview struct {
	ProjectName   string
	Env           string
	Tags          string
	SuccessRate   float32
	ExecutionTime string
	Timestamp     string
	Summary       *summary
	BasePath      string
}

type specsMeta struct {
	SpecName      string
	ExecutionTime string
	Failed        bool
	Skipped       bool
	Tags          []string
	ReportFile    string
}

type sidebar struct {
	IsBeforeHookFailure bool
	Specs               []*specsMeta
}

type specHeader struct {
	SpecName      string
	ExecutionTime string
	FileName      string
	Tags          []string
	Summary       *summary
}

type errorType string
type tokenKind string
type status string

type buildError struct {
	ErrorType  errorType
	FileName   string
	LineNumber int
	Message    string
}

// SuiteResult holds the aggregated execution information for a run
type SuiteResult struct {
	ProjectName            string       `json:"ProjectName"`
	Timestamp              string       `json:"Timestamp"`
	SuccessRate            float32      `json:"SuccessRate"`
	Environment            string       `json:"Environment"`
	Tags                   string       `json:"Tags"`
	ExecutionTime          int64        `json:"ExecutionTime"`
	ExecutionStatus        status       `json:"ExecutionStatus"`
	SpecResults            []*spec      `json:"SpecResults"`
	BeforeSuiteHookFailure *hookFailure `json:"BeforeSuiteHookFailure"`
	AfterSuiteHookFailure  *hookFailure `json:"AfterSuiteHookFailure"`
	PassedSpecsCount       int          `json:"PassedSpecsCount"`
	FailedSpecsCount       int          `json:"FailedSpecsCount"`
	SkippedSpecsCount      int          `json:"SkippedSpecsCount"`
	BasePath               string       `json:"BasePath"`
}

type spec struct {
	CommentsBeforeDatatable []string       `json:"CommentsBeforeDatatable"`
	CommentsAfterDatatable  []string       `json:"CommentsAfterDatatable"`
	SpecHeading             string         `json:"SpecHeading"`
	FileName                string         `json:"FileName"`
	Tags                    []string       `json:"Tags"`
	ExecutionTime           int64          `json:"ExecutionTime"`
	ExecutionStatus         status         `json:"ExecutionStatus"`
	Scenarios               []*scenario    `json:"Scenarios"`
	IsTableDriven           bool           `json:"IsTableDriven"`
	Datatable               *table         `json:"Datatable"`
	BeforeSpecHookFailures  []*hookFailure `json:"BeforeSpecHookFailures"`
	AfterSpecHookFailures   []*hookFailure `json:"AfterSpecHookFailures"`
	PassedScenarioCount     int            `json:"PassedScenarioCount"`
	FailedScenarioCount     int            `json:"FailedScenarioCount"`
	SkippedScenarioCount    int            `json:"SkippedScenarioCount"`
	Errors                  []buildError   `json:"Errors"`
}

type scenario struct {
	Heading                   string       `json:"Heading"`
	Tags                      []string     `json:"Tags"`
	ExecutionTime             string       `json:"ExecutionTime"`
	ExecutionStatus           status       `json:"ExecutionStatus"`
	Contexts                  []item       `json:"Contexts"`
	Teardowns                 []item       `json:"Teardowns"`
	Items                     []item       `json:"Items"`
	BeforeScenarioHookFailure *hookFailure `json:"BeforeScenarioHookFailure"`
	AfterScenarioHookFailure  *hookFailure `json:"AfterScenarioHookFailure"`
	SkipErrors                []string     `json:"SkipErrors"`
	TableRowIndex             int          `json:"TableRowIndex"`
}

type step struct {
	Fragments             []*fragment  `json:"Fragments"`
	ItemType              tokenKind    `json:"ItemType"`
	StepText              string       `json:"StepText"`
	Table                 *table       `json:"Table"`
	BeforeStepHookFailure *hookFailure `json:"BeforeStepHookFailure"`
	AfterStepHookFailure  *hookFailure `json:"AfterStepHookFailure"`
	Result                *result      `json:"Result"`
}

func (s *step) Kind() tokenKind {
	return stepKind
}

type result struct {
	Status        status    `json:"Status"`
	StackTrace    string    `json:"StackTrace"`
	Screenshot    string    `json:"Screenshot"`
	ErrorMessage  string    `json:"ErrorMessage"`
	ExecutionTime string    `json:"ExecutionTime"`
	SkippedReason string    `json:"SkippedReason"`
	Messages      []string  `json:"Messages"`
	ErrorType     errorType `json:"ErrorType"`
}

type hookFailure struct {
	HookName      string `json:"HookName"`
	ErrMsg        string `json:"ErrMsg"`
	Screenshot    string `json:"Screenshot"`
	StackTrace    string `json:"StackTrace"`
	TableRowIndex int32  `json:"TableRowIndex"`
}

type concept struct {
	ItemType    tokenKind `json:"ItemType"`
	ConceptStep *step     `json:"ConceptStep"`
	Items       []item    `json:"Items"`
	Result      result    `json:"Result"`
}

func (s *concept) Kind() tokenKind {
	return conceptKind
}

type table struct {
	Headers []string `json:"Headers"`
	Rows    []*row   `json:"Rows"`
}

type row struct {
	Cells  []string `json:"Cells"`
	Result status   `json:"Status"`
}

func (e buildError) Error() string {
	if e.isParseError() {
		return "[Parse Error] " + e.Message
	}
	return "[Validation Error] " + e.Message
}

func (e buildError) isParseError() bool {
	return e.ErrorType == parseErrorType
}

type item struct {
	Kind    tokenKind
	Step    *step
	Concept *concept
	Comment *comment
}

type comment struct {
	Text string
}

func (c *comment) Kind() tokenKind {
	return commentKind
}

type searchIndex struct {
	Tags  map[string][]string `json:"Tags"`
	Specs map[string][]string `json:"Specs"`
}

const (
	pass                  status    = "pass"
	fail                  status    = "fail"
	skip                  status    = "skip"
	notExecuted           status    = "not executed"
	stepKind              tokenKind = "step"
	conceptKind           tokenKind = "concept"
	commentKind           tokenKind = "comment"
	assertionErrorType    errorType = "assertion"
	parseErrorType        errorType = "parse"
	verificationErrorType errorType = "verification"
	validationErrorType   errorType = "validation"
)

var parsedTemplates *template.Template

func readTemplates(themePath string) {
	var encodeNewLine = func(s string) string {
		return strings.Replace(s, "\n", "<br/>", -1)
	}
	var parseMarkdown = func(args ...interface{}) string {
		s := blackfriday.MarkdownCommon([]byte(fmt.Sprintf("%s", args...)))
		return string(s)
	}
	var sanitizeHTML = func(s string) string {
		var b bytes.Buffer
		var html = bluemonday.UGCPolicy().SanitizeBytes([]byte(s))
		b.Write(html)
		return b.String()
	}

	var funcs = template.FuncMap{
		"parseMarkdown":       parseMarkdown,
		"sanitize":            sanitizeHTML,
		"escapeHTML":          template.HTMLEscapeString,
		"encodeNewLine":       encodeNewLine,
		"containsParseErrors": containsParseErrors,
		"toSpecHeader":        toSpecHeader,
		"toSidebar":           toSidebar,
		"toOverview":          toOverview,
		"toPath":              path.Join,
	}
	f, err := ioutil.ReadFile(filepath.Join(getAbsThemePath(themePath), "views", "partials.tmpl"))
	if err != nil {
		log.Fatalf(err.Error())
	}
	parsedTemplates, err = template.New("Reports").Funcs(funcs).Parse(string(f))
	if err != nil {
		log.Fatalf(err.Error())
	}
}

func getAbsThemePath(themePath string) string {
	if filepath.IsAbs(themePath) {
		return themePath
	}
	return filepath.Join(projectRoot, themePath)
}

func execTemplate(tmplName string, w io.Writer, data interface{}) {
	err := parsedTemplates.ExecuteTemplate(w, tmplName, data)
	if err != nil {
		log.Fatalf(err.Error())
	}
}

// ProjectRoot is root dir of current project
var projectRoot string

// GenerateReports generates HTML report in the given report dir location
func GenerateReports(res *SuiteResult, reportsDir, themePath string) error {
	readTemplates(themePath)
	f, err := os.Create(filepath.Join(reportsDir, "index.html"))
	if err != nil {
		return err
	}
	defer f.Close()
	if res.BeforeSuiteHookFailure != nil {
		execTemplate("indexPageFailure", f, res)
	} else {
		var wg sync.WaitGroup
		wg.Add(1)
		res.BasePath = ""
		go generateIndexPage(res, f, &wg)
		if env.ShouldUseNestedSpecs() {
			go generateIndexPages(res, reportsDir, &wg)
		}
		specRes := res.SpecResults
		for _, r := range specRes {
			relPath, _ := filepath.Rel(projectRoot, r.FileName)
			env.CreateDirectory(filepath.Join(reportsDir, filepath.Dir(relPath)))
			sf, err := os.Create(filepath.Join(reportsDir, toHTMLFileName(r.FileName, projectRoot)))
			if err != nil {
				return err
			}
			defer sf.Close()
			wg.Add(1)
			go generateSpecPage(res, r, sf, &wg)
		}
		wg.Wait()
	}
	err = generateSearchIndex(res, reportsDir)
	if err != nil {
		return err
	}
	return nil
}

func GenerateReport(res *SuiteResult, reportDir, themePath string) {
	err := GenerateReports(res, reportDir, themePath)
	if err != nil {
		log.Fatalf("Failed to generate reports: %s\n", err.Error())
	}
	err = theme.CopyReportTemplateFiles(themePath, reportDir)
	if err != nil {
		log.Fatalf("Error copying template directory :%s\n", err.Error())
	}
	fmt.Printf("Successfully generated html-report to => %s\n", filepath.Join(reportDir, "index.html"))
}

func newSearchIndex() *searchIndex {
	var i searchIndex
	i.Tags = make(map[string][]string)
	i.Specs = make(map[string][]string)
	return &i
}

func (i *searchIndex) hasValueForTag(tag string, spec string) bool {
	for _, s := range i.Tags[tag] {
		if s == spec {
			return true
		}
	}
	return false
}

func (i *searchIndex) hasSpec(specHeading string, specFileName string) bool {
	for _, s := range i.Specs[specHeading] {
		if s == specFileName {
			return true
		}
	}
	return false
}

func containsParseErrors(errors []buildError) bool {
	for _, e := range errors {
		if e.isParseError() {
			return true
		}
	}
	return false
}

func generateSearchIndex(suiteRes *SuiteResult, reportsDir string) error {
	env.CreateDirectory(filepath.Join(reportsDir, "js"))
	f, err := os.Create(filepath.Join(reportsDir, "js", "search_index.js"))
	if err != nil {
		return err
	}
	defer f.Close()
	index := newSearchIndex()
	for _, r := range suiteRes.SpecResults {
		specFileName := toHTMLFileName(r.FileName, projectRoot)
		for _, t := range r.Tags {
			if !index.hasValueForTag(t, specFileName) {
				index.Tags[t] = append(index.Tags[t], specFileName)
			}
		}
		for _, s := range r.Scenarios {
			for _, t := range s.Tags {
				if !index.hasValueForTag(t, specFileName) {
					index.Tags[t] = append(index.Tags[t], specFileName)
				}
			}
		}
		specHeading := r.SpecHeading
		if !index.hasSpec(specHeading, specFileName) {
			index.Specs[specHeading] = append(index.Specs[specHeading], specFileName)
		}
	}
	s, err := json.Marshal(index)
	if err != nil {
		return err
	}
	f.WriteString(fmt.Sprintf("var index = %s;", s))
	return nil
}

func generateIndexPage(suiteRes *SuiteResult, w io.Writer, wg *sync.WaitGroup) {
	defer wg.Done()
	execTemplate("indexPage", w, suiteRes)
}

func generateIndexPages(suiteRes *SuiteResult, reportsDir string, wg *sync.WaitGroup) {
	dirs := make(map[string]int)
	for _, s := range suiteRes.SpecResults {
		p, err := filepath.Rel(projectRoot, filepath.Dir(s.FileName))
		if err != nil {
			log.Fatal(err)
		}
		childDirs := filepath.SplitList(p)
		basePath := ""
		for _, d := range childDirs {
			if _, ok := dirs[d]; !ok {
				dirs[d] = 1
			}
			basePath = filepath.Join(basePath, d)
		}

	}
	delete(dirs, ".")
	for d := range dirs {
		dirPath := filepath.Join(reportsDir, d)
		os.MkdirAll(dirPath, common.NewDirectoryPermissions)
		p := filepath.Join(dirPath, "index.html")
		f, err := os.Create(p)
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()
		wg.Add(1)
		res := toNestedSuiteResult(d, suiteRes)
		generateIndexPage(res, f, wg)
	}
}

func generateSpecPage(suiteRes *SuiteResult, specRes *spec, w io.Writer, wg *sync.WaitGroup) {
	defer wg.Done()
	execTemplate("specPage", w, struct {
		SuiteRes *SuiteResult
		SpecRes  *spec
	}{suiteRes, specRes})
}
<|endoftext|>


<commit_before>package geoloader

import (
	"context"
	"log"
	"regexp"
	"sort"
	"sync"
	"time"

	"cloud.google.com/go/storage"
	"google.golang.org/api/iterator"

	"github.com/m-lab/annotation-service/api"
)

// GeoLite2StartDate is the date we have the first GeoLite2 dataset.
// Any request earlier than this date using legacy binary datasets
// later than this date using GeoLite2 datasets
// TODO make this local
var GeoLite2StartDate = time.Unix(1502755200, 0) //"August 15, 2017"

// earliestArchiveDate is the date of the earliest archived dataset.
var earliestArchiveDate = time.Unix(1377648000, 0) // "August 28, 2013")

// datasetDirV4 and datasetDirV6 stores info on all the available datasets.  It is initially empty, just to
// provide the LatestDate() function.
// The current directory is regarded as immutable, but the pointer is dynamically updated, so accesses
// should only be done through getDirectory() and setDirectory().
var datasetDirV4 = &directory{}
var datasetDirV6 = &directory{}
var datasetDirLock sync.RWMutex // lock to be held when accessing or updating datasetDir pointer.

var datasetFilenames []string
var datasetFilenamesLock sync.RWMutex // lock to be held when accessing or updating datasetFilenames.

func getDatasetFilenames() []string {
	datasetFilenamesLock.RLock()
	defer datasetFilenamesLock.RUnlock()
	return datasetFilenames
}

func setDatasetFilenames(filename []string) {
	datasetFilenamesLock.Lock()
	defer datasetFilenamesLock.Unlock()
	datasetFilenames = filename
}

func getDirectoryV4() *directory {
	datasetDirLock.RLock()
	defer datasetDirLock.RUnlock()
	return datasetDirV4
}

func getDirectoryV6() *directory {
	datasetDirLock.RLock()
	defer datasetDirLock.RUnlock()
	return datasetDirV6
}

func setDirectory(dirv4 *directory, dirv6 *directory) {
	datasetDirLock.Lock()
	defer datasetDirLock.Unlock()
	datasetDirV4 = dirv4
	datasetDirV6 = dirv6
}

type dateEntry struct {
	date     time.Time
	filename string
}

// directory maintains a list of datasets.
type directory struct {
	entries map[string]*dateEntry // Map to filename associated with date.
	dates   []string              // Date strings associated with files.
}

func newDirectory(size int) directory {
	return directory{entries: make(map[string]*dateEntry, size), dates: make([]string, 0, size)}
}

// Insert inserts a new filename into the directory at the given date.
// NOTE: This does not detect or eliminate duplicates.
// TODO - make this local.
func (dir *directory) Insert(date time.Time, fn string) {
	dateString := date.Format("20060102")
	entry, ok := dir.entries[dateString]
	if !ok {
		// Insert the new date into the date slice.
		index := sort.SearchStrings(dir.dates, dateString)
		dir.dates = append(dir.dates, "")
		copy(dir.dates[index+1:], dir.dates[index:])
		dir.dates[index] = dateString

		// Create new entry for the date.
		entry = &dateEntry{filename: fn, date: date}
		dir.entries[dateString] = entry
	}
}

func (dir *directory) latestDate() time.Time {
	if len(dir.dates) < 1 {
		return time.Time{}
	}
	d := dir.dates[len(dir.dates)-1]
	return dir.entries[d].date
}

// LastFilenameEarlierThan returns the filename associated with the provided date.
// Except for dates prior to 2013, it will return the latest filename with date prior
// to the provided date.
// Returns empty string if the directory is empty.
func (dir *directory) LastFilenameEarlierThan(date time.Time) string {
	if len(dir.dates) == 0 {
		return ""
	}

	dateString := date.Format("20060102")
	index := sort.SearchStrings(dir.dates, dateString)
	if index == 0 {
		return dir.entries[dir.dates[index]].filename
	}
	return dir.entries[dir.dates[index-1]].filename
}

// TODO: These regex are duplicated in geolite2 and legacy packages.
// This is the regex used to filter for which files we want to consider acceptable for using with Geolite2
var GeoLite2Regex = regexp.MustCompile(`Maxmind/\d{4}/\d{2}/\d{2}/\d{8}T\d{6}Z-GeoLite2-City-CSV\.zip`)

// This is the regex used to filter for which files we want to consider acceptable for using with legacy dataset
var GeoLegacyRegex = regexp.MustCompile(`.*-GeoLiteCity.dat.*`)
var GeoLegacyv6Regex = regexp.MustCompile(`.*-GeoLiteCityv6.dat.*`)

// UpdateArchivedFilenames extracts the dataset filenames from downloader bucket
// This job is run at the beginning of deployment and daily cron job.
func UpdateArchivedFilenames() error {
	sizev4 := len(getDirectoryV4().dates) + 2
	sizev6 := len(getDirectoryV6().dates) + 2
	dirV4 := directory{entries: make(map[string]*dateEntry, sizev4), dates: make([]string, 0, sizev4)}
	dirV6 := directory{entries: make(map[string]*dateEntry, sizev6), dates: make([]string, 0, sizev6)}

	ctx := context.Background()
	client, err := storage.NewClient(ctx)
	if err != nil {
		return err
	}
	filenames := []string{}
	prospectiveFiles := client.Bucket(api.MaxmindBucketName).Objects(ctx, &storage.Query{Prefix: api.MaxmindPrefix})
	for file, err := prospectiveFiles.Next(); err != iterator.Done; file, err = prospectiveFiles.Next() {
		if err != nil {
			return err
		}
		if !GeoLite2Regex.MatchString(file.Name) && !GeoLegacyRegex.MatchString(file.Name) && !GeoLegacyv6Regex.MatchString(file.Name) {
			continue
		}
		// We archived but do not use legacy datasets after GeoLite2StartDate.
		fileDate, err := api.ExtractDateFromFilename(file.Name)
		if err != nil {
			continue
		}
		IPtype := 0
		if fileDate.Before(GeoLite2StartDate) {
			// Check whether this legacy dataset is IPv4 or IPv6
			if GeoLegacyRegex.MatchString(file.Name) {
				IPtype = 4
			} else if GeoLegacyv6Regex.MatchString(file.Name) {
				IPtype = 6
			} else {
				continue
			}
		}

		if !fileDate.Before(GeoLite2StartDate) && !GeoLite2Regex.MatchString(file.Name) {
			continue
		}

		// Build 2 dirs here. One for IPv4 and one for IPv6
		if IPtype == 4 {
			dirV4.Insert(fileDate, file.Name)
		} else if IPtype == 6 {
			dirV6.Insert(fileDate, file.Name)
		} else {
			dirV4.Insert(fileDate, file.Name)
			dirV6.Insert(fileDate, file.Name)
		}
		filenames = append(filenames, file.Name)
	}
	if err != nil {
		log.Println(err)
	}

	setDirectory(&dirV4, &dirV6)
	setDatasetFilenames(filenames)
	return nil
}

// Latest returns the date of the latest dataset.
// May return time.Time{} if no dates have been loaded.
func LatestDatasetDate() time.Time {
	dd := getDirectoryV4()
	return dd.latestDate()
}

// BestAnnotatorFilename return legacy IPv4 or IPv6 or Geolite2 filename based on request date and IP type
func BestAnnotatorFilename(request *api.RequestData) string {
	if request.IPFormat == 4 {
		dd := getDirectoryV4()
		return dd.LastFilenameEarlierThan(request.Timestamp)
	} else if request.IPFormat == 6 {
		dd := getDirectoryV6()
		return dd.LastFilenameEarlierThan(request.Timestamp)
	} else {
		return ""
	}
}
<commit_msg>Update filename.go<commit_after>package geoloader

import (
	"context"
	"log"
	"regexp"
	"sort"
	"sync"
	"time"

	"cloud.google.com/go/storage"
	"google.golang.org/api/iterator"

	"github.com/m-lab/annotation-service/api"
)

// GeoLite2StartDate is the date we have the first GeoLite2 dataset.
// Any request earlier than this date using legacy binary datasets
// later than this date using GeoLite2 datasets
// TODO make this local
var GeoLite2StartDate = time.Unix(1502755200, 0) //"August 15, 2017"

// earliestArchiveDate is the date of the earliest archived dataset.
var earliestArchiveDate = time.Unix(1377648000, 0) // "August 28, 2013")

// datasetDirV4 and datasetDirV6 stores info on all the available datasets.  It is initially empty, just to
// provide the LatestDate() function.
// The current directory is regarded as immutable, but the pointer is dynamically updated, so accesses
// should only be done through getDirectory() and setDirectory().
var datasetDirV4 = &directory{}
var datasetDirV6 = &directory{}
var datasetDirLock sync.RWMutex // lock to be held when accessing or updating datasetDir pointer.

var datasetFilenames []string
var datasetFilenamesLock sync.RWMutex // lock to be held when accessing or updating datasetFilenames.

func GetDatasetFilenames() []string {
	datasetFilenamesLock.RLock()
	defer datasetFilenamesLock.RUnlock()
	return datasetFilenames
}

func setDatasetFilenames(filename []string) {
	datasetFilenamesLock.Lock()
	defer datasetFilenamesLock.Unlock()
	datasetFilenames = filename
}

func getDirectoryV4() *directory {
	datasetDirLock.RLock()
	defer datasetDirLock.RUnlock()
	return datasetDirV4
}

func getDirectoryV6() *directory {
	datasetDirLock.RLock()
	defer datasetDirLock.RUnlock()
	return datasetDirV6
}

func setDirectory(dirv4 *directory, dirv6 *directory) {
	datasetDirLock.Lock()
	defer datasetDirLock.Unlock()
	datasetDirV4 = dirv4
	datasetDirV6 = dirv6
}

type dateEntry struct {
	date     time.Time
	filename string
}

// directory maintains a list of datasets.
type directory struct {
	entries map[string]*dateEntry // Map to filename associated with date.
	dates   []string              // Date strings associated with files.
}

func newDirectory(size int) directory {
	return directory{entries: make(map[string]*dateEntry, size), dates: make([]string, 0, size)}
}

// Insert inserts a new filename into the directory at the given date.
// NOTE: This does not detect or eliminate duplicates.
// TODO - make this local.
func (dir *directory) Insert(date time.Time, fn string) {
	dateString := date.Format("20060102")
	entry, ok := dir.entries[dateString]
	if !ok {
		// Insert the new date into the date slice.
		index := sort.SearchStrings(dir.dates, dateString)
		dir.dates = append(dir.dates, "")
		copy(dir.dates[index+1:], dir.dates[index:])
		dir.dates[index] = dateString

		// Create new entry for the date.
		entry = &dateEntry{filename: fn, date: date}
		dir.entries[dateString] = entry
	}
}

func (dir *directory) latestDate() time.Time {
	if len(dir.dates) < 1 {
		return time.Time{}
	}
	d := dir.dates[len(dir.dates)-1]
	return dir.entries[d].date
}

// LastFilenameEarlierThan returns the filename associated with the provided date.
// Except for dates prior to 2013, it will return the latest filename with date prior
// to the provided date.
// Returns empty string if the directory is empty.
func (dir *directory) LastFilenameEarlierThan(date time.Time) string {
	if len(dir.dates) == 0 {
		return ""
	}

	dateString := date.Format("20060102")
	index := sort.SearchStrings(dir.dates, dateString)
	if index == 0 {
		return dir.entries[dir.dates[index]].filename
	}
	return dir.entries[dir.dates[index-1]].filename
}

// TODO: These regex are duplicated in geolite2 and legacy packages.
// This is the regex used to filter for which files we want to consider acceptable for using with Geolite2
var GeoLite2Regex = regexp.MustCompile(`Maxmind/\d{4}/\d{2}/\d{2}/\d{8}T\d{6}Z-GeoLite2-City-CSV\.zip`)

// This is the regex used to filter for which files we want to consider acceptable for using with legacy dataset
var GeoLegacyRegex = regexp.MustCompile(`.*-GeoLiteCity.dat.*`)
var GeoLegacyv6Regex = regexp.MustCompile(`.*-GeoLiteCityv6.dat.*`)

// UpdateArchivedFilenames extracts the dataset filenames from downloader bucket
// This job is run at the beginning of deployment and daily cron job.
func UpdateArchivedFilenames() error {
	sizev4 := len(getDirectoryV4().dates) + 2
	sizev6 := len(getDirectoryV6().dates) + 2
	dirV4 := directory{entries: make(map[string]*dateEntry, sizev4), dates: make([]string, 0, sizev4)}
	dirV6 := directory{entries: make(map[string]*dateEntry, sizev6), dates: make([]string, 0, sizev6)}

	ctx := context.Background()
	client, err := storage.NewClient(ctx)
	if err != nil {
		return err
	}
	filenames := []string{}
	prospectiveFiles := client.Bucket(api.MaxmindBucketName).Objects(ctx, &storage.Query{Prefix: api.MaxmindPrefix})
	for file, err := prospectiveFiles.Next(); err != iterator.Done; file, err = prospectiveFiles.Next() {
		if err != nil {
			return err
		}
		if !GeoLite2Regex.MatchString(file.Name) && !GeoLegacyRegex.MatchString(file.Name) && !GeoLegacyv6Regex.MatchString(file.Name) {
			continue
		}
		// We archived but do not use legacy datasets after GeoLite2StartDate.
		fileDate, err := api.ExtractDateFromFilename(file.Name)
		if err != nil {
			continue
		}
		IPtype := 0
		if fileDate.Before(GeoLite2StartDate) {
			// Check whether this legacy dataset is IPv4 or IPv6
			if GeoLegacyRegex.MatchString(file.Name) {
				IPtype = 4
			} else if GeoLegacyv6Regex.MatchString(file.Name) {
				IPtype = 6
			} else {
				continue
			}
		}

		if !fileDate.Before(GeoLite2StartDate) && !GeoLite2Regex.MatchString(file.Name) {
			continue
		}

		// Build 2 dirs here. One for IPv4 and one for IPv6
		if IPtype == 4 {
			dirV4.Insert(fileDate, file.Name)
		} else if IPtype == 6 {
			dirV6.Insert(fileDate, file.Name)
		} else {
			dirV4.Insert(fileDate, file.Name)
			dirV6.Insert(fileDate, file.Name)
		}
		filenames = append(filenames, file.Name)
	}
	if err != nil {
		log.Println(err)
	}

	setDirectory(&dirV4, &dirV6)
	setDatasetFilenames(filenames)
	return nil
}

// Latest returns the date of the latest dataset.
// May return time.Time{} if no dates have been loaded.
func LatestDatasetDate() time.Time {
	dd := getDirectoryV4()
	return dd.latestDate()
}

// BestAnnotatorFilename return legacy IPv4 or IPv6 or Geolite2 filename based on request date and IP type
func BestAnnotatorFilename(request *api.RequestData) string {
	if request.IPFormat == 4 {
		dd := getDirectoryV4()
		return dd.LastFilenameEarlierThan(request.Timestamp)
	} else if request.IPFormat == 6 {
		dd := getDirectoryV6()
		return dd.LastFilenameEarlierThan(request.Timestamp)
	} else {
		return ""
	}
}
<|endoftext|>


<commit_before>// +build e2e

/*
Copyright 2019 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"os"
	"strings"
	"testing"

	"knative.dev/pkg/test/logstream"
)

var packages = []string{
	"github.com/google/knative-gcp/test/cmd/target",
	"github.com/google/knative-gcp/test/cmd/storage_target",
}

var packageToImageConfig = map[string]string{}
var packageToImageConfigDone bool

func TestMain(m *testing.M) {
	for _, pack := range packages {
		image, err := KoPublish(pack)
		if err != nil {
			fmt.Printf("error attempting to ko publish: %s\n", err)
			panic(err)
		}
		i := strings.Split(pack, "/")
		packageToImageConfig[i[len(i)-1]+"Image"] = image
	}
	packageToImageConfigDone = true

	os.Exit(m.Run())
}

// This test is more for debugging the ko publish process.
func TestKoPublish(t *testing.T) {
	for k, v := range packageToImageConfig {
		t.Log(k, "-->", v)
	}
}

// Rest of e2e tests go below:

// TestSmoke makes sure we can run tests.
func TestSmokeChannel(t *testing.T) {
	cancel := logstream.Start(t)
	defer cancel()
	SmokeTestChannelImpl(t)
}

// TestSmokePullSubscription makes sure we can run tests on PullSubscriptions.
func TestSmokePullSubscription(t *testing.T) {
	cancel := logstream.Start(t)
	defer cancel()
	SmokePullSubscriptionTestImpl(t)
}

// TestPullSubscriptionWithTarget tests we can knock down a target.
func TestPullSubscriptionWithTarget(t *testing.T) {
	cancel := logstream.Start(t)
	defer cancel()

	PullSubscriptionWithTargetTestImpl(t, packageToImageConfig)
}

// TestStorage tests we can knock down a target fot storage
func TestStorage(t *testing.T) {
	cancel := logstream.Start(t)
	defer cancel()
	StorageWithTestImpl(t, packageToImageConfig)
}
<commit_msg>remove skip<commit_after>// +build e2e

/*
Copyright 2019 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"os"
	"strings"
	"testing"

	"knative.dev/pkg/test/logstream"
)

var packages = []string{
	"github.com/google/knative-gcp/test/cmd/target",
	"github.com/google/knative-gcp/test/cmd/storage_target",
}

var packageToImageConfig = map[string]string{}
var packageToImageConfigDone bool

func TestMain(m *testing.M) {
	for _, pack := range packages {
		image, err := KoPublish(pack)
		if err != nil {
			fmt.Printf("error attempting to ko publish: %s\n", err)
			panic(err)
		}
		i := strings.Split(pack, "/")
		packageToImageConfig[i[len(i)-1]+"Image"] = image
	}
	packageToImageConfigDone = true

	os.Exit(m.Run())
}

// This test is more for debugging the ko publish process.
func TestKoPublish(t *testing.T) {
	for k, v := range packageToImageConfig {
		t.Log(k, "-->", v)
	}
}

// Rest of e2e tests go below:

// TestSmoke makes sure we can run tests.
func TestSmokeChannel(t *testing.T) {
	cancel := logstream.Start(t)
	defer cancel()
	SmokeTestChannelImpl(t)
}

// TestSmokePullSubscription makes sure we can run tests on PullSubscriptions.
func TestSmokePullSubscription(t *testing.T) {
	cancel := logstream.Start(t)
	defer cancel()
	SmokePullSubscriptionTestImpl(t)
}

// TestPullSubscriptionWithTarget tests we can knock down a target.
func TestPullSubscriptionWithTarget(t *testing.T) {
	cancel := logstream.Start(t)
	defer cancel()
	PullSubscriptionWithTargetTestImpl(t, packageToImageConfig)
}

// TestStorage tests we can knock down a target fot storage
func TestStorage(t *testing.T) {
	cancel := logstream.Start(t)
	defer cancel()
	StorageWithTestImpl(t, packageToImageConfig)
}
<|endoftext|>


<commit_before>package e2e

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"testing"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/coreos/kube-etcd-controller/pkg/cluster"
	"github.com/coreos/kube-etcd-controller/test/e2e/framework"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/labels"
	"k8s.io/kubernetes/pkg/util/wait"
)

func TestCreateCluster(t *testing.T) {
	f := framework.Global
	name := "my-etcd"
	myetcd := makeEtcdCluster(name, 3)

	if err := createEtcdCluster(f, myetcd); err != nil {
		t.Fatal(err)
	}

	defer func() {
		if err := deleteEtcdCluster(f, name); err != nil {
			t.Fatal(err)
		}
	}()

	if err := waitUntilSizeReached(f, name, 3); err != nil {
		t.Errorf("failed to create 3 members etcd cluster: %v", err)
	}
}

func TestResizeCluster3to5(t *testing.T) {
	f := framework.Global
	name := "my-etcd"
	myetcd := makeEtcdCluster(name, 3)

	if err := createEtcdCluster(f, myetcd); err != nil {
		t.Fatal(err)
	}

	defer func() {
		if err := deleteEtcdCluster(f, name); err != nil {
			t.Fatal(err)
		}
	}()

	if err := waitUntilSizeReached(f, name, 3); err != nil {
		t.Errorf("failed to create 3 members etcd cluster: %v", err)
	}
	t.Log("reached to size 3")

	myetcd.Spec.Size = 5
	if err := updateEtcdCluster(f, name, myetcd); err != nil {
		t.Fatal(err)
	}

	if err := waitUntilSizeReached(f, name, 5); err != nil {
		t.Errorf("failed to create 5 members etcd cluster: %v", err)
	}
}

func waitUntilSizeReached(f *framework.Framework, name string, size int) error {
	return wait.Poll(5*time.Second, 1*time.Minute, func() (done bool, err error) {
		pods, err := f.KubeClient.Pods(f.Namespace.Name).List(api.ListOptions{
			LabelSelector: labels.SelectorFromSet(map[string]string{
				"etcd_cluster": name,
			}),
		})
		if err != nil {
			return false, err
		}
		logrus.Infof("Currently running pods: %v", getPodNames(pods.Items))
		if len(pods.Items) != size {
			// TODO: check etcd commands.
			return false, nil
		}
		return true, nil
	})
}

func getPodNames(pods []api.Pod) []string {
	res := []string{}
	for _, p := range pods {
		res = append(res, p.Name)
	}
	return res
}

func makeEtcdCluster(name string, size int) *cluster.EtcdCluster {
	return &cluster.EtcdCluster{
		TypeMeta: unversioned.TypeMeta{
			Kind:       "EtcdCluster",
			APIVersion: "coreos.com/v1",
		},
		ObjectMeta: api.ObjectMeta{
			Name: name,
		},
		Spec: cluster.Spec{
			Size: size,
		},
	}
}

func createEtcdCluster(f *framework.Framework, e *cluster.EtcdCluster) error {
	b, err := json.Marshal(e)
	if err != nil {
		return err
	}
	resp, err := f.KubeClient.Client.Post(
		fmt.Sprintf("%s/apis/coreos.com/v1/namespaces/%s/etcdclusters", f.MasterHost, f.Namespace.Name),
		"application/json", bytes.NewReader(b))
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusCreated {
		return fmt.Errorf("unexpected status: %v", resp.Status)
	}
	return nil
}

func updateEtcdCluster(f *framework.Framework, name string, e *cluster.EtcdCluster) error {
	b, err := json.Marshal(e)
	if err != nil {
		return err
	}
	req, err := http.NewRequest("PUT",
		fmt.Sprintf("%s/apis/coreos.com/v1/namespaces/%s/etcdclusters/%s", f.MasterHost, f.Namespace.Name, name),
		bytes.NewReader(b))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	resp, err := f.KubeClient.Client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("unexpected status: %v", resp.Status)
	}
	return nil
}

func deleteEtcdCluster(f *framework.Framework, name string) error {
	req, err := http.NewRequest("DELETE",
		fmt.Sprintf("%s/apis/coreos.com/v1/namespaces/%s/etcdclusters/%s", f.MasterHost, f.Namespace.Name, name), nil)
	if err != nil {
		return err
	}
	resp, err := f.KubeClient.Client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("unexpected status: %v", resp.Status)
	}
	return nil
}
<commit_msg>e2e: add test TestResizeCluster5to3<commit_after>package e2e

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"testing"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/coreos/kube-etcd-controller/pkg/cluster"
	"github.com/coreos/kube-etcd-controller/test/e2e/framework"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/labels"
	"k8s.io/kubernetes/pkg/util/wait"
)

func TestCreateCluster(t *testing.T) {
	f := framework.Global
	testEtcd, err := createEtcdCluster(f, makeEtcdCluster("test-etcd-", 3))
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		if err := deleteEtcdCluster(f, testEtcd.Name); err != nil {
			t.Fatal(err)
		}
	}()

	if err := waitUntilSizeReached(f, testEtcd.Name, 3); err != nil {
		t.Errorf("failed to create 3 members etcd cluster: %v", err)
	}
}

func TestResizeCluster3to5(t *testing.T) {
	f := framework.Global
	testEtcd, err := createEtcdCluster(f, makeEtcdCluster("test-etcd-", 3))
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		if err := deleteEtcdCluster(f, testEtcd.Name); err != nil {
			t.Fatal(err)
		}
	}()

	if err := waitUntilSizeReached(f, testEtcd.Name, 3); err != nil {
		t.Errorf("failed to create 3 members etcd cluster: %v", err)
	}
	t.Log("reached to 3 members cluster")

	testEtcd.Spec.Size = 5
	if err := updateEtcdCluster(f, testEtcd); err != nil {
		t.Fatal(err)
	}

	if err := waitUntilSizeReached(f, testEtcd.Name, 5); err != nil {
		t.Errorf("failed to resize to 5 members etcd cluster: %v", err)
	}
}

func TestResizeCluster5to3(t *testing.T) {
	f := framework.Global
	testEtcd, err := createEtcdCluster(f, makeEtcdCluster("test-etcd-", 5))
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		if err := deleteEtcdCluster(f, testEtcd.Name); err != nil {
			t.Fatal(err)
		}
	}()

	if err := waitUntilSizeReached(f, testEtcd.Name, 5); err != nil {
		t.Errorf("failed to create 5 members etcd cluster: %v", err)
	}
	t.Log("reached to 5 members cluster")

	testEtcd.Spec.Size = 3
	if err := updateEtcdCluster(f, testEtcd); err != nil {
		t.Fatal(err)
	}

	if err := waitUntilSizeReached(f, testEtcd.Name, 3); err != nil {
		t.Errorf("failed to resize to 3 members etcd cluster: %v", err)
	}
}

func waitUntilSizeReached(f *framework.Framework, name string, size int) error {
	return wait.Poll(5*time.Second, 1*time.Minute, func() (done bool, err error) {
		pods, err := f.KubeClient.Pods(f.Namespace.Name).List(api.ListOptions{
			LabelSelector: labels.SelectorFromSet(map[string]string{
				"etcd_cluster": name,
			}),
		})
		if err != nil {
			return false, err
		}
		logrus.Infof("Currently running pods: %v", getPodNames(pods.Items))
		if len(pods.Items) != size {
			// TODO: check etcd commands.
			return false, nil
		}
		return true, nil
	})
}

func getPodNames(pods []api.Pod) []string {
	res := []string{}
	for _, p := range pods {
		res = append(res, p.Name)
	}
	return res
}

func makeEtcdCluster(genName string, size int) *cluster.EtcdCluster {
	return &cluster.EtcdCluster{
		TypeMeta: unversioned.TypeMeta{
			Kind:       "EtcdCluster",
			APIVersion: "coreos.com/v1",
		},
		ObjectMeta: api.ObjectMeta{
			GenerateName: genName,
		},
		Spec: cluster.Spec{
			Size: size,
		},
	}
}

func createEtcdCluster(f *framework.Framework, e *cluster.EtcdCluster) (*cluster.EtcdCluster, error) {
	b, err := json.Marshal(e)
	if err != nil {
		return nil, err
	}
	resp, err := f.KubeClient.Client.Post(
		fmt.Sprintf("%s/apis/coreos.com/v1/namespaces/%s/etcdclusters", f.MasterHost, f.Namespace.Name),
		"application/json", bytes.NewReader(b))
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusCreated {
		return nil, fmt.Errorf("unexpected status: %v", resp.Status)
	}
	decoder := json.NewDecoder(resp.Body)
	res := &cluster.EtcdCluster{}
	if err := decoder.Decode(res); err != nil {
		return nil, err
	}
	return res, nil
}

func updateEtcdCluster(f *framework.Framework, e *cluster.EtcdCluster) error {
	b, err := json.Marshal(e)
	if err != nil {
		return err
	}
	req, err := http.NewRequest("PUT",
		fmt.Sprintf("%s/apis/coreos.com/v1/namespaces/%s/etcdclusters/%s", f.MasterHost, f.Namespace.Name, e.Name),
		bytes.NewReader(b))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	resp, err := f.KubeClient.Client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("unexpected status: %v", resp.Status)
	}
	return nil
}

func deleteEtcdCluster(f *framework.Framework, name string) error {
	req, err := http.NewRequest("DELETE",
		fmt.Sprintf("%s/apis/coreos.com/v1/namespaces/%s/etcdclusters/%s", f.MasterHost, f.Namespace.Name, name), nil)
	if err != nil {
		return err
	}
	resp, err := f.KubeClient.Client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("unexpected status: %v", resp.Status)
	}
	return nil
}
<|endoftext|>


<commit_before>package error

var (
	// OK
	ERR_CODE_OK = 0
	ERR_STR_OK  = "ok"

	// SYSTEM ERROR CODE
	ERR_CODE_SYS = 1000
	ERR_STR_SYS  = "system error"
	// TODO: ADD OTHER NEW SYSTEM ERROR, see auth error as example
	ERR_CODE_IO = 1001
	ERR_STR_IO  = "i/o error"

	// DATABASE ERROR CODE
	ERR_CODE_DB = 1100
	ERR_STR_DB  = "db error"
	// TODO: ADD OTHER NEW DATABASE ERROR, see auth error as example

	// NETWORK ERROR CODE
	ERR_CODE_NETWORK = 1200
	ERR_STR_NETWORK  = "network error"
	// TODO: ADD OTHER NEW NETWORK ERROR, see auth error as example

	// PARAMETER ERROR CODE
	ERR_CODE_PARA = 1300
	ERR_STR_PARA  = "parameter error"
	// TODO: ADD OTHER NEW PARAMETER ERROR,

	// AUTH ERROR CODE
	ERR_CODE_AUTH = 1400
	ERR_STR_AUTH  = "auth failed"

	// TODO: ADD OTHER NEW AUTH ERROR, AFTER ERR_CODE_AUTH_EXPIRED AS BELOW
	ERR_CODE_AUTH_EXPIRED = 1401
	ERR_STR_AUTH_EXPIRED  = "auth expired"

	ERR_CODE_CONVERT      = 2000
	ERR_STR_CONVERT       = "convert error"
	ERR_CODE_CONVERT_NIL  = 2001
	ERR_STR_CONVERT_NIL   = "invalid value"
	ERR_CODE_CONVERT_TYPE = 2000
	ERR_STR_CONVERT_TYPE  = "type mismatch"

	// UNKNOWN ERROR
	// ERR_CODE_UNKNOWN = 1900
	ERR_STR_UNKNOWN = "unknown error"
	// TODO: ADD OTHER NEW UNKNOW ERROR, see auth error as example

	errMap = make(map[int]string)
)

func init() {
	// ok
	errMap[ERR_CODE_OK] = ERR_STR_OK

	// system
	errMap[ERR_CODE_SYS] = ERR_STR_SYS
	errMap[ERR_CODE_IO] = ERR_STR_IO

	// database
	errMap[ERR_CODE_DB] = ERR_STR_DB

	// network
	errMap[ERR_CODE_NETWORK] = ERR_STR_NETWORK

	// parameter
	errMap[ERR_CODE_PARA] = ERR_STR_PARA

	// authentication
	errMap[ERR_CODE_AUTH] = ERR_STR_AUTH
	errMap[ERR_CODE_AUTH_EXPIRED] = ERR_STR_AUTH_EXPIRED
	errMap[ERR_CODE_CONVERT] = ERR_STR_CONVERT
	errMap[ERR_CODE_CONVERT_NIL] = ERR_STR_CONVERT_NIL
	errMap[ERR_CODE_CONVERT_TYPE] = ERR_STR_CONVERT_TYPE
}

func fetchErrString(code int) string {
	v, ok := errMap[code]
	if !ok {
		return ERR_STR_UNKNOWN
	}

	return v
}
<commit_msg>添加错误码<commit_after>package error

var (
	// OK
	ERR_CODE_OK = 0
	ERR_STR_OK  = "ok"

	// SYSTEM ERROR CODE
	ERR_CODE_SYS = 1000
	ERR_STR_SYS  = "system error"
	// TODO: ADD OTHER NEW SYSTEM ERROR, see auth error as example
	ERR_CODE_IO = 1001
	ERR_STR_IO  = "i/o error"

	// DATABASE ERROR CODE
	ERR_CODE_DB = 1100
	ERR_STR_DB  = "db error"
	// TODO: ADD OTHER NEW DATABASE ERROR, see auth error as example

	// NETWORK ERROR CODE
	ERR_CODE_NETWORK = 1200
	ERR_STR_NETWORK  = "network error"
	// TODO: ADD OTHER NEW NETWORK ERROR, see auth error as example

	// PARAMETER ERROR CODE
	ERR_CODE_PARA = 1300
	ERR_STR_PARA  = "parameter error"
	// TODO: ADD OTHER NEW PARAMETER ERROR,

	// AUTH ERROR CODE
	ERR_CODE_AUTH = 1400
	ERR_STR_AUTH  = "auth failed"

	// TODO: ADD OTHER NEW AUTH ERROR, AFTER ERR_CODE_AUTH_EXPIRED AS BELOW
	ERR_CODE_AUTH_EXPIRED = 1401
	ERR_STR_AUTH_EXPIRED  = "auth expired"

	ERR_CODE_CONVERT      = 2000
	ERR_STR_CONVERT       = "convert error"
	ERR_CODE_CONVERT_NIL  = 2001
	ERR_STR_CONVERT_NIL   = "invalid value"
	ERR_CODE_CONVERT_TYPE = 2000
	ERR_STR_CONVERT_TYPE  = "type mismatch"

	ERR_CODE_APPID_EXIST = 3000
	ERR_STR_APPID_EXIST  = "appid exist"

	ERR_CODE_APPID_NOT_EXIST = 3001
	ERR_STR_APPID_NOT_EXIST  = "appid not exist"

	// UNKNOWN ERROR
	// ERR_CODE_UNKNOWN = 1900
	ERR_STR_UNKNOWN = "unknown error"
	// TODO: ADD OTHER NEW UNKNOW ERROR, see auth error as example

	errMap = make(map[int]string)
)

func init() {
	// ok
	errMap[ERR_CODE_OK] = ERR_STR_OK

	// system
	errMap[ERR_CODE_SYS] = ERR_STR_SYS
	errMap[ERR_CODE_IO] = ERR_STR_IO

	// database
	errMap[ERR_CODE_DB] = ERR_STR_DB

	// network
	errMap[ERR_CODE_NETWORK] = ERR_STR_NETWORK

	// parameter
	errMap[ERR_CODE_PARA] = ERR_STR_PARA

	// authentication
	errMap[ERR_CODE_AUTH] = ERR_STR_AUTH
	errMap[ERR_CODE_AUTH_EXPIRED] = ERR_STR_AUTH_EXPIRED
	errMap[ERR_CODE_CONVERT] = ERR_STR_CONVERT
	errMap[ERR_CODE_CONVERT_NIL] = ERR_STR_CONVERT_NIL
	errMap[ERR_CODE_CONVERT_TYPE] = ERR_STR_CONVERT_TYPE

	errMap[ERR_CODE_APPID_EXIST] = ERR_STR_APPID_EXIST
	errMap[ERR_CODE_APPID_NOT_EXIST] = ERR_STR_APPID_NOT_EXIST
}

func fetchErrString(code int) string {
	v, ok := errMap[code]
	if !ok {
		return ERR_STR_UNKNOWN
	}

	return v
}
<|endoftext|>


<commit_before>// +build ignore

package main

import (
	"code.google.com/p/plotinum/plot"
	"code.google.com/p/plotinum/plotter"
	"code.google.com/p/plotinum/plotutil"
	"math/rand"
)

func main() {
	// Get some random data.
	n, m := 5, 10
	pts := make([]plotter.XYer, n)
	for i := range pts {
		xys := make(plotter.XYs, m)
		pts[i] = xys
		center := float64(i)
		for j := range xys {
			xys[j].X = center + (rand.Float64() - 0.5)
			xys[j].Y = center + (rand.Float64() - 0.5)
		}
	}

	plt, err := plot.New()
	if err != nil {
		panic(err)
	}

	mean95, err := plotutil.NewErrorPoints(plotutil.MeanAndConf95, pts...)
	if err != nil {
		panic(err)
	}
	medMinMax, err := plotutil.NewErrorPoints(plotutil.MedianAndMinMax, pts...)
	if err != nil {
		panic(err)
	}
	plotutil.AddLinePoints(plt,
		"mean and 95% confidence", mean95,
		"median and minimum and maximum", medMinMax)
	if err := plotutil.AddErrorBars(plt, mean95, medMinMax); err != nil {
		panic(err)
	}
	if err := plotutil.AddScatters(plt, pts[0], pts[1], pts[2], pts[3], pts[4]); err != nil {
		panic(err)
	}

	plt.Save(4, 4, "errpoints.png")
}
<commit_msg>Prepare plotutil/main.go to create multiple example plots<commit_after>// +build ignore

package main

import (
	"code.google.com/p/plotinum/plot"
	"code.google.com/p/plotinum/plotter"
	"code.google.com/p/plotinum/plotutil"
	"math/rand"
)

var examples = []struct {
	name   string
	mkplot func() *plot.Plot
}{
	{"example_errpoints", Example_errpoints},
}

func main() {
	for _, ex := range examples {
		drawEps(ex.name, ex.mkplot)
		drawSvg(ex.name, ex.mkplot)
		drawPng(ex.name, ex.mkplot)
		drawTiff(ex.name, ex.mkplot)
		drawJpg(ex.name, ex.mkplot)
		drawPdf(ex.name, ex.mkplot)
	}
}

func drawEps(name string, mkplot func() *plot.Plot) {
	if err := mkplot().Save(4, 4, name+".eps"); err != nil {
		panic(err)
	}
}

func drawPdf(name string, mkplot func() *plot.Plot) {
	if err := mkplot().Save(4, 4, name+".pdf"); err != nil {
		panic(err)
	}
}

func drawSvg(name string, mkplot func() *plot.Plot) {
	if err := mkplot().Save(4, 4, name+".svg"); err != nil {
		panic(err)
	}
}

func drawPng(name string, mkplot func() *plot.Plot) {
	if err := mkplot().Save(4, 4, name+".png"); err != nil {
		panic(err)
	}
}

func drawTiff(name string, mkplot func() *plot.Plot) {
	if err := mkplot().Save(4, 4, name+".tiff"); err != nil {
		panic(err)
	}
}

func drawJpg(name string, mkplot func() *plot.Plot) {
	if err := mkplot().Save(4, 4, name+".jpg"); err != nil {
		panic(err)
	}
}

// Example_errpoints draws some error points.
func Example_errpoints() *plot.Plot {
	// Get some random data.
	n, m := 5, 10
	pts := make([]plotter.XYer, n)
	for i := range pts {
		xys := make(plotter.XYs, m)
		pts[i] = xys
		center := float64(i)
		for j := range xys {
			xys[j].X = center + (rand.Float64() - 0.5)
			xys[j].Y = center + (rand.Float64() - 0.5)
		}
	}

	plt, err := plot.New()
	if err != nil {
		panic(err)
	}

	mean95, err := plotutil.NewErrorPoints(plotutil.MeanAndConf95, pts...)
	if err != nil {
		panic(err)
	}
	medMinMax, err := plotutil.NewErrorPoints(plotutil.MedianAndMinMax, pts...)
	if err != nil {
		panic(err)
	}
	plotutil.AddLinePoints(plt,
		"mean and 95% confidence", mean95,
		"median and minimum and maximum", medMinMax)
	if err := plotutil.AddErrorBars(plt, mean95, medMinMax); err != nil {
		panic(err)
	}
	if err := plotutil.AddScatters(plt, pts[0], pts[1], pts[2], pts[3], pts[4]); err != nil {
		panic(err)
	}

	return plt
}
<|endoftext|>


<commit_before>//  Copyright (c) 2019 Cisco and/or its affiliates.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at:
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package vpp

import (
	"testing"

	"github.com/ligato/cn-infra/logging/logrus"

	vpp_interfaces "github.com/ligato/vpp-agent/api/models/vpp/interfaces"
	_ "github.com/ligato/vpp-agent/plugins/vpp/ifplugin"
	ifplugin_vppcalls "github.com/ligato/vpp-agent/plugins/vpp/ifplugin/vppcalls"
)

func TestInterfaceDumpState(t *testing.T) {
	ctx := setupVPP(t)
	defer ctx.teardownVPP()

	h := ifplugin_vppcalls.CompatibleInterfaceVppHandler(ctx.vppBinapi, logrus.NewLogger("test"))

	ifIdx0, err := h.AddLoopbackInterface("loop0")
	if err != nil {
		t.Fatalf("creating loopback interface failed: %v", err)
	}
	t.Logf("loop0 index: %+v", ifIdx0)

	ifIdx, err := h.AddLoopbackInterface("loop1")
	if err != nil {
		t.Fatalf("creating loopback interface failed: %v", err)
	}
	t.Logf("loop1 index: %+v", ifIdx)

	ifaces, err := h.DumpInterfaceStates()
	if err != nil {
		t.Fatalf("dumping interface states failed: %v", err)
	}
	if len(ifaces) != 3 {
		t.Errorf("expected 3 interface states in dump, got: %d", len(ifaces))
	}

	ifaces, err = h.DumpInterfaceStates(ifIdx)
	if err != nil {
		t.Fatalf("dumping interface states failed: %v", err)
	}
	iface := ifaces[ifIdx]
	t.Logf("interface state: %+v", iface)

	if iface == nil {
		t.Fatalf("expected interface, got: nil")
	}
	if iface.InternalName != "loop1" {
		t.Errorf("expected interface internal name to be loop1, got: %v", iface.InternalName)
	}
	if len(iface.PhysAddress) == 0 {
		t.Errorf("expected interface phys address to not be empty, got: %q", iface.PhysAddress)
	}
}

func TestLoopbackInterface(t *testing.T) {
	ctx := setupVPP(t)
	defer ctx.teardownVPP()

	h := ifplugin_vppcalls.CompatibleInterfaceVppHandler(ctx.vppBinapi, logrus.NewLogger("test"))

	ifIdx, err := h.AddLoopbackInterface("loop1")
	if err != nil {
		t.Fatalf("creating loopback interface failed: %v", err)
	}
	t.Logf("loopback index: %+v", ifIdx)

	ifaces, err := h.DumpInterfaces()
	if err != nil {
		t.Fatalf("dumping interfaces failed: %v", err)
	}
	iface, ok := ifaces[ifIdx]
	if !ok {
		t.Fatalf("loopback interface not found in dump")
	}
	t.Logf("interface: %+v", iface.Interface)

	if iface.Interface.Name != "loop1" {
		t.Errorf("expected interface name to be loop1, got: %v", iface.Interface.Name)
	}
	if iface.Interface.PhysAddress == "" {
		t.Errorf("expected interface phys address to not be empty, got: %v", iface.Interface.PhysAddress)
	}
	if iface.Interface.Enabled == true {
		t.Errorf("expected interface to not be enabled")
	}
	if iface.Interface.Type != vpp_interfaces.Interface_SOFTWARE_LOOPBACK {
		t.Errorf("expected interface type to be SOFTWARE_LOOPBACK, got: %v", iface.Interface.Type)
	}
	if iface.Interface.Link != nil {
		t.Errorf("expected interface link to be nil, got: %T", iface.Interface.Link)
	}
}

func TestMemifInterface(t *testing.T) {
	ctx := setupVPP(t)
	defer ctx.teardownVPP()

	h := ifplugin_vppcalls.CompatibleInterfaceVppHandler(ctx.vppBinapi, logrus.NewLogger("test"))

	ifIdx, err := h.AddMemifInterface("memif1", &vpp_interfaces.MemifLink{
		Id:     1,
		Mode:   vpp_interfaces.MemifLink_ETHERNET,
		Secret: "secret",
		Master: true,
	}, 0)
	if err != nil {
		t.Fatalf("creating memif interface failed: %v", err)
	}
	t.Logf("memif index: %+v", ifIdx)

	ifaces, err := h.DumpInterfaces()
	if err != nil {
		t.Fatalf("dumping interfaces failed: %v", err)
	}
	iface, ok := ifaces[ifIdx]
	if !ok {
		t.Fatalf("Memif interface not found in dump")
	}
	t.Logf("interface: %+v", iface.Interface)

	if iface.Interface.Name != "memif1" {
		t.Errorf("expected interface name to be memif1, got: %v", iface.Interface.Name)
	}
	if iface.Interface.Type != vpp_interfaces.Interface_MEMIF {
		t.Errorf("expected interface type to be memif, got: %v", iface.Interface.Type)
	}
	link, ok := iface.Interface.Link.(*vpp_interfaces.Interface_Memif)
	if !ok {
		t.Fatalf("expected interface link to be memif, got: %T", iface.Interface.Link)
	}
	if link.Memif.Id != 1 {
		t.Errorf("expected memif ID to be 1, got: %v", link.Memif.Id)
	}
	if link.Memif.Mode != vpp_interfaces.MemifLink_ETHERNET {
		t.Errorf("expected memif mode to be ETHERNET, got: %v", link.Memif.Mode)
	}
}
<commit_msg>Add integration test for IP address (#1421)<commit_after>//  Copyright (c) 2019 Cisco and/or its affiliates.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at:
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package vpp

import (
	"net"
	"testing"

	"github.com/ligato/cn-infra/logging/logrus"

	vpp_interfaces "github.com/ligato/vpp-agent/api/models/vpp/interfaces"
	_ "github.com/ligato/vpp-agent/plugins/vpp/ifplugin"
	ifplugin_vppcalls "github.com/ligato/vpp-agent/plugins/vpp/ifplugin/vppcalls"
)

func TestInterfaceIP(t *testing.T) {
	ctx := setupVPP(t)
	defer ctx.teardownVPP()

	h := ifplugin_vppcalls.CompatibleInterfaceVppHandler(ctx.vppBinapi, logrus.NewLogger("test"))

	tests := []struct {
		name  string
		ipnet net.IPNet
	}{
		{"basic ipv4", net.IPNet{IP: net.IPv4(10, 0, 0, 1), Mask: net.IPMask{255, 255, 255, 0}}},
		{"basic ipv6", net.IPNet{IP: net.ParseIP("::1"), Mask: net.IPMask{255, 255, 255, 0}}},
	}
	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			ifIdx, err := h.AddLoopbackInterface("loop0")
			if err != nil {
				t.Fatalf("creating loopback interface failed: %v", err)
			}
			t.Logf("loop0 index: %+v", ifIdx)

			if err := h.AddInterfaceIP(ifIdx, &test.ipnet); err != nil {
				t.Fatalf("adding interface IP failed: %v", err)
			}
		})
	}
}

func TestInterfaceDumpState(t *testing.T) {
	ctx := setupVPP(t)
	defer ctx.teardownVPP()

	h := ifplugin_vppcalls.CompatibleInterfaceVppHandler(ctx.vppBinapi, logrus.NewLogger("test"))

	ifIdx0, err := h.AddLoopbackInterface("loop0")
	if err != nil {
		t.Fatalf("creating loopback interface failed: %v", err)
	}
	t.Logf("loop0 index: %+v", ifIdx0)

	ifIdx, err := h.AddLoopbackInterface("loop1")
	if err != nil {
		t.Fatalf("creating loopback interface failed: %v", err)
	}
	t.Logf("loop1 index: %+v", ifIdx)

	ifaces, err := h.DumpInterfaceStates()
	if err != nil {
		t.Fatalf("dumping interface states failed: %v", err)
	}
	if len(ifaces) != 3 {
		t.Errorf("expected 3 interface states in dump, got: %d", len(ifaces))
	}

	ifaces, err = h.DumpInterfaceStates(ifIdx)
	if err != nil {
		t.Fatalf("dumping interface states failed: %v", err)
	}
	iface := ifaces[ifIdx]
	t.Logf("interface state: %+v", iface)

	if iface == nil {
		t.Fatalf("expected interface, got: nil")
	}
	if iface.InternalName != "loop1" {
		t.Errorf("expected interface internal name to be loop1, got: %v", iface.InternalName)
	}
	if len(iface.PhysAddress) == 0 {
		t.Errorf("expected interface phys address to not be empty, got: %q", iface.PhysAddress)
	}
}

func TestLoopbackInterface(t *testing.T) {
	ctx := setupVPP(t)
	defer ctx.teardownVPP()

	h := ifplugin_vppcalls.CompatibleInterfaceVppHandler(ctx.vppBinapi, logrus.NewLogger("test"))

	ifIdx, err := h.AddLoopbackInterface("loop1")
	if err != nil {
		t.Fatalf("creating loopback interface failed: %v", err)
	}
	t.Logf("loopback index: %+v", ifIdx)

	ifaces, err := h.DumpInterfaces()
	if err != nil {
		t.Fatalf("dumping interfaces failed: %v", err)
	}
	iface, ok := ifaces[ifIdx]
	if !ok {
		t.Fatalf("loopback interface not found in dump")
	}
	t.Logf("interface: %+v", iface.Interface)

	if iface.Interface.Name != "loop1" {
		t.Errorf("expected interface name to be loop1, got: %v", iface.Interface.Name)
	}
	if iface.Interface.PhysAddress == "" {
		t.Errorf("expected interface phys address to not be empty, got: %v", iface.Interface.PhysAddress)
	}
	if iface.Interface.Enabled == true {
		t.Errorf("expected interface to not be enabled")
	}
	if iface.Interface.Type != vpp_interfaces.Interface_SOFTWARE_LOOPBACK {
		t.Errorf("expected interface type to be SOFTWARE_LOOPBACK, got: %v", iface.Interface.Type)
	}
	if iface.Interface.Link != nil {
		t.Errorf("expected interface link to be nil, got: %T", iface.Interface.Link)
	}
}

func TestMemifInterface(t *testing.T) {
	ctx := setupVPP(t)
	defer ctx.teardownVPP()

	h := ifplugin_vppcalls.CompatibleInterfaceVppHandler(ctx.vppBinapi, logrus.NewLogger("test"))

	ifIdx, err := h.AddMemifInterface("memif1", &vpp_interfaces.MemifLink{
		Id:     1,
		Mode:   vpp_interfaces.MemifLink_ETHERNET,
		Secret: "secret",
		Master: true,
	}, 0)
	if err != nil {
		t.Fatalf("creating memif interface failed: %v", err)
	}
	t.Logf("memif index: %+v", ifIdx)

	ifaces, err := h.DumpInterfaces()
	if err != nil {
		t.Fatalf("dumping interfaces failed: %v", err)
	}
	iface, ok := ifaces[ifIdx]
	if !ok {
		t.Fatalf("Memif interface not found in dump")
	}
	t.Logf("interface: %+v", iface.Interface)

	if iface.Interface.Name != "memif1" {
		t.Errorf("expected interface name to be memif1, got: %v", iface.Interface.Name)
	}
	if iface.Interface.Type != vpp_interfaces.Interface_MEMIF {
		t.Errorf("expected interface type to be memif, got: %v", iface.Interface.Type)
	}
	link, ok := iface.Interface.Link.(*vpp_interfaces.Interface_Memif)
	if !ok {
		t.Fatalf("expected interface link to be memif, got: %T", iface.Interface.Link)
	}
	if link.Memif.Id != 1 {
		t.Errorf("expected memif ID to be 1, got: %v", link.Memif.Id)
	}
	if link.Memif.Mode != vpp_interfaces.MemifLink_ETHERNET {
		t.Errorf("expected memif mode to be ETHERNET, got: %v", link.Memif.Mode)
	}
}
<|endoftext|>


<commit_before>package fs

import (
	"errors"
	"flag"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"net/url"
	"os"
	pathutil "path/filepath"
	"time"

	"github.com/7sDream/rikka/common/util"
	"github.com/7sDream/rikka/plugins"
)

var filesDir = flag.String("dir", "files", "Where files will be save when use fs plugin.")
var tempDir string
var l = plugins.SubLogger("[FS]")

const (
	stateCopying     = "copying"
	stateCopyingCode = 1
	stateCopyingDesc = "Photo is being copied to rikka file system"
)

type fsPlugin struct{}

var FsPlugin fsPlugin = fsPlugin{}

func (fsp fsPlugin) Init() {
	l.Info("Args dir =", *filesDir)
	absFilesDir, err := pathutil.Abs(*filesDir)
	if err == nil {
		l.Info("Abs path of photo file dir:", absFilesDir)
		tempDir = absFilesDir
	} else {
		l.Fatal("A error happened when change photo dir to absolute path:", err)
	}

	if util.CheckExist(absFilesDir) {
		l.Info("Photo file dir already exist")
	} else {
		l.Info("Photo dir not eixst, try to create it")
		err = os.MkdirAll(absFilesDir, 0755)
		if err == nil {
			l.Info("Create dir", absFilesDir, "successfully")
		} else {
			l.Fatal("A error happened when try to create photo dir:", err)
		}
	}

	l.Info("Fs plugin start successfully")
}

func deleteFile(filepath string) {
	if util.CheckExist(filepath) {
		if err := os.Remove(filepath); err != nil {
			l.Fatal("A error happened when try to delete file", filepath, ":", err)
		}
	}
}

func buildCopyingState(taskID string) plugins.State {
	return plugins.State{
		TaskID:      taskID,
		StateCode:   stateCopyingCode,
		State:       stateCopying,
		Description: stateCopyingDesc,
	}
}

func saveFile(uploadFile multipart.File, saveTo *os.File, taskID string) {
	l.Info("Starting task", taskID)

	filepath := saveTo.Name()

	if err := plugins.CreateTask(buildCopyingState(taskID)); err != nil {
		l.Warn("A error happend when add task", taskID, ":", err)
		saveTo.Close()
		uploadFile.Close()
		deleteFile(filepath)
		return
	}

	_, err := io.Copy(saveTo, uploadFile)
	saveTo.Close()
	uploadFile.Close()

	if err == nil {
		// copy file successfully
		l.Info("File copy on task", taskID, "successfully")

		if err := plugins.DeleteTask(taskID); err == nil {
			l.Info("Task", taskID, "finished")
		} else {
			deleteFile(filepath)
			l.Fatal("A error happened when delete task", taskID, ":", err)
		}
	} else {
		// copy file failed
		deleteFile(filepath)
		l.Warn("A error happened when copy file", taskID, ":", err)

		if err := plugins.ChangeTaskState(plugins.BuildErrorState(taskID, err.Error())); err == nil {
			l.Info("Turn task", taskID, "state to error")
		} else {
			l.Fatal("A error happend when change task", taskID, "to error state:", err)
		}
	}
}

func (fsp fsPlugin) SaveRequestHandle(q *plugins.SaveRequest) (taskID string, err error) {
	l.Info("Recieved a file save request")
	now := time.Now().UTC()
	saveTo, err := ioutil.TempFile(tempDir, now.Format("2006-01-02-"))

	if err == nil {
		l.Info("Create file on fs successfully:", saveTo.Name())
	} else {
		l.Warn("Create file on fs error:", err)
		return "", err
	}

	_, taskID = pathutil.Split(saveTo.Name())

	go saveFile(q.File, saveTo, taskID)

	return taskID, nil
}

func (fsp fsPlugin) StateRequestHandle(taskID string) (pState *plugins.State, err error) {
	if pState, err = plugins.GetTaskState(taskID); err == nil {
		return pState, nil
	}
	if util.CheckExist(pathutil.Join(tempDir, taskID)) {
		finishState := plugins.BuildFinishState(taskID)
		return &finishState, nil
	}
	return nil, err
}

func buildURL(r *http.Request, taskID string) string {
	res := url.URL{
		Scheme: "http",
		Host:   r.Host,
		Path:   "files/" + taskID,
	}
	return res.String()
}

func (fsp fsPlugin) URLRequestHandle(q *plugins.URLRequest) (url *plugins.URLJSON, err error) {
	taskID := q.TaskID
	r := q.HTTPRequest
	if util.CheckExist(pathutil.Join(tempDir, taskID)) {
		url := buildURL(r, taskID)
		return &plugins.URLJSON{URL: url}, nil
	}
	return nil, errors.New("File not exist.")
}

func (fsp fsPlugin) ExtraHandlers() (handlers []plugins.HandlerWithPattern) {
	fileFs := http.StripPrefix(
		"/files",
		util.DisableListDir(
			http.FileServer(http.Dir(tempDir)),
		),
	)
	handlers = []plugins.HandlerWithPattern{
		plugins.HandlerWithPattern{
			Pattern: "/files/", Handler: fileFs,
		},
	}

	return handlers
}
<commit_msg>add doc for fs plugin<commit_after>package fs

import (
	"errors"
	"flag"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"net/url"
	"os"
	pathutil "path/filepath"
	"time"

	"github.com/7sDream/rikka/common/util"
	"github.com/7sDream/rikka/plugins"
)

var filesDir = flag.String("dir", "files", "Where files will be save when use fs plugin.")
var tempDir string
var l = plugins.SubLogger("[FS]")

const (
	stateCopying     = "copying"
	stateCopyingCode = 1
	stateCopyingDesc = "Photo is being copied to rikka file system"
)

type fsPlugin struct{}

// FsPlugin is the main plugin instance.
var FsPlugin = fsPlugin{}

// Init is the plugin init function, will be called when plugin be load.
func (fsp fsPlugin) Init() {
	// where to store file
	l.Info("Args dir =", *filesDir)
	absFilesDir, err := pathutil.Abs(*filesDir)
	if err == nil {
		l.Info("Abs path of photo file dir:", absFilesDir)
		tempDir = absFilesDir
	} else {
		l.Fatal("A error happened when change photo dir to absolute path:", err)
	}
	// if target dir not exist, create it
	if util.CheckExist(absFilesDir) {
		l.Info("Photo file dir already exist")
	} else {
		l.Info("Photo dir not eixst, try to create it")
		err = os.MkdirAll(absFilesDir, 0755)
		if err == nil {
			l.Info("Create dir", absFilesDir, "successfully")
		} else {
			l.Fatal("A error happened when try to create photo dir:", err)
		}
	}

	l.Info("Fs plugin start successfully")
}

func deleteFile(filepath string) {
	if util.CheckExist(filepath) {
		if err := os.Remove(filepath); err != nil {
			l.Fatal("A error happened when try to delete file", filepath, ":", err)
		}
	}
}

// A shortcut funtion to build state we need.
func buildCopyingState(taskID string) plugins.State {
	return plugins.State{
		TaskID:      taskID,
		StateCode:   stateCopyingCode,
		State:       stateCopying,
		Description: stateCopyingDesc,
	}
}

// background operate, save file to disk
func saveFile(uploadFile multipart.File, saveTo *os.File, taskID string) {
	l.Info("Starting task", taskID)

	filepath := saveTo.Name()

	// If error happend when create task, delete file
	if err := plugins.CreateTask(buildCopyingState(taskID)); err != nil {
		l.Warn("A error happend when add task", taskID, ":", err)
		saveTo.Close()
		uploadFile.Close()
		deleteFile(filepath)
		return
	}

	// copy file to disk, then close
	_, err := io.Copy(saveTo, uploadFile)
	saveTo.Close()
	uploadFile.Close()

	if err == nil {
		// copy file successfully
		l.Info("File copy on task", taskID, "successfully")

		if err := plugins.DeleteTask(taskID); err == nil {
			l.Info("Task", taskID, "finished")
		} else {
			// delete task failed, delete file and exit
			deleteFile(filepath)
			l.Fatal("A error happened when delete task", taskID, ":", err)
		}
	} else {
		// copy file failed, delete file and turn state to error
		deleteFile(filepath)
		l.Warn("A error happened when copy file", taskID, ":", err)

		if err := plugins.ChangeTaskState(plugins.BuildErrorState(taskID, err.Error())); err == nil {
			l.Info("Turn task", taskID, "state to error")
		} else {
			// change task state error, exit.
			l.Fatal("A error happend when change task", taskID, "to error state:", err)
		}
	}
}

// SaveRequestHandle Will be called when recieve a file save request.
func (fsp fsPlugin) SaveRequestHandle(q *plugins.SaveRequest) (taskID string, err error) {
	l.Info("Recieved a file save request")

	// Task ID use time prefix and a number follow it(produce by TempFile)
	now := time.Now().UTC()
	saveTo, err := ioutil.TempFile(tempDir, now.Format("2006-01-02-"))

	if err == nil {
		l.Info("Create file on fs successfully:", saveTo.Name())
	} else {
		l.Error("Create file on fs error:", err)
		return "", err
	}

	_, taskID = pathutil.Split(saveTo.Name())

	// start background copy operate
	go saveFile(q.File, saveTo, taskID)

	return taskID, nil
}

// StateRequestHandle Will be called when recieve a get state request.
func (fsp fsPlugin) StateRequestHandle(taskID string) (pState *plugins.State, err error) {
	// taskID exist on task list, just return it
	if pState, err = plugins.GetTaskState(taskID); err == nil {
		return pState, nil
	}
	// TaskID not exist or error when get it, check if image file already exist
	if util.CheckExist(pathutil.Join(tempDir, taskID)) {
		// file exist as a finished state
		finishState := plugins.BuildFinishState(taskID)
		return &finishState, nil
	}
	// get state error
	return nil, err
}

// buildURL build complete url from request's Host header and task ID
func buildURL(r *http.Request, taskID string) string {
	res := url.URL{
		Scheme: "http",
		Host:   r.Host,
		Path:   "files/" + taskID,
	}
	return res.String()
}

// URLRequestHandle will be called when recieve a get image url by taskID request
func (fsp fsPlugin) URLRequestHandle(q *plugins.URLRequest) (url *plugins.URLJSON, err error) {
	taskID := q.TaskID
	r := q.HTTPRequest
	// If file exist, return url
	if util.CheckExist(pathutil.Join(tempDir, taskID)) {
		url := buildURL(r, taskID)
		return &plugins.URLJSON{URL: url}, nil
	}
	return nil, errors.New("File not exist.")
}

// ExtraHandlers return value will be add to http handle list.
// In fs plugin, we start a static file server to serve image file we accped in /files/taskID path.
func (fsp fsPlugin) ExtraHandlers() (handlers []plugins.HandlerWithPattern) {
	// get a base file server
	fileServer := http.StripPrefix(
		"/files",
		// Disable list dir
		util.DisableListDir(
			http.FileServer(http.Dir(tempDir)),
		),
	)
	// only accped GET request
	requestFilterFileServer := util.RequestFilter(
		"", "GET", l,
		func(w http.ResponseWriter, q *http.Request) {
			fileServer.ServeHTTP(w, q)
		},
	)

	handlers = []plugins.HandlerWithPattern{
		plugins.HandlerWithPattern{
			Pattern: "/files/", Handler: requestFilterFileServer,
		},
	}

	return handlers
}
<|endoftext|>


<commit_before>package sudoku

import (
	"math/rand"
)

type RankedObject interface {
	Rank() int
}

type FiniteQueue struct {
	min     int
	max     int
	objects []*finiteQueueList
}

type finiteQueueList struct {
	objects []RankedObject
	numNils int
	rank    int
}

type SyncedFiniteQueue struct {
	queue FiniteQueue
	In    chan RankedObject
	Out   chan RankedObject
	Exit  chan bool
}

//Returns a new queue that will work for items with a rank as low as min or as high as max (inclusive)
func NewFiniteQueue(min int, max int) *FiniteQueue {
	result := FiniteQueue{min, max, make([]*finiteQueueList, max-min+1)}
	for i := 0; i < max-min+1; i++ {
		result.objects[i] = &finiteQueueList{make([]RankedObject, 1), 0, i + result.min}
	}
	return &result
}

func NewSyncedFiniteQueue(min int, max int) *SyncedFiniteQueue {
	result := &SyncedFiniteQueue{*NewFiniteQueue(min, max), make(chan RankedObject), make(chan RankedObject), make(chan bool, 1)}
	go result.workLoop()
	return result
}

func (self *finiteQueueList) trimNils() {
	//TODO: if the numNils is greater than some proportion of len, do a full reallocate.
	for len(self.objects) > 0 && self.objects[0] == nil {
		self.objects = self.objects[1:]
		self.numNils--
	}
}

func (self *finiteQueueList) setNil(index int) {
	self.objects[index] = nil
	self.numNils++
}

func (self *finiteQueueList) getItem() RankedObject {
	self.trimNils()
	for len(self.objects) > 0 {
		index := rand.Intn(len(self.objects))
		obj := self.objects[index]
		if obj == nil {
			continue
		}
		if obj.Rank() != self.rank {
			self.setNil(index)
		}
		self.trimNils()
	}
	return nil
}

func (self *finiteQueueList) addItem(item RankedObject) {
	//Scrub the list for this item.
	for _, obj := range self.objects {
		//Structs will compare equal if all of their fields are the same.
		if item == obj {
			//It's already there, just return.
			return
		}
	}
	self.objects = append(self.objects, item)
}

func (self *SyncedFiniteQueue) workLoop() {
	for {
		firstItem := self.queue.Get()
		itemSent := false
		if firstItem == nil {
			//We can take in new things or accept an exit.
			select {
			case <-self.Exit:
				return
			case incoming := <-self.In:
				self.queue.Insert(incoming)
			}
		} else {
			//We can take in new things, send out smallest one, or exit.
			select {
			case <-self.Exit:
				return
			case incoming := <-self.In:
				self.queue.Insert(incoming)
			case self.Out <- firstItem:
				itemSent = true
			}
			//If we didn't send the item out, we need to put it back in.
			if !itemSent {
				self.queue.Insert(firstItem)
			}
		}
	}
}

func (self *FiniteQueue) Min() int {
	return self.min
}

func (self *FiniteQueue) Max() int {
	return self.max
}

func (self *FiniteQueue) Insert(obj RankedObject) {
	rank := obj.Rank()
	list, ok := self.getList(rank)
	if !ok {
		//Apparently rank wasn't legal.
		return
	}
	list.addItem(obj)
}

func (self *FiniteQueue) Get() RankedObject {
	return self.GetSmallerThan(self.max + 1)
}

func (self *FiniteQueue) GetSmallerThan(max int) RankedObject {
	return self.getSmallerThan(max, make(map[RankedObject]int))
}

func (self *FiniteQueue) getSmallerThan(max int, ignoredObjects map[RankedObject]int) RankedObject {
	//TOOD: actually respect ignoredObjects
	for i, list := range self.objects {
		if i+self.min >= max {
			return nil
		}
		result := list.getItem()
		if result != nil {
			return result
		}
	}
	return nil
}

func (self *FiniteQueue) legalRank(rank int) bool {
	return rank >= self.min && rank <= self.max
}

func (self *FiniteQueue) getList(rank int) (*finiteQueueList, bool) {
	if !self.legalRank(rank) {
		return nil, false
	}
	return self.objects[rank-self.min], true
}
<commit_msg>TESTS FAIL (by  not completing). mark items passed out so we don't pass them out again. But the tests still loop forever.<commit_after>package sudoku

import (
	"math/rand"
)

type RankedObject interface {
	Rank() int
}

type FiniteQueue struct {
	min     int
	max     int
	objects []*finiteQueueList
}

type finiteQueueList struct {
	objects []RankedObject
	numNils int
	rank    int
}

type SyncedFiniteQueue struct {
	queue FiniteQueue
	In    chan RankedObject
	Out   chan RankedObject
	Exit  chan bool
}

//Returns a new queue that will work for items with a rank as low as min or as high as max (inclusive)
func NewFiniteQueue(min int, max int) *FiniteQueue {
	result := FiniteQueue{min, max, make([]*finiteQueueList, max-min+1)}
	for i := 0; i < max-min+1; i++ {
		result.objects[i] = &finiteQueueList{make([]RankedObject, 1), 0, i + result.min}
	}
	return &result
}

func NewSyncedFiniteQueue(min int, max int) *SyncedFiniteQueue {
	result := &SyncedFiniteQueue{*NewFiniteQueue(min, max), make(chan RankedObject), make(chan RankedObject), make(chan bool, 1)}
	go result.workLoop()
	return result
}

func (self *finiteQueueList) trimNils() {
	//TODO: if the numNils is greater than some proportion of len, do a full reallocate.
	for len(self.objects) > 0 && self.objects[0] == nil {
		self.objects = self.objects[1:]
		self.numNils--
	}
}

func (self *finiteQueueList) setNil(index int) {
	self.objects[index] = nil
	self.numNils++
}

func (self *finiteQueueList) getItem() RankedObject {
	self.trimNils()
	for len(self.objects) > 0 {
		index := rand.Intn(len(self.objects))
		obj := self.objects[index]
		//Mark this object as gathered.
		self.setNil(index)
		if obj == nil {
			continue
		}
		if obj.Rank() != self.rank {
			continue
		}
		self.trimNils()
	}
	return nil
}

func (self *finiteQueueList) addItem(item RankedObject) {
	//Scrub the list for this item.
	for _, obj := range self.objects {
		//Structs will compare equal if all of their fields are the same.
		if item == obj {
			//It's already there, just return.
			return
		}
	}
	self.objects = append(self.objects, item)
}

func (self *SyncedFiniteQueue) workLoop() {
	for {
		firstItem := self.queue.Get()
		itemSent := false
		if firstItem == nil {
			//We can take in new things or accept an exit.
			select {
			case <-self.Exit:
				return
			case incoming := <-self.In:
				self.queue.Insert(incoming)
			}
		} else {
			//We can take in new things, send out smallest one, or exit.
			select {
			case <-self.Exit:
				return
			case incoming := <-self.In:
				self.queue.Insert(incoming)
			case self.Out <- firstItem:
				itemSent = true
			}
			//If we didn't send the item out, we need to put it back in.
			if !itemSent {
				self.queue.Insert(firstItem)
			}
		}
	}
}

func (self *FiniteQueue) Min() int {
	return self.min
}

func (self *FiniteQueue) Max() int {
	return self.max
}

func (self *FiniteQueue) Insert(obj RankedObject) {
	rank := obj.Rank()
	list, ok := self.getList(rank)
	if !ok {
		//Apparently rank wasn't legal.
		return
	}
	list.addItem(obj)
}

func (self *FiniteQueue) Get() RankedObject {
	return self.GetSmallerThan(self.max + 1)
}

func (self *FiniteQueue) GetSmallerThan(max int) RankedObject {
	return self.getSmallerThan(max, make(map[RankedObject]int))
}

func (self *FiniteQueue) getSmallerThan(max int, ignoredObjects map[RankedObject]int) RankedObject {
	//TOOD: actually respect ignoredObjects
	for i, list := range self.objects {
		if i+self.min >= max {
			return nil
		}
		result := list.getItem()
		if result != nil {
			return result
		}
	}
	return nil
}

func (self *FiniteQueue) legalRank(rank int) bool {
	return rank >= self.min && rank <= self.max
}

func (self *FiniteQueue) getList(rank int) (*finiteQueueList, bool) {
	if !self.legalRank(rank) {
		return nil, false
	}
	return self.objects[rank-self.min], true
}
<|endoftext|>


<commit_before>/*
Package ant provides an abstraction for the functionality of 'ants' in the
antfarm. Ants are Sia clients that have a myriad of user stories programmed as
their behavior and report their successfullness at each user store.
*/
package ant

import (
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/NebulousLabs/Sia/api"
)

// newSiad spawns a new siad process using os/exec and waits for the api to
// become available.  siadPath is the path to Siad, passed directly to
// exec.Command.  An error is returned if starting siad fails, otherwise a
// pointer to siad's os.Cmd object is returned.  The data directory `datadir`
// is passed as siad's `--sia-directory`.
func newSiad(siadPath string, datadir string, apiAddr string, rpcAddr string, hostAddr string) (*exec.Cmd, error) {
	if err := checkSiadConstants(siadPath); err != nil {
		return nil, err
	}
	// create a logfile for Sia's stderr and stdout.
	logfile, err := os.Create(filepath.Join(datadir, "sia-output.log"))
	if err != nil {
		return nil, err
	}
	cmd := exec.Command(siadPath, "--no-bootstrap", "--sia-directory="+datadir, "--api-addr="+apiAddr, "--rpc-addr="+rpcAddr, "--host-addr="+hostAddr)
	cmd.Stderr = logfile
	cmd.Stdout = logfile

	if err := cmd.Start(); err != nil {
		return nil, err
	}

	if err := waitForAPI(apiAddr, cmd); err != nil {
		return nil, err
	}

	return cmd, nil
}

// checkSiadConstants runs `siad version` and verifies that the supplied siad
// is running the correct, dev, constants. Returns an error if the correct
// constants are not running, otherwise returns nil.
func checkSiadConstants(siadPath string) error {
	cmd := exec.Command(siadPath, "version")
	output, err := cmd.Output()
	if err != nil {
		return err
	}

	if !strings.Contains(string(output), "-dev") {
		return errors.New("supplied siad is not running required dev constants")
	}

	return nil
}

// stopSiad tries to stop the siad running at `apiAddr`, issuing a kill to its
// `process` after a timeout.
func stopSiad(apiAddr string, process *os.Process) {
	if err := api.NewClient(apiAddr, "").Get("/daemon/stop", nil); err != nil {
		process.Kill()
	}

	// wait for 120 seconds for siad to terminate, then issue a kill signal.
	done := make(chan error)
	go func() {
		_, err := process.Wait()
		done <- err
	}()
	select {
	case <-done:
	case <-time.After(120 * time.Second):
		process.Kill()
	}
}

// waitForAPI blocks until the Sia API at apiAddr becomes available.
// if siad returns while waiting for the api, return an error.
func waitForAPI(apiAddr string, siad *exec.Cmd) error {
	c := api.NewClient(apiAddr, "")

	exitchan := make(chan error)
	go func() {
		_, err := siad.Process.Wait()
		exitchan <- err
	}()

	// Wait for the Sia API to become available.
	success := false
	for start := time.Now(); time.Since(start) < 5*time.Minute; time.Sleep(time.Millisecond * 100) {
		if success {
			break
		}
		select {
		case err := <-exitchan:
			return fmt.Errorf("siad exited unexpectedly while waiting for api, exited with error: %v\n", err)
		default:
			if err := c.Get("/consensus", nil); err == nil {
				success = true
			}
		}
	}
	if !success {
		stopSiad(apiAddr, siad.Process)
		return errors.New("timeout: couldnt reach api after 5 minutes")
	}
	return nil
}
<commit_msg>add --modules flag to run required modules in antfarm<commit_after>/*
Package ant provides an abstraction for the functionality of 'ants' in the
antfarm. Ants are Sia clients that have a myriad of user stories programmed as
their behavior and report their successfullness at each user store.
*/
package ant

import (
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/NebulousLabs/Sia/api"
)

// newSiad spawns a new siad process using os/exec and waits for the api to
// become available.  siadPath is the path to Siad, passed directly to
// exec.Command.  An error is returned if starting siad fails, otherwise a
// pointer to siad's os.Cmd object is returned.  The data directory `datadir`
// is passed as siad's `--sia-directory`.
func newSiad(siadPath string, datadir string, apiAddr string, rpcAddr string, hostAddr string) (*exec.Cmd, error) {
	if err := checkSiadConstants(siadPath); err != nil {
		return nil, err
	}
	// create a logfile for Sia's stderr and stdout.
	logfile, err := os.Create(filepath.Join(datadir, "sia-output.log"))
	if err != nil {
		return nil, err
	}
	cmd := exec.Command(siadPath, "--modules=cgthmrw", "--no-bootstrap", "--sia-directory="+datadir, "--api-addr="+apiAddr, "--rpc-addr="+rpcAddr, "--host-addr="+hostAddr)
	cmd.Stderr = logfile
	cmd.Stdout = logfile

	if err := cmd.Start(); err != nil {
		return nil, err
	}

	if err := waitForAPI(apiAddr, cmd); err != nil {
		return nil, err
	}

	return cmd, nil
}

// checkSiadConstants runs `siad version` and verifies that the supplied siad
// is running the correct, dev, constants. Returns an error if the correct
// constants are not running, otherwise returns nil.
func checkSiadConstants(siadPath string) error {
	cmd := exec.Command(siadPath, "version")
	output, err := cmd.Output()
	if err != nil {
		return err
	}

	if !strings.Contains(string(output), "-dev") {
		return errors.New("supplied siad is not running required dev constants")
	}

	return nil
}

// stopSiad tries to stop the siad running at `apiAddr`, issuing a kill to its
// `process` after a timeout.
func stopSiad(apiAddr string, process *os.Process) {
	if err := api.NewClient(apiAddr, "").Get("/daemon/stop", nil); err != nil {
		process.Kill()
	}

	// wait for 120 seconds for siad to terminate, then issue a kill signal.
	done := make(chan error)
	go func() {
		_, err := process.Wait()
		done <- err
	}()
	select {
	case <-done:
	case <-time.After(120 * time.Second):
		process.Kill()
	}
}

// waitForAPI blocks until the Sia API at apiAddr becomes available.
// if siad returns while waiting for the api, return an error.
func waitForAPI(apiAddr string, siad *exec.Cmd) error {
	c := api.NewClient(apiAddr, "")

	exitchan := make(chan error)
	go func() {
		_, err := siad.Process.Wait()
		exitchan <- err
	}()

	// Wait for the Sia API to become available.
	success := false
	for start := time.Now(); time.Since(start) < 5*time.Minute; time.Sleep(time.Millisecond * 100) {
		if success {
			break
		}
		select {
		case err := <-exitchan:
			return fmt.Errorf("siad exited unexpectedly while waiting for api, exited with error: %v\n", err)
		default:
			if err := c.Get("/consensus", nil); err == nil {
				success = true
			}
		}
	}
	if !success {
		stopSiad(apiAddr, siad.Process)
		return errors.New("timeout: couldnt reach api after 5 minutes")
	}
	return nil
}
<|endoftext|>