<commit_before>package godog

import (
	"fmt"
	"math"
	"regexp"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/DATA-DOG/godog/gherkin"
)

func init() {
	Format("pretty", "Prints every feature with runtime statuses.", &pretty{
		basefmt: basefmt{
			started: time.Now(),
			indent:  2,
		},
	})
}

var outlinePlaceholderRegexp = regexp.MustCompile("<[^>]+>")

// a built in default pretty formatter
type pretty struct {
	basefmt

	// currently processed
	feature  *gherkin.Feature
	scenario *gherkin.Scenario
	outline  *gherkin.ScenarioOutline

	// state
	bgSteps    int
	steps      int
	commentPos int

	// outline
	outlineSteps       []*stepResult
	outlineNumExample  int
	outlineNumExamples int
}

func (f *pretty) Feature(ft *gherkin.Feature, p string) {
	if len(f.features) != 0 {
		// not a first feature, add a newline
		fmt.Println("")
	}
	f.features = append(f.features, &feature{Path: p, Feature: ft})
	fmt.Println(bcl(ft.Keyword+": ", white) + ft.Name)
	if strings.TrimSpace(ft.Description) != "" {
		for _, line := range strings.Split(ft.Description, "\n") {
			fmt.Println(s(f.indent) + strings.TrimSpace(line))
		}
	}

	f.feature = ft
	f.scenario = nil
	f.outline = nil
	f.bgSteps = 0
	if ft.Background != nil {
		f.bgSteps = len(ft.Background.Steps)
	}
}

// Node takes a gherkin node for formatting
func (f *pretty) Node(node interface{}) {
	f.basefmt.Node(node)

	switch t := node.(type) {
	case *gherkin.Examples:
		f.outlineNumExamples = len(t.TableBody)
		f.outlineNumExample++
	case *gherkin.Scenario:
		f.scenario = t
		f.outline = nil
		f.steps = len(t.Steps)
	case *gherkin.ScenarioOutline:
		f.outline = t
		f.scenario = nil
		f.steps = len(t.Steps)
		f.outlineNumExample = -1
	}
}

// Summary sumarize the feature formatter output
func (f *pretty) Summary() {
	// failed steps on background are not scenarios
	var failedScenarios []*stepResult
	for _, fail := range f.failed {
		switch fail.owner.(type) {
		case *gherkin.Scenario:
			failedScenarios = append(failedScenarios, fail)
		case *gherkin.ScenarioOutline:
			failedScenarios = append(failedScenarios, fail)
		}
	}
	if len(failedScenarios) > 0 {
		fmt.Println("\n--- " + cl("Failed scenarios:", red) + "\n")
		var unique []string
		for _, fail := range failedScenarios {
			var found bool
			for _, in := range unique {
				if in == fail.line() {
					found = true
					break
				}
			}
			if !found {
				unique = append(unique, fail.line())
			}
		}

		for _, fail := range unique {
			fmt.Println("    " + cl(fail, red))
		}
	}
	f.basefmt.Summary()
}

func (f *pretty) printOutlineExample(outline *gherkin.ScenarioOutline) {
	var msg string
	clr := green

	ex := outline.Examples[f.outlineNumExample]
	example, hasExamples := examples(ex)
	if !hasExamples {
		// do not print empty examples
		return
	}

	firstExample := f.outlineNumExamples == len(example.TableBody)
	printSteps := firstExample && f.outlineNumExample == 0

	for i, res := range f.outlineSteps {
		// determine example row status
		switch {
		case res.typ == failed:
			msg = res.err.Error()
			clr = res.typ.clr()
		case res.typ == undefined || res.typ == pending:
			clr = res.typ.clr()
		case res.typ == skipped && clr == green:
			clr = cyan
		}
		if printSteps {
			// in first example, we need to print steps
			var text string
			ostep := outline.Steps[i]
			if res.def != nil {
				if m := outlinePlaceholderRegexp.FindAllStringIndex(ostep.Text, -1); len(m) > 0 {
					var pos int
					for i := 0; i < len(m); i++ {
						pair := m[i]
						text += cl(ostep.Text[pos:pair[0]], cyan)
						text += bcl(ostep.Text[pair[0]:pair[1]], cyan)
						pos = pair[1]
					}
					text += cl(ostep.Text[pos:len(ostep.Text)], cyan)
				} else {
					text = cl(ostep.Text, cyan)
				}
				text += s(f.commentPos-f.length(ostep)+1) + cl(fmt.Sprintf("# %s", res.def.funcName()), black)
			} else {
				text = cl(ostep.Text, cyan)
			}
			// print the step outline
			fmt.Println(s(f.indent*2) + cl(strings.TrimSpace(ostep.Keyword), cyan) + " " + text)
		}
	}

	cells := make([]string, len(example.TableHeader.Cells))
	max := longest(example)
	// an example table header
	if firstExample {
		fmt.Println("")
		fmt.Println(s(f.indent*2) + bcl(example.Keyword+": ", white) + example.Name)

		for i, cell := range example.TableHeader.Cells {
			cells[i] = cl(cell.Value, cyan) + s(max[i]-len(cell.Value))
		}
		fmt.Println(s(f.indent*3) + "| " + strings.Join(cells, " | ") + " |")
	}

	// an example table row
	row := example.TableBody[len(example.TableBody)-f.outlineNumExamples]
	for i, cell := range row.Cells {
		cells[i] = cl(cell.Value, clr) + s(max[i]-len(cell.Value))
	}
	fmt.Println(s(f.indent*3) + "| " + strings.Join(cells, " | ") + " |")

	// if there is an error
	if msg != "" {
		fmt.Println(s(f.indent*4) + bcl(msg, red))
	}
}

func (f *pretty) printStep(step *gherkin.Step, def *StepDef, c color) {
	text := s(f.indent*2) + cl(strings.TrimSpace(step.Keyword), c) + " "
	switch {
	case def != nil:
		if m := (def.Expr.FindStringSubmatchIndex(step.Text))[2:]; len(m) > 0 {
			var pos, i int
			for pos, i = 0, 0; i < len(m); i++ {
				if math.Mod(float64(i), 2) == 0 {
					text += cl(step.Text[pos:m[i]], c)
				} else {
					text += bcl(step.Text[pos:m[i]], c)
				}
				pos = m[i]
			}
			text += cl(step.Text[pos:len(step.Text)], c)
		} else {
			text += cl(step.Text, c)
		}
		text += s(f.commentPos-f.length(step)+1) + cl(fmt.Sprintf("# %s", def.funcName()), black)
	default:
		text += cl(step.Text, c)
	}

	fmt.Println(text)
	switch t := step.Argument.(type) {
	case *gherkin.DataTable:
		f.printTable(t, c)
	case *gherkin.DocString:
		var ct string
		if len(t.ContentType) > 0 {
			ct = " " + cl(t.ContentType, c)
		}
		fmt.Println(s(f.indent*3) + cl(t.Delimitter, c) + ct)
		for _, ln := range strings.Split(t.Content, "\n") {
			fmt.Println(s(f.indent*3) + cl(ln, c))
		}
		fmt.Println(s(f.indent*3) + cl(t.Delimitter, c))
	}
}

func (f *pretty) printStepKind(res *stepResult) {
	_, isBgStep := res.owner.(*gherkin.Background)

	// if has not printed background yet
	switch {
	// first background step
	case f.bgSteps > 0 && f.bgSteps == len(f.feature.Background.Steps):
		f.commentPos = f.longestStep(f.feature.Background.Steps, f.length(f.feature.Background))
		fmt.Println("\n" + s(f.indent) + bcl(f.feature.Background.Keyword+": "+f.feature.Background.Name, white))
		f.bgSteps--
	// subsequent background steps
	case f.bgSteps > 0:
		f.bgSteps--
	// a background step for another scenario, but all bg steps are
	// already printed. so just skip it
	case isBgStep:
		return
	// first step of scenario, print header and calculate comment position
	case f.scenario != nil && f.steps == len(f.scenario.Steps):
		f.commentPos = f.longestStep(f.scenario.Steps, f.length(f.scenario))
		text := s(f.indent) + bcl(f.scenario.Keyword+": ", white) + f.scenario.Name
		text += s(f.commentPos-f.length(f.scenario)+1) + f.line(f.scenario.Location)
		fmt.Println("\n" + text)
		f.steps--
	// all subsequent scenario steps
	case f.scenario != nil:
		f.steps--
	// first step of outline scenario, print header and calculate comment position
	case f.outline != nil && f.steps == len(f.outline.Steps):
		f.commentPos = f.longestStep(f.outline.Steps, f.length(f.outline))
		text := s(f.indent) + bcl(f.outline.Keyword+": ", white) + f.outline.Name
		text += s(f.commentPos-f.length(f.outline)+1) + f.line(f.outline.Location)
		fmt.Println("\n" + text)
		f.outlineSteps = append(f.outlineSteps, res)
		f.steps--
		if len(f.outlineSteps) == len(f.outline.Steps) {
			// an outline example steps has went through
			f.printOutlineExample(f.outline)
			f.outlineSteps = []*stepResult{}
			f.outlineNumExamples--
		}
		return
	// all subsequent outline steps
	case f.outline != nil:
		f.outlineSteps = append(f.outlineSteps, res)
		f.steps--
		if len(f.outlineSteps) == len(f.outline.Steps) {
			// an outline example steps has went through
			f.printOutlineExample(f.outline)
			f.outlineSteps = []*stepResult{}
			f.outlineNumExamples--
		}
		return
	}

	f.printStep(res.step, res.def, res.typ.clr())
	if res.err != nil {
		fmt.Println(s(f.indent*2) + bcl(res.err, red))
	}
	if res.typ == pending {
		fmt.Println(s(f.indent*3) + cl("TODO: write pending definition", yellow))
	}
}

// print table with aligned table cells
func (f *pretty) printTable(t *gherkin.DataTable, c color) {
	var l = longest(t)
	var cols = make([]string, len(t.Rows[0].Cells))
	for _, row := range t.Rows {
		for i, cell := range row.Cells {
			cols[i] = cell.Value + s(l[i]-len(cell.Value))
		}
		fmt.Println(s(f.indent*3) + cl("| "+strings.Join(cols, " | ")+" |", c))
	}
}

func (f *pretty) Passed(step *gherkin.Step, match *StepDef) {
	f.basefmt.Passed(step, match)
	f.printStepKind(f.passed[len(f.passed)-1])
}

func (f *pretty) Skipped(step *gherkin.Step) {
	f.basefmt.Skipped(step)
	f.printStepKind(f.skipped[len(f.skipped)-1])
}

func (f *pretty) Undefined(step *gherkin.Step) {
	f.basefmt.Undefined(step)
	f.printStepKind(f.undefined[len(f.undefined)-1])
}

func (f *pretty) Failed(step *gherkin.Step, match *StepDef, err error) {
	f.basefmt.Failed(step, match, err)
	f.printStepKind(f.failed[len(f.failed)-1])
}

func (f *pretty) Pending(step *gherkin.Step, match *StepDef) {
	f.basefmt.Pending(step, match)
	f.printStepKind(f.pending[len(f.pending)-1])
}

// longest gives a list of longest columns of all rows in Table
func longest(tbl interface{}) []int {
	var rows []*gherkin.TableRow
	switch t := tbl.(type) {
	case *gherkin.Examples:
		rows = append(rows, t.TableHeader)
		rows = append(rows, t.TableBody...)
	case *gherkin.DataTable:
		rows = append(rows, t.Rows...)
	}

	longest := make([]int, len(rows[0].Cells))
	for _, row := range rows {
		for i, cell := range row.Cells {
			if longest[i] < len(cell.Value) {
				longest[i] = len(cell.Value)
			}
		}
	}
	return longest
}

func (f *pretty) longestStep(steps []*gherkin.Step, base int) int {
	ret := base
	for _, step := range steps {
		length := f.length(step)
		if length > ret {
			ret = length
		}
	}
	return ret
}

// a line number representation in feature file
func (f *pretty) line(loc *gherkin.Location) string {
	return cl(fmt.Sprintf("# %s:%d", f.features[len(f.features)-1].Path, loc.Line), black)
}

func (f *pretty) length(node interface{}) int {
	switch t := node.(type) {
	case *gherkin.Background:
		return f.indent + utf8.RuneCountInString(strings.TrimSpace(t.Keyword)+": "+t.Name)
	case *gherkin.Step:
		return f.indent*2 + utf8.RuneCountInString(strings.TrimSpace(t.Keyword)+" "+t.Text)
	case *gherkin.Scenario:
		return f.indent + utf8.RuneCountInString(strings.TrimSpace(t.Keyword)+": "+t.Name)
	case *gherkin.ScenarioOutline:
		return f.indent + utf8.RuneCountInString(strings.TrimSpace(t.Keyword)+": "+t.Name)
	}
	panic(fmt.Sprintf("unexpected node %T to determine length", node))
}
<commit_msg>use maximum step length of background and scenario<commit_after>package godog

import (
	"fmt"
	"math"
	"regexp"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/DATA-DOG/godog/gherkin"
)

func init() {
	Format("pretty", "Prints every feature with runtime statuses.", &pretty{
		basefmt: basefmt{
			started: time.Now(),
			indent:  2,
		},
	})
}

var outlinePlaceholderRegexp = regexp.MustCompile("<[^>]+>")

// a built in default pretty formatter
type pretty struct {
	basefmt

	// currently processed
	feature  *gherkin.Feature
	scenario *gherkin.Scenario
	outline  *gherkin.ScenarioOutline

	// state
	bgSteps    int
	steps      int
	commentPos int

	// outline
	outlineSteps       []*stepResult
	outlineNumExample  int
	outlineNumExamples int
}

func (f *pretty) Feature(ft *gherkin.Feature, p string) {
	if len(f.features) != 0 {
		// not a first feature, add a newline
		fmt.Println("")
	}
	f.features = append(f.features, &feature{Path: p, Feature: ft})
	fmt.Println(bcl(ft.Keyword+": ", white) + ft.Name)
	if strings.TrimSpace(ft.Description) != "" {
		for _, line := range strings.Split(ft.Description, "\n") {
			fmt.Println(s(f.indent) + strings.TrimSpace(line))
		}
	}

	f.feature = ft
	f.scenario = nil
	f.outline = nil
	f.bgSteps = 0
	if ft.Background != nil {
		f.bgSteps = len(ft.Background.Steps)
	}
}

// Node takes a gherkin node for formatting
func (f *pretty) Node(node interface{}) {
	f.basefmt.Node(node)

	switch t := node.(type) {
	case *gherkin.Examples:
		f.outlineNumExamples = len(t.TableBody)
		f.outlineNumExample++
	case *gherkin.Scenario:
		f.scenario = t
		f.outline = nil
		f.steps = len(t.Steps)
	case *gherkin.ScenarioOutline:
		f.outline = t
		f.scenario = nil
		f.steps = len(t.Steps)
		f.outlineNumExample = -1
	}
}

// Summary sumarize the feature formatter output
func (f *pretty) Summary() {
	// failed steps on background are not scenarios
	var failedScenarios []*stepResult
	for _, fail := range f.failed {
		switch fail.owner.(type) {
		case *gherkin.Scenario:
			failedScenarios = append(failedScenarios, fail)
		case *gherkin.ScenarioOutline:
			failedScenarios = append(failedScenarios, fail)
		}
	}
	if len(failedScenarios) > 0 {
		fmt.Println("\n--- " + cl("Failed scenarios:", red) + "\n")
		var unique []string
		for _, fail := range failedScenarios {
			var found bool
			for _, in := range unique {
				if in == fail.line() {
					found = true
					break
				}
			}
			if !found {
				unique = append(unique, fail.line())
			}
		}

		for _, fail := range unique {
			fmt.Println("    " + cl(fail, red))
		}
	}
	f.basefmt.Summary()
}

func (f *pretty) printOutlineExample(outline *gherkin.ScenarioOutline) {
	var msg string
	clr := green

	ex := outline.Examples[f.outlineNumExample]
	example, hasExamples := examples(ex)
	if !hasExamples {
		// do not print empty examples
		return
	}

	firstExample := f.outlineNumExamples == len(example.TableBody)
	printSteps := firstExample && f.outlineNumExample == 0

	for i, res := range f.outlineSteps {
		// determine example row status
		switch {
		case res.typ == failed:
			msg = res.err.Error()
			clr = res.typ.clr()
		case res.typ == undefined || res.typ == pending:
			clr = res.typ.clr()
		case res.typ == skipped && clr == green:
			clr = cyan
		}
		if printSteps {
			// in first example, we need to print steps
			var text string
			ostep := outline.Steps[i]
			if res.def != nil {
				if m := outlinePlaceholderRegexp.FindAllStringIndex(ostep.Text, -1); len(m) > 0 {
					var pos int
					for i := 0; i < len(m); i++ {
						pair := m[i]
						text += cl(ostep.Text[pos:pair[0]], cyan)
						text += bcl(ostep.Text[pair[0]:pair[1]], cyan)
						pos = pair[1]
					}
					text += cl(ostep.Text[pos:len(ostep.Text)], cyan)
				} else {
					text = cl(ostep.Text, cyan)
				}
				text += s(f.commentPos-f.length(ostep)+1) + cl(fmt.Sprintf("# %s", res.def.funcName()), black)
			} else {
				text = cl(ostep.Text, cyan)
			}
			// print the step outline
			fmt.Println(s(f.indent*2) + cl(strings.TrimSpace(ostep.Keyword), cyan) + " " + text)
		}
	}

	cells := make([]string, len(example.TableHeader.Cells))
	max := longest(example)
	// an example table header
	if firstExample {
		fmt.Println("")
		fmt.Println(s(f.indent*2) + bcl(example.Keyword+": ", white) + example.Name)

		for i, cell := range example.TableHeader.Cells {
			cells[i] = cl(cell.Value, cyan) + s(max[i]-len(cell.Value))
		}
		fmt.Println(s(f.indent*3) + "| " + strings.Join(cells, " | ") + " |")
	}

	// an example table row
	row := example.TableBody[len(example.TableBody)-f.outlineNumExamples]
	for i, cell := range row.Cells {
		cells[i] = cl(cell.Value, clr) + s(max[i]-len(cell.Value))
	}
	fmt.Println(s(f.indent*3) + "| " + strings.Join(cells, " | ") + " |")

	// if there is an error
	if msg != "" {
		fmt.Println(s(f.indent*4) + bcl(msg, red))
	}
}

func (f *pretty) printStep(step *gherkin.Step, def *StepDef, c color) {
	text := s(f.indent*2) + cl(strings.TrimSpace(step.Keyword), c) + " "
	switch {
	case def != nil:
		if m := (def.Expr.FindStringSubmatchIndex(step.Text))[2:]; len(m) > 0 {
			var pos, i int
			for pos, i = 0, 0; i < len(m); i++ {
				if math.Mod(float64(i), 2) == 0 {
					text += cl(step.Text[pos:m[i]], c)
				} else {
					text += bcl(step.Text[pos:m[i]], c)
				}
				pos = m[i]
			}
			text += cl(step.Text[pos:len(step.Text)], c)
		} else {
			text += cl(step.Text, c)
		}
		text += s(f.commentPos-f.length(step)+1) + cl(fmt.Sprintf("# %s", def.funcName()), black)
	default:
		text += cl(step.Text, c)
	}

	fmt.Println(text)
	switch t := step.Argument.(type) {
	case *gherkin.DataTable:
		f.printTable(t, c)
	case *gherkin.DocString:
		var ct string
		if len(t.ContentType) > 0 {
			ct = " " + cl(t.ContentType, c)
		}
		fmt.Println(s(f.indent*3) + cl(t.Delimitter, c) + ct)
		for _, ln := range strings.Split(t.Content, "\n") {
			fmt.Println(s(f.indent*3) + cl(ln, c))
		}
		fmt.Println(s(f.indent*3) + cl(t.Delimitter, c))
	}
}

func (f *pretty) printStepKind(res *stepResult) {
	_, isBgStep := res.owner.(*gherkin.Background)

	// if has not printed background yet
	switch {
	// first background step
	case f.bgSteps > 0 && f.bgSteps == len(f.feature.Background.Steps):
		f.commentPos = f.longestStep(f.feature.Background.Steps, f.length(f.feature.Background))
		fmt.Println("\n" + s(f.indent) + bcl(f.feature.Background.Keyword+": "+f.feature.Background.Name, white))
		f.bgSteps--
	// subsequent background steps
	case f.bgSteps > 0:
		f.bgSteps--
	// a background step for another scenario, but all bg steps are
	// already printed. so just skip it
	case isBgStep:
		return
	// first step of scenario, print header and calculate comment position
	case f.scenario != nil && f.steps == len(f.scenario.Steps):
		f.commentPos = f.longestStep(f.scenario.Steps, f.length(f.scenario))
		if bgLen := f.longestStep(f.feature.Background.Steps, f.length(f.feature.Background)); bgLen > f.commentPos {
			f.commentPos = bgLen
		}
		text := s(f.indent) + bcl(f.scenario.Keyword+": ", white) + f.scenario.Name
		text += s(f.commentPos-f.length(f.scenario)+1) + f.line(f.scenario.Location)
		fmt.Println("\n" + text)
		f.steps--
	// all subsequent scenario steps
	case f.scenario != nil:
		f.steps--
	// first step of outline scenario, print header and calculate comment position
	case f.outline != nil && f.steps == len(f.outline.Steps):
		f.commentPos = f.longestStep(f.outline.Steps, f.length(f.outline))
		text := s(f.indent) + bcl(f.outline.Keyword+": ", white) + f.outline.Name
		text += s(f.commentPos-f.length(f.outline)+1) + f.line(f.outline.Location)
		fmt.Println("\n" + text)
		f.outlineSteps = append(f.outlineSteps, res)
		f.steps--
		if len(f.outlineSteps) == len(f.outline.Steps) {
			// an outline example steps has went through
			f.printOutlineExample(f.outline)
			f.outlineSteps = []*stepResult{}
			f.outlineNumExamples--
		}
		return
	// all subsequent outline steps
	case f.outline != nil:
		f.outlineSteps = append(f.outlineSteps, res)
		f.steps--
		if len(f.outlineSteps) == len(f.outline.Steps) {
			// an outline example steps has went through
			f.printOutlineExample(f.outline)
			f.outlineSteps = []*stepResult{}
			f.outlineNumExamples--
		}
		return
	}

	f.printStep(res.step, res.def, res.typ.clr())
	if res.err != nil {
		fmt.Println(s(f.indent*2) + bcl(res.err, red))
	}
	if res.typ == pending {
		fmt.Println(s(f.indent*3) + cl("TODO: write pending definition", yellow))
	}
}

// print table with aligned table cells
func (f *pretty) printTable(t *gherkin.DataTable, c color) {
	var l = longest(t)
	var cols = make([]string, len(t.Rows[0].Cells))
	for _, row := range t.Rows {
		for i, cell := range row.Cells {
			cols[i] = cell.Value + s(l[i]-len(cell.Value))
		}
		fmt.Println(s(f.indent*3) + cl("| "+strings.Join(cols, " | ")+" |", c))
	}
}

func (f *pretty) Passed(step *gherkin.Step, match *StepDef) {
	f.basefmt.Passed(step, match)
	f.printStepKind(f.passed[len(f.passed)-1])
}

func (f *pretty) Skipped(step *gherkin.Step) {
	f.basefmt.Skipped(step)
	f.printStepKind(f.skipped[len(f.skipped)-1])
}

func (f *pretty) Undefined(step *gherkin.Step) {
	f.basefmt.Undefined(step)
	f.printStepKind(f.undefined[len(f.undefined)-1])
}

func (f *pretty) Failed(step *gherkin.Step, match *StepDef, err error) {
	f.basefmt.Failed(step, match, err)
	f.printStepKind(f.failed[len(f.failed)-1])
}

func (f *pretty) Pending(step *gherkin.Step, match *StepDef) {
	f.basefmt.Pending(step, match)
	f.printStepKind(f.pending[len(f.pending)-1])
}

// longest gives a list of longest columns of all rows in Table
func longest(tbl interface{}) []int {
	var rows []*gherkin.TableRow
	switch t := tbl.(type) {
	case *gherkin.Examples:
		rows = append(rows, t.TableHeader)
		rows = append(rows, t.TableBody...)
	case *gherkin.DataTable:
		rows = append(rows, t.Rows...)
	}

	longest := make([]int, len(rows[0].Cells))
	for _, row := range rows {
		for i, cell := range row.Cells {
			if longest[i] < len(cell.Value) {
				longest[i] = len(cell.Value)
			}
		}
	}
	return longest
}

func (f *pretty) longestStep(steps []*gherkin.Step, base int) int {
	ret := base
	for _, step := range steps {
		length := f.length(step)
		if length > ret {
			ret = length
		}
	}
	return ret
}

// a line number representation in feature file
func (f *pretty) line(loc *gherkin.Location) string {
	return cl(fmt.Sprintf("# %s:%d", f.features[len(f.features)-1].Path, loc.Line), black)
}

func (f *pretty) length(node interface{}) int {
	switch t := node.(type) {
	case *gherkin.Background:
		return f.indent + utf8.RuneCountInString(strings.TrimSpace(t.Keyword)+": "+t.Name)
	case *gherkin.Step:
		return f.indent*2 + utf8.RuneCountInString(strings.TrimSpace(t.Keyword)+" "+t.Text)
	case *gherkin.Scenario:
		return f.indent + utf8.RuneCountInString(strings.TrimSpace(t.Keyword)+": "+t.Name)
	case *gherkin.ScenarioOutline:
		return f.indent + utf8.RuneCountInString(strings.TrimSpace(t.Keyword)+": "+t.Name)
	}
	panic(fmt.Sprintf("unexpected node %T to determine length", node))
}
<|endoftext|>


<commit_before>// Define the options for Open

package fs

import (
	"fmt"
	"net/http"
	"strconv"
	"strings"

	"github.com/ncw/rclone/fs/hash"
	"github.com/pkg/errors"
)

// OpenOption is an interface describing options for Open
type OpenOption interface {
	fmt.Stringer

	// Header returns the option as an HTTP header
	Header() (key string, value string)

	// Mandatory returns whether this option can be ignored or not
	Mandatory() bool
}

// RangeOption defines an HTTP Range option with start and end.  If
// either start or end are < 0 then they will be omitted.
//
// End may be bigger than the Size of the object in which case it will
// be capped to the size of the object.
//
// Note that the End is inclusive, so to fetch 100 bytes you would use
// RangeOption{Start: 0, End: 99}
//
// A RangeOption implements a single byte-range-spec from
// https://tools.ietf.org/html/rfc7233#section-2.1
type RangeOption struct {
	Start int64
	End   int64
}

// Header formats the option as an http header
func (o *RangeOption) Header() (key string, value string) {
	key = "Range"
	value = "bytes="
	if o.Start >= 0 {
		value += strconv.FormatInt(o.Start, 10)

	}
	value += "-"
	if o.End >= 0 {
		value += strconv.FormatInt(o.End, 10)
	}
	return key, value
}

// ParseRangeOption parses a RangeOption from a Range: header.
// It only appects single ranges.
func ParseRangeOption(s string) (po *RangeOption, err error) {
	const preamble = "bytes="
	if !strings.HasPrefix(s, preamble) {
		return nil, errors.New("Range: header invalid: doesn't start with " + preamble)
	}
	s = s[len(preamble):]
	if strings.IndexRune(s, ',') >= 0 {
		return nil, errors.New("Range: header invalid: contains multiple ranges which isn't supported")
	}
	dash := strings.IndexRune(s, '-')
	if dash < 0 {
		return nil, errors.New("Range: header invalid: contains no '-'")
	}
	start, end := strings.TrimSpace(s[:dash]), strings.TrimSpace(s[dash+1:])
	o := RangeOption{Start: -1, End: -1}
	if start != "" {
		o.Start, err = strconv.ParseInt(start, 10, 64)
		if err != nil || o.Start < 0 {
			return nil, errors.New("Range: header invalid: bad start")
		}
	}
	if end != "" {
		o.End, err = strconv.ParseInt(end, 10, 64)
		if err != nil || o.End < 0 {
			return nil, errors.New("Range: header invalid: bad end")
		}
	}
	return &o, nil
}

// String formats the option into human readable form
func (o *RangeOption) String() string {
	return fmt.Sprintf("RangeOption(%d,%d)", o.Start, o.End)
}

// Mandatory returns whether the option must be parsed or can be ignored
func (o *RangeOption) Mandatory() bool {
	return false
}

// Decode interprets the RangeOption into an offset and a limit
//
// The offset is the start of the stream and the limit is how many
// bytes should be read from it.  If the limit is -1 then the stream
// should be read to the end.
func (o *RangeOption) Decode(size int64) (offset, limit int64) {
	if o.Start >= 0 {
		offset = o.Start
		if o.End >= 0 {
			limit = o.End - o.Start + 1
		} else {
			limit = -1
		}
	} else {
		if o.End >= 0 {
			offset = size - o.End
		} else {
			offset = 0
		}
		limit = -1
	}
	return offset, limit
}

// FixRangeOption looks through the slice of options and adjusts any
// RangeOption~s found that request a fetch from the end into an
// absolute fetch using the size passed in.  Some remotes (eg
// Onedrive, Box) don't support range requests which index from the
// end.
func FixRangeOption(options []OpenOption, size int64) {
	for i := range options {
		option := options[i]
		if x, ok := option.(*RangeOption); ok {
			// If start is < 0 then fetch from the end
			if x.Start < 0 {
				x = &RangeOption{Start: size - x.End, End: -1}
				options[i] = x
			}
		}
	}
}

// SeekOption defines an HTTP Range option with start only.
type SeekOption struct {
	Offset int64
}

// Header formats the option as an http header
func (o *SeekOption) Header() (key string, value string) {
	key = "Range"
	value = fmt.Sprintf("bytes=%d-", o.Offset)
	return key, value
}

// String formats the option into human readable form
func (o *SeekOption) String() string {
	return fmt.Sprintf("SeekOption(%d)", o.Offset)
}

// Mandatory returns whether the option must be parsed or can be ignored
func (o *SeekOption) Mandatory() bool {
	return true
}

// HTTPOption defines a general purpose HTTP option
type HTTPOption struct {
	Key   string
	Value string
}

// Header formats the option as an http header
func (o *HTTPOption) Header() (key string, value string) {
	return o.Key, o.Value
}

// String formats the option into human readable form
func (o *HTTPOption) String() string {
	return fmt.Sprintf("HTTPOption(%q,%q)", o.Key, o.Value)
}

// Mandatory returns whether the option must be parsed or can be ignored
func (o *HTTPOption) Mandatory() bool {
	return false
}

// HashesOption defines an option used to tell the local fs to limit
// the number of hashes it calculates.
type HashesOption struct {
	Hashes hash.Set
}

// Header formats the option as an http header
func (o *HashesOption) Header() (key string, value string) {
	return "", ""
}

// String formats the option into human readable form
func (o *HashesOption) String() string {
	return fmt.Sprintf("HashesOption(%v)", o.Hashes)
}

// Mandatory returns whether the option must be parsed or can be ignored
func (o *HashesOption) Mandatory() bool {
	return false
}

// OpenOptionAddHeaders adds each header found in options to the
// headers map provided the key was non empty.
func OpenOptionAddHeaders(options []OpenOption, headers map[string]string) {
	for _, option := range options {
		key, value := option.Header()
		if key != "" && value != "" {
			headers[key] = value
		}
	}
}

// OpenOptionHeaders adds each header found in options to the
// headers map provided the key was non empty.
//
// It returns a nil map if options was empty
func OpenOptionHeaders(options []OpenOption) (headers map[string]string) {
	if len(options) == 0 {
		return nil
	}
	headers = make(map[string]string, len(options))
	OpenOptionAddHeaders(options, headers)
	return headers
}

// OpenOptionAddHTTPHeaders Sets each header found in options to the
// http.Header map provided the key was non empty.
func OpenOptionAddHTTPHeaders(headers http.Header, options []OpenOption) {
	for _, option := range options {
		key, value := option.Header()
		if key != "" && value != "" {
			headers.Set(key, value)
		}
	}
}

// check interface
var (
	_ OpenOption = (*RangeOption)(nil)
	_ OpenOption = (*SeekOption)(nil)
	_ OpenOption = (*HTTPOption)(nil)
)
<commit_msg>fs: Make RangeOption mandatory #1825<commit_after>// Define the options for Open

package fs

import (
	"fmt"
	"net/http"
	"strconv"
	"strings"

	"github.com/ncw/rclone/fs/hash"
	"github.com/pkg/errors"
)

// OpenOption is an interface describing options for Open
type OpenOption interface {
	fmt.Stringer

	// Header returns the option as an HTTP header
	Header() (key string, value string)

	// Mandatory returns whether this option can be ignored or not
	Mandatory() bool
}

// RangeOption defines an HTTP Range option with start and end.  If
// either start or end are < 0 then they will be omitted.
//
// End may be bigger than the Size of the object in which case it will
// be capped to the size of the object.
//
// Note that the End is inclusive, so to fetch 100 bytes you would use
// RangeOption{Start: 0, End: 99}
//
// If Start is specified but End is not then it will fetch from Start
// to the end of the file.
//
// If End is specified, but Start is not then it will fetch the last
// End bytes.
//
// Examples:
//
//     RangeOption{Start: 0, End: 99} - fetch the first 100 bytes
//     RangeOption{Start: 100, End: 199} - fetch the second 100 bytes
//     RangeOption{Start: 100} - fetch bytes from offset 100 to the end
//     RangeOption{End: 100} - fetch the last 100 bytes
//
// A RangeOption implements a single byte-range-spec from
// https://tools.ietf.org/html/rfc7233#section-2.1
type RangeOption struct {
	Start int64
	End   int64
}

// Header formats the option as an http header
func (o *RangeOption) Header() (key string, value string) {
	key = "Range"
	value = "bytes="
	if o.Start >= 0 {
		value += strconv.FormatInt(o.Start, 10)

	}
	value += "-"
	if o.End >= 0 {
		value += strconv.FormatInt(o.End, 10)
	}
	return key, value
}

// ParseRangeOption parses a RangeOption from a Range: header.
// It only appects single ranges.
func ParseRangeOption(s string) (po *RangeOption, err error) {
	const preamble = "bytes="
	if !strings.HasPrefix(s, preamble) {
		return nil, errors.New("Range: header invalid: doesn't start with " + preamble)
	}
	s = s[len(preamble):]
	if strings.IndexRune(s, ',') >= 0 {
		return nil, errors.New("Range: header invalid: contains multiple ranges which isn't supported")
	}
	dash := strings.IndexRune(s, '-')
	if dash < 0 {
		return nil, errors.New("Range: header invalid: contains no '-'")
	}
	start, end := strings.TrimSpace(s[:dash]), strings.TrimSpace(s[dash+1:])
	o := RangeOption{Start: -1, End: -1}
	if start != "" {
		o.Start, err = strconv.ParseInt(start, 10, 64)
		if err != nil || o.Start < 0 {
			return nil, errors.New("Range: header invalid: bad start")
		}
	}
	if end != "" {
		o.End, err = strconv.ParseInt(end, 10, 64)
		if err != nil || o.End < 0 {
			return nil, errors.New("Range: header invalid: bad end")
		}
	}
	return &o, nil
}

// String formats the option into human readable form
func (o *RangeOption) String() string {
	return fmt.Sprintf("RangeOption(%d,%d)", o.Start, o.End)
}

// Mandatory returns whether the option must be parsed or can be ignored
func (o *RangeOption) Mandatory() bool {
	return true
}

// Decode interprets the RangeOption into an offset and a limit
//
// The offset is the start of the stream and the limit is how many
// bytes should be read from it.  If the limit is -1 then the stream
// should be read to the end.
func (o *RangeOption) Decode(size int64) (offset, limit int64) {
	if o.Start >= 0 {
		offset = o.Start
		if o.End >= 0 {
			limit = o.End - o.Start + 1
		} else {
			limit = -1
		}
	} else {
		if o.End >= 0 {
			offset = size - o.End
		} else {
			offset = 0
		}
		limit = -1
	}
	return offset, limit
}

// FixRangeOption looks through the slice of options and adjusts any
// RangeOption~s found that request a fetch from the end into an
// absolute fetch using the size passed in.  Some remotes (eg
// Onedrive, Box) don't support range requests which index from the
// end.
func FixRangeOption(options []OpenOption, size int64) {
	for i := range options {
		option := options[i]
		if x, ok := option.(*RangeOption); ok {
			// If start is < 0 then fetch from the end
			if x.Start < 0 {
				x = &RangeOption{Start: size - x.End, End: -1}
				options[i] = x
			}
		}
	}
}

// SeekOption defines an HTTP Range option with start only.
type SeekOption struct {
	Offset int64
}

// Header formats the option as an http header
func (o *SeekOption) Header() (key string, value string) {
	key = "Range"
	value = fmt.Sprintf("bytes=%d-", o.Offset)
	return key, value
}

// String formats the option into human readable form
func (o *SeekOption) String() string {
	return fmt.Sprintf("SeekOption(%d)", o.Offset)
}

// Mandatory returns whether the option must be parsed or can be ignored
func (o *SeekOption) Mandatory() bool {
	return true
}

// HTTPOption defines a general purpose HTTP option
type HTTPOption struct {
	Key   string
	Value string
}

// Header formats the option as an http header
func (o *HTTPOption) Header() (key string, value string) {
	return o.Key, o.Value
}

// String formats the option into human readable form
func (o *HTTPOption) String() string {
	return fmt.Sprintf("HTTPOption(%q,%q)", o.Key, o.Value)
}

// Mandatory returns whether the option must be parsed or can be ignored
func (o *HTTPOption) Mandatory() bool {
	return false
}

// HashesOption defines an option used to tell the local fs to limit
// the number of hashes it calculates.
type HashesOption struct {
	Hashes hash.Set
}

// Header formats the option as an http header
func (o *HashesOption) Header() (key string, value string) {
	return "", ""
}

// String formats the option into human readable form
func (o *HashesOption) String() string {
	return fmt.Sprintf("HashesOption(%v)", o.Hashes)
}

// Mandatory returns whether the option must be parsed or can be ignored
func (o *HashesOption) Mandatory() bool {
	return false
}

// OpenOptionAddHeaders adds each header found in options to the
// headers map provided the key was non empty.
func OpenOptionAddHeaders(options []OpenOption, headers map[string]string) {
	for _, option := range options {
		key, value := option.Header()
		if key != "" && value != "" {
			headers[key] = value
		}
	}
}

// OpenOptionHeaders adds each header found in options to the
// headers map provided the key was non empty.
//
// It returns a nil map if options was empty
func OpenOptionHeaders(options []OpenOption) (headers map[string]string) {
	if len(options) == 0 {
		return nil
	}
	headers = make(map[string]string, len(options))
	OpenOptionAddHeaders(options, headers)
	return headers
}

// OpenOptionAddHTTPHeaders Sets each header found in options to the
// http.Header map provided the key was non empty.
func OpenOptionAddHTTPHeaders(headers http.Header, options []OpenOption) {
	for _, option := range options {
		key, value := option.Header()
		if key != "" && value != "" {
			headers.Set(key, value)
		}
	}
}

// check interface
var (
	_ OpenOption = (*RangeOption)(nil)
	_ OpenOption = (*SeekOption)(nil)
	_ OpenOption = (*HTTPOption)(nil)
)
<|endoftext|>


<commit_before>package oskite

import (
	"fmt"
	"koding/virt"
	"time"

	"gopkg.in/fatih/set.v0"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

type VmCollection map[bson.ObjectId]*virt.VM

var blacklist = set.New()

// mongodbVMs returns a map of VMs that are bind to the given
// serviceUniquename/hostkite in mongodb
func mongodbVMs(serviceUniquename string) (VmCollection, error) {
	vms := make([]*virt.VM, 0)

	query := func(c *mgo.Collection) error {
		return c.Find(bson.M{"hostKite": serviceUniquename}).All(&vms)
	}

	if err := mongodbConn.Run("jVMs", query); err != nil {
		return nil, fmt.Errorf("allVMs fetching err: %s", err.Error())
	}

	vmsMap := make(map[bson.ObjectId]*virt.VM, len(vms))

	for _, vm := range vms {
		vmsMap[vm.Id] = vm
	}

	return vmsMap, nil
}

// updateStates updates the state field of the given ids to the given state argument.
func updateStates(ids []bson.ObjectId, state string) error {
	if len(ids) == 0 {
		return nil // no need to update
	}

	log.Info("Updating %s vms to the state %s", len(ids), state)
	return mongodbConn.Run("jVMs", func(c *mgo.Collection) error {
		_, err := c.UpdateAll(bson.M{"_id": bson.M{"$in": ids}}, bson.M{"$set": bson.M{"state": state}})
		return err
	})
}

// vmUpdater updates the states of current available VMs on the host machine.
func (o *Oskite) vmUpdater() {
	vms := make([]virt.VM, 0)

	query := func(c *mgo.Collection) error {
		vm := virt.VM{}

		iter := c.Find(bson.M{"hostKite": o.ServiceUniquename}).Batch(50).Iter()
		for iter.Next(&vm) {
			vms = append(vms, vm)

			if !blacklist.Has(vm.Id.Hex()) {
				o.startAlwaysOn(vm)
				continue
			}
		}

		return iter.Close()
	}

	for _ = range time.Tick(time.Second * 30) {
		// start alwaysOn Vms
		if err := mongodbConn.Run("jVMs", query); err != nil {
			log.Error("allVMs fetching err: %s", err.Error())
		}

		// batch update the states now
		currentStates := make(map[bson.ObjectId]string, 0)

		for _, vm := range vms {
			state := virt.GetVMState(vm.Id)
			if state == "" {
				state = "UNKNOWN"
			}

			// do not update if it's the same state
			if vm.State == state {
				continue
			}

			currentStates[vm.Id] = state
		}

		filter := func(desiredState string) []bson.ObjectId {
			ids := make([]bson.ObjectId, 0)

			for id, state := range currentStates {
				if state == desiredState {
					ids = append(ids, id)
				}
			}

			return ids
		}

		if err := updateStates(filter("RUNNING"), "RUNNING"); err != nil {
			log.Error("Updating RUNNING vms %v", err)
		}

		if err := updateStates(filter("STOPPED"), "STOPPED"); err != nil {
			log.Error("Updating STOPPED vms %v", err)
		}

		currentStates = nil // garbage collection

		// re initialize for next iteration
		vms = make([]virt.VM, 0)
	}

}

func unprepareInQueue(vm *virt.VM) {
	prepareQueue <- &QueueJob{
		msg: fmt.Sprintf("Vm is stopped, unpreparing it: %s", vm.HostnameAlias),
		f: func() error {
			info := getInfo(vm)
			info.mutex.Lock()
			defer info.mutex.Unlock()

			return unprepareProgress(nil, vm, false)
		},
	}
}

// startAlwaysOn  starts a vm if it's alwaysOn and not pinned to the current
// hostname
func (o *Oskite) startAlwaysOn(vm virt.VM) {
	if !vm.AlwaysOn {
		return
	}

	// means this vm is intended to be start on another kontainer machine
	if vm.PinnedToHost != "" && vm.PinnedToHost != o.ServiceUniquename {
		return
	}

	go func() {
		if vm.State != "RUNNING" {
			log.Info("alwaysOn is starting [%s - %v]", vm.HostnameAlias, vm.Id)
			err := o.startSingleVM(vm, nil)
			if err != nil {
				log.Error("alwaysOn vm %s couldn't be started. err: %v", vm.HostnameAlias, err)
			} else {
				log.Info("alwaysOn vm started successfull [%s - %v]", vm.HostnameAlias, vm.Id)
			}
		}
	}()
}
<commit_msg>Type error<commit_after>package oskite

import (
	"fmt"
	"koding/virt"
	"time"

	"gopkg.in/fatih/set.v0"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

type VmCollection map[bson.ObjectId]*virt.VM

var blacklist = set.New()

// mongodbVMs returns a map of VMs that are bind to the given
// serviceUniquename/hostkite in mongodb
func mongodbVMs(serviceUniquename string) (VmCollection, error) {
	vms := make([]*virt.VM, 0)

	query := func(c *mgo.Collection) error {
		return c.Find(bson.M{"hostKite": serviceUniquename}).All(&vms)
	}

	if err := mongodbConn.Run("jVMs", query); err != nil {
		return nil, fmt.Errorf("allVMs fetching err: %s", err.Error())
	}

	vmsMap := make(map[bson.ObjectId]*virt.VM, len(vms))

	for _, vm := range vms {
		vmsMap[vm.Id] = vm
	}

	return vmsMap, nil
}

// updateStates updates the state field of the given ids to the given state argument.
func updateStates(ids []bson.ObjectId, state string) error {
	if len(ids) == 0 {
		return nil // no need to update
	}

	log.Info("Updating %d vms to the state %s", len(ids), state)
	return mongodbConn.Run("jVMs", func(c *mgo.Collection) error {
		_, err := c.UpdateAll(bson.M{"_id": bson.M{"$in": ids}}, bson.M{"$set": bson.M{"state": state}})
		return err
	})
}

// vmUpdater updates the states of current available VMs on the host machine.
func (o *Oskite) vmUpdater() {
	vms := make([]virt.VM, 0)

	query := func(c *mgo.Collection) error {
		vm := virt.VM{}

		iter := c.Find(bson.M{"hostKite": o.ServiceUniquename}).Batch(50).Iter()
		for iter.Next(&vm) {
			vms = append(vms, vm)

			if !blacklist.Has(vm.Id.Hex()) {
				o.startAlwaysOn(vm)
				continue
			}
		}

		return iter.Close()
	}

	for _ = range time.Tick(time.Second * 30) {
		// start alwaysOn Vms
		if err := mongodbConn.Run("jVMs", query); err != nil {
			log.Error("allVMs fetching err: %s", err.Error())
		}

		// batch update the states now
		currentStates := make(map[bson.ObjectId]string, 0)

		for _, vm := range vms {
			state := vm.GetState()
			// do not update if it's the same state
			if vm.State == state {
				continue
			}

			currentStates[vm.Id] = state
		}

		filter := func(desiredState string) []bson.ObjectId {
			ids := make([]bson.ObjectId, 0)

			for id, state := range currentStates {
				if state == desiredState {
					ids = append(ids, id)
				}
			}

			return ids
		}

		if err := updateStates(filter("RUNNING"), "RUNNING"); err != nil {
			log.Error("Updating RUNNING vms %v", err)
		}

		if err := updateStates(filter("STOPPED"), "STOPPED"); err != nil {
			log.Error("Updating STOPPED vms %v", err)
		}

        if err := updateStates(filter("UNKNOWN"), "UNKNOWN"); err != nil {
			log.Error("Updating UNKNOWN vms %v", err)
		}

		currentStates = nil // garbage collection

		// re initialize for next iteration
		vms = make([]virt.VM, 0)
	}

}

func unprepareInQueue(vm *virt.VM) {
	prepareQueue <- &QueueJob{
		msg: fmt.Sprintf("Vm is stopped, unpreparing it: %s", vm.HostnameAlias),
		f: func() error {
			info := getInfo(vm)
			info.mutex.Lock()
			defer info.mutex.Unlock()

			return unprepareProgress(nil, vm, false)
		},
	}
}

// startAlwaysOn  starts a vm if it's alwaysOn and not pinned to the current
// hostname
func (o *Oskite) startAlwaysOn(vm virt.VM) {
	if !vm.AlwaysOn {
		return
	}

	// means this vm is intended to be start on another kontainer machine
	if vm.PinnedToHost != "" && vm.PinnedToHost != o.ServiceUniquename {
		return
	}

	go func() {
		if vm.State != "RUNNING" &&  vm.State != "UNKNOWN" {
			log.Info("alwaysOn is starting [%s - %v]", vm.HostnameAlias, vm.Id)
			err := o.startSingleVM(vm, nil)
			if err != nil {
				log.Error("alwaysOn vm %s couldn't be started. err: %v", vm.HostnameAlias, err)
			} else {
				log.Info("alwaysOn vm started successfull [%s - %v]", vm.HostnameAlias, vm.Id)
			}
		}
	}()
}
<|endoftext|>


<commit_before>package fsync

import (
	"bytes"
	"io/ioutil"
	"os"
	"testing"
	"time"
)

func TestSync(t *testing.T) {
	// create test directory and chdir to it
	dir, err := ioutil.TempDir(os.TempDir(), "fsync_test")
	check(err)
	check(os.Chdir(dir))

	// create test files and directories
	check(os.MkdirAll("src/a", 0755))
	check(ioutil.WriteFile("src/a/b", []byte("file b"), 0644))
	check(ioutil.WriteFile("src/c", []byte("file c"), 0644))
	// set times in the past to make sure times are synced, not accidentally
	// the same
	tt := time.Now().Add(-1 * time.Hour)
	check(os.Chtimes("src/a", tt, tt))
	check(os.Chtimes("src/a/b", tt, tt))
	check(os.Chtimes("src/c", tt, tt))

	// create Syncer
	s := NewSyncer()

	// sync
	check(s.SyncTo("dst", "src/a", "src/c"))

	// check results
	testDirContents("dst", 2, t)
	testDirContents("dst/a", 1, t)
	testFile("dst/a/b", []byte("file b"), t)
	testFile("dst/c", []byte("file c"), t)
	testPerms("dst/a", getPerms("src/a"), t)
	testPerms("dst/a/b", getPerms("src/a/b"), t)
	testPerms("dst/c", getPerms("src/c"), t)
	testModTime("dst", getModTime("src"), t)
	testModTime("dst/a", getModTime("src/a"), t)
	testModTime("dst/a/b", getModTime("src/a/b"), t)
	testModTime("dst/c", getModTime("src/c"), t)

	// modify src
	check(ioutil.WriteFile("src/a/b", []byte("file b changed"), 0644))
	check(os.Chmod("src/a", 0775))

	// sync
	check(s.Sync("dst", "src"))

	// check results
	testFile("dst/a/b", []byte("file b changed"), t)
	testPerms("dst/a", getPerms("src/a"), t)
	testModTime("dst/a", getModTime("src/a"), t)
	testModTime("dst/a/b", getModTime("src/a/b"), t)
	testModTime("dst/c", getModTime("src/c"), t)

	// remove c
	check(os.Remove("src/c"))

	// sync
	check(s.Sync("dst", "src"))

	// check results; c should still exist
	testDirContents("dst", 2, t)
	testExistence("dst/c", true, t)

	// sync
	s.Delete = true
	check(s.Sync("dst", "src"))

	// check results; c should no longer exist
	testDirContents("dst", 1, t)
	testExistence("dst/c", false, t)

	s.Delete = false
	if err = s.Sync("dst", "src/a/b"); err == nil {
		t.Errorf("expecting ErrFileOverDir, got nothing.\n")
	} else if err != nil && err != ErrFileOverDir {
		panic(err)
	}
}

func testFile(name string, b []byte, t *testing.T) {
	testExistence(name, true, t)
	c, err := ioutil.ReadFile(name)
	check(err)
	if !bytes.Equal(b, c) {
		t.Errorf("content of file \"%s\" is:\n%s\nexpected:\n%s\n",
			name, c, b)
	}
}

func testExistence(name string, e bool, t *testing.T) {
	_, err := os.Stat(name)
	if os.IsNotExist(err) {
		if e {
			t.Errorf("file \"%s\" does not exist.\n", name)
		}
	} else if err != nil {
		panic(err)
	} else {
		if !e {
			t.Errorf("file \"%s\" exists.\n", name)
		}
	}
}

func testDirContents(name string, count int, t *testing.T) {
	files, err := ioutil.ReadDir(name)
	check(err)
	if len(files) != count {
		t.Errorf("directory \"%s\" has %d children, shoud have %d.\n",
			name, len(files), count)
	}
}

func testPerms(name string, p os.FileMode, t *testing.T) {
	info, err := os.Stat(name)
	check(err)
	if info.Mode().Perm() != p {
		t.Errorf("permissions for \"%s\" is %v, should be %v.\n",
			name, info.Mode().Perm(), p)
	}
}

func testModTime(name string, m time.Time, t *testing.T) {
	info, err := os.Stat(name)
	check(err)
	if !info.ModTime().Equal(m) {
		t.Errorf("modification time for \"%s\" is %v, should be %v.\n",
			name, info.ModTime(), m)
	}
}

func getModTime(name string) time.Time {
	info, err := os.Stat(name)
	check(err)
	return info.ModTime()
}

func getPerms(name string) os.FileMode {
	info, err := os.Stat(name)
	check(err)
	return info.Mode().Perm()
}
<commit_msg>Test stats on the "dst" after it have been synced<commit_after>package fsync

import (
	"bytes"
	"io/ioutil"
	"os"
	"testing"
	"time"
)

func TestSync(t *testing.T) {
	// create test directory and chdir to it
	dir, err := ioutil.TempDir(os.TempDir(), "fsync_test")
	check(err)
	check(os.Chdir(dir))

	// create test files and directories
	check(os.MkdirAll("src/a", 0755))
	check(ioutil.WriteFile("src/a/b", []byte("file b"), 0644))
	check(ioutil.WriteFile("src/c", []byte("file c"), 0644))
	// set times in the past to make sure times are synced, not accidentally
	// the same
	tt := time.Now().Add(-1 * time.Hour)
	check(os.Chtimes("src/a/b", tt, tt))
	check(os.Chtimes("src/a", tt, tt))
	check(os.Chtimes("src/c", tt, tt))
	check(os.Chtimes("src", tt, tt))

	// create Syncer
	s := NewSyncer()

	// sync
	check(s.SyncTo("dst", "src/a", "src/c"))

	// check results
	testDirContents("dst", 2, t)
	testDirContents("dst/a", 1, t)
	testFile("dst/a/b", []byte("file b"), t)
	testFile("dst/c", []byte("file c"), t)
	testPerms("dst/a", getPerms("src/a"), t)
	testPerms("dst/a/b", getPerms("src/a/b"), t)
	testPerms("dst/c", getPerms("src/c"), t)
	testModTime("dst/a", getModTime("src/a"), t)
	testModTime("dst/a/b", getModTime("src/a/b"), t)
	testModTime("dst/c", getModTime("src/c"), t)

	// sync the parent directory too
	check(s.Sync("dst", "src"))

	// check the results
	testPerms("dst", getPerms("src"), t)
	testModTime("dst", getModTime("src"), t)

	// modify src
	check(ioutil.WriteFile("src/a/b", []byte("file b changed"), 0644))
	check(os.Chmod("src/a", 0775))

	// sync
	check(s.Sync("dst", "src"))

	// check results
	testFile("dst/a/b", []byte("file b changed"), t)
	testPerms("dst/a", getPerms("src/a"), t)
	testModTime("dst", getModTime("src"), t)
	testModTime("dst/a", getModTime("src/a"), t)
	testModTime("dst/a/b", getModTime("src/a/b"), t)
	testModTime("dst/c", getModTime("src/c"), t)

	// remove c
	check(os.Remove("src/c"))

	// sync
	check(s.Sync("dst", "src"))

	// check results; c should still exist
	testDirContents("dst", 2, t)
	testExistence("dst/c", true, t)

	// sync
	s.Delete = true
	check(s.Sync("dst", "src"))

	// check results; c should no longer exist
	testDirContents("dst", 1, t)
	testExistence("dst/c", false, t)

	s.Delete = false
	if err = s.Sync("dst", "src/a/b"); err == nil {
		t.Errorf("expecting ErrFileOverDir, got nothing.\n")
	} else if err != nil && err != ErrFileOverDir {
		panic(err)
	}
}

func testFile(name string, b []byte, t *testing.T) {
	testExistence(name, true, t)
	c, err := ioutil.ReadFile(name)
	check(err)
	if !bytes.Equal(b, c) {
		t.Errorf("content of file \"%s\" is:\n%s\nexpected:\n%s\n",
			name, c, b)
	}
}

func testExistence(name string, e bool, t *testing.T) {
	_, err := os.Stat(name)
	if os.IsNotExist(err) {
		if e {
			t.Errorf("file \"%s\" does not exist.\n", name)
		}
	} else if err != nil {
		panic(err)
	} else {
		if !e {
			t.Errorf("file \"%s\" exists.\n", name)
		}
	}
}

func testDirContents(name string, count int, t *testing.T) {
	files, err := ioutil.ReadDir(name)
	check(err)
	if len(files) != count {
		t.Errorf("directory \"%s\" has %d children, shoud have %d.\n",
			name, len(files), count)
	}
}

func testPerms(name string, p os.FileMode, t *testing.T) {
	info, err := os.Stat(name)
	check(err)
	if info.Mode().Perm() != p {
		t.Errorf("permissions for \"%s\" is %v, should be %v.\n",
			name, info.Mode().Perm(), p)
	}
}

func testModTime(name string, m time.Time, t *testing.T) {
	info, err := os.Stat(name)
	check(err)
	if !info.ModTime().Equal(m) {
		t.Errorf("modification time for \"%s\" is %v, should be %v.\n",
			name, info.ModTime(), m)
	}
}

func getModTime(name string) time.Time {
	info, err := os.Stat(name)
	check(err)
	return info.ModTime()
}

func getPerms(name string) os.FileMode {
	info, err := os.Stat(name)
	check(err)
	return info.Mode().Perm()
}
<|endoftext|>


<commit_before>package vindexes

import (
	"encoding/json"
	"fmt"
	"encoding/hex"
)

func init() {
	Register("lookup", NewLookup)
	Register("lookup_unique", NewLookupUnique)
}

/* LookupNonUnique defines a vindex that uses a lookup table and create a mapping between id and KeyspaceId.
 * The table is expected to define the id column as unique. It's
 * NonUnique and a Lookup.
 */
type LookupNonUnique struct {
	name string
	lkp  lookup
}

// NewLookup creates a LookupNonUnique vindex.
func NewLookup(name string, m map[string]string) (Vindex, error) {
	lookup := &LookupNonUnique{name: name}
	lookup.lkp.Init(m)
	return lookup, nil
}

// String returns the name of the vindex.
func (vindex *LookupNonUnique) String() string {
	return vindex.name
}

// Cost returns the cost of this vindex as 20.
func (vindex *LookupNonUnique) Cost() int {
	return 20
}

// Map returns the corresponding KeyspaceId values for the given ids.
func (vindex *LookupNonUnique) Map(vcursor VCursor, ids []interface{}) ([][][]byte, error) {
	return vindex.lkp.MapLookup(vcursor, ids)
}

// Verify returns true if id maps to ksid.
func (vindex *LookupNonUnique) Verify(vcursor VCursor, id interface{}, ksid []byte) (bool, error) {
	return vindex.lkp.VerifyLookup(vcursor, id, ksid)
}

// Create reserves the id by inserting it into the vindex table.
func (vindex *LookupNonUnique) Create(vcursor VCursor, id interface{}, ksid []byte) error {
	return vindex.lkp.CreateLookup(vcursor, id, ksid)
}

// Delete deletes the entry from the vindex table.
func (vindex *LookupNonUnique) Delete(vcursor VCursor, ids []interface{}, ksid []byte) error {
	return vindex.lkp.DeleteLookup(vcursor, ids, ksid)
}

// MarshalJSON returns a JSON representation of LookupHash.
func (vindex *LookupNonUnique) MarshalJSON() ([]byte, error) {
	return json.Marshal(vindex.lkp)
}

/* LookupUnique defines a vindex that uses a lookup table.
 * The table is expected to define the id column as unique. It's
 * Unique and a Lookup.
 */
type LookupUnique struct {
	name string
	lkp  lookup
}

// NewLookupUnique creates a LookupHashUnique vindex.
func NewLookupUnique(name string, m map[string]string) (Vindex, error) {
	lu := &LookupUnique{name: name}
	lu.lkp.Init(m)
	return lu, nil
}

// String returns the name of the vindex.
func (vindex *LookupUnique) String() string {
	return vindex.name
}

// Cost returns the cost of this vindex as 10.
func (vindex *LookupUnique) Cost() int {
	return 10
}

// Map returns the corresponding KeyspaceId values for the given ids.
func (vindex *LookupUnique) Map(vcursor VCursor, ids []interface{}) ([][]byte, error) {
	return vindex.lkp.MapLookupUnique(vcursor, ids)
}

// Verify returns true if id maps to ksid.
func (vindex *LookupUnique) Verify(vcursor VCursor, id interface{}, ksid []byte) (bool, error) {
	return vindex.lkp.VerifyLookup(vcursor, id, ksid)
}

// Create reserves the id by inserting it into the vindex table.
func (vindex *LookupUnique) Create(vcursor VCursor, id interface{}, ksid []byte) error {
	return vindex.lkp.CreateLookup(vcursor, id, ksid)
}

// Delete deletes the entry from the vindex table.
func (vindex *LookupUnique) Delete(vcursor VCursor, ids []interface{}, ksid []byte) error {
	return vindex.lkp.DeleteLookup(vcursor, ids, ksid)
}

// MarshalJSON returns a JSON representation of LookupHashUnique.
func (vindex *LookupUnique) MarshalJSON() ([]byte, error) {
	return json.Marshal(vindex.lkp)
}

// MapLookupUnique is for a unique vindex.
func (lkp *lookup) MapLookupUnique(vcursor VCursor, ids []interface{}) ([][]byte, error) {
	out := make([][]byte, 0, len(ids))
	for _, id := range ids {
		result, err := vcursor.Execute(lkp.sel, map[string]interface{}{
			lkp.From: id,
		})
		if err != nil {
			return nil, fmt.Errorf("lookup.Map: %v", err)
		}
		if len(result.Rows) == 0 {
			out = append(out, []byte{})
			continue
		}
		if len(result.Rows) != 1 {
			return nil, fmt.Errorf("lookup.Map: unexpected multiple results from vindex %s: %v", lkp.Table, id)
		}
		source, err := getBytes(result.Rows[0][0].ToNative())
		if err != nil {
			return nil, err
		}
		data,err := hex.DecodeString(string(source))
		if err != nil {
			return nil, err
		}
		out = append(out, data)
	}
	return out, nil
}

// MapLookup is for a non-unique vindex.
func (lkp *lookup) MapLookup(vcursor VCursor, ids []interface{}) ([][][]byte, error) {
	out := make([][][]byte, 0, len(ids))
	for _, id := range ids {
		result, err := vcursor.Execute(lkp.sel, map[string]interface{}{
			lkp.From: id,
		})
		if err != nil {
			return nil, fmt.Errorf("lookup.Map: %v", err)
		}
		var ksids [][]byte
		for _, row := range result.Rows {
			source, err := getBytes(row[0].ToNative())
			if err != nil {
				return nil, err
			}
			data, err := hex.DecodeString(string(source))
			if err != nil {
				return nil, fmt.Errorf("lookup.Map: %v", err)
			}
			ksids = append(ksids, data)
		}
		out = append(out, ksids)
	}
	return out, nil
}

// Create creates an association between id and keyspaceid by inserting a row in the vindex table.
func (lkp *lookup) CreateLookup(vcursor VCursor, id interface{}, keyspaceid []byte) error {
	keyspace_id := hex.EncodeToString(keyspaceid)
	if _, err := vcursor.Execute(lkp.ins, map[string]interface{}{
		lkp.From: id,
		lkp.To:   keyspace_id,
	}); err != nil {
		return fmt.Errorf("lookup.Create: %v", err)
	}
	return nil
}

// Delete deletes the association between ids and keyspaceid.
func (lkp *lookup) DeleteLookup(vcursor VCursor, ids []interface{}, keyspaceid []byte) error {
	keyspace_id := hex.EncodeToString(keyspaceid)
	bindvars := map[string]interface{}{
		lkp.To: keyspace_id,
	}
	for _, id := range ids {
		bindvars[lkp.From] = id
		if _, err := vcursor.Execute(lkp.del, bindvars); err != nil {
			return fmt.Errorf("lookup.Delete: %v", err)
		}
	}
	return nil
}

// VerifyLookup returns true if id maps to ksid.
func (lkp *lookup) VerifyLookup(vcursor VCursor, id interface{}, keyspaceid []byte) (bool, error) {
	keyspace_id := hex.EncodeToString(keyspaceid)
	result, err := vcursor.Execute(lkp.ver, map[string]interface{}{
		lkp.From: id,
		lkp.To:   keyspace_id,
	})
	if err != nil {
		return false, fmt.Errorf("lookup.Verify: %v", err)
	}
	if len(result.Rows) == 0 {
		return false, nil
	}
	return true, nil
}
<commit_msg>Changing the Variables<commit_after>package vindexes

import (
	"encoding/hex"
	"encoding/json"
	"fmt"
)

func init() {
	Register("lookup", NewLookup)
	Register("lookup_unique", NewLookupUnique)
}

// LookupNonUnique defines a vindex that uses a lookup table and create a mapping between id and KeyspaceId.
// The table is expected to define the id column as unique. It's
// NonUnique and a Lookup.
type LookupNonUnique struct {
	name string
	lkp  lookup
}

// NewLookup creates a LookupNonUnique vindex.
func NewLookup(name string, m map[string]string) (Vindex, error) {
	lookup := &LookupNonUnique{name: name}
	lookup.lkp.Init(m)
	return lookup, nil
}

// String returns the name of the vindex.
func (vindex *LookupNonUnique) String() string {
	return vindex.name
}

// Cost returns the cost of this vindex as 20.
func (vindex *LookupNonUnique) Cost() int {
	return 20
}

// Map returns the corresponding KeyspaceId values for the given ids.
func (vindex *LookupNonUnique) Map(vcursor VCursor, ids []interface{}) ([][][]byte, error) {
	return vindex.lkp.MapLookup(vcursor, ids)
}

// Verify returns true if id maps to ksid.
func (vindex *LookupNonUnique) Verify(vcursor VCursor, id interface{}, ksid []byte) (bool, error) {
	return vindex.lkp.VerifyLookup(vcursor, id, ksid)
}

// Create reserves the id by inserting it into the vindex table.
func (vindex *LookupNonUnique) Create(vcursor VCursor, id interface{}, ksid []byte) error {
	return vindex.lkp.CreateLookup(vcursor, id, ksid)
}

// Delete deletes the entry from the vindex table.
func (vindex *LookupNonUnique) Delete(vcursor VCursor, ids []interface{}, ksid []byte) error {
	return vindex.lkp.DeleteLookup(vcursor, ids, ksid)
}

// MarshalJSON returns a JSON representation of LookupHash.
func (vindex *LookupNonUnique) MarshalJSON() ([]byte, error) {
	return json.Marshal(vindex.lkp)
}

// LookupUnique defines a vindex that uses a lookup table.
// The table is expected to define the id column as unique. It's
// Unique and a Lookup.
type LookupUnique struct {
	name string
	lkp  lookup
}

// NewLookupUnique creates a LookupHashUnique vindex.
func NewLookupUnique(name string, m map[string]string) (Vindex, error) {
	lu := &LookupUnique{name: name}
	lu.lkp.Init(m)
	return lu, nil
}

// String returns the name of the vindex.
func (vindex *LookupUnique) String() string {
	return vindex.name
}

// Cost returns the cost of this vindex as 10.
func (vindex *LookupUnique) Cost() int {
	return 10
}

// Map returns the corresponding KeyspaceId values for the given ids.
func (vindex *LookupUnique) Map(vcursor VCursor, ids []interface{}) ([][]byte, error) {
	return vindex.lkp.MapLookupUnique(vcursor, ids)
}

// Verify returns true if id maps to ksid.
func (vindex *LookupUnique) Verify(vcursor VCursor, id interface{}, ksid []byte) (bool, error) {
	return vindex.lkp.VerifyLookup(vcursor, id, ksid)
}

// Create reserves the id by inserting it into the vindex table.
func (vindex *LookupUnique) Create(vcursor VCursor, id interface{}, ksid []byte) error {
	return vindex.lkp.CreateLookup(vcursor, id, ksid)
}

// Delete deletes the entry from the vindex table.
func (vindex *LookupUnique) Delete(vcursor VCursor, ids []interface{}, ksid []byte) error {
	return vindex.lkp.DeleteLookup(vcursor, ids, ksid)
}

// MarshalJSON returns a JSON representation of LookupHashUnique.
func (vindex *LookupUnique) MarshalJSON() ([]byte, error) {
	return json.Marshal(vindex.lkp)
}

// MapLookupUnique is for a unique vindex.
func (lkp *lookup) MapLookupUnique(vcursor VCursor, ids []interface{}) ([][]byte, error) {
	out := make([][]byte, 0, len(ids))
	for _, id := range ids {
		result, err := vcursor.Execute(lkp.sel, map[string]interface{}{
			lkp.From: id,
		})
		if err != nil {
			return nil, fmt.Errorf("lookup.Map: %v", err)
		}
		if len(result.Rows) == 0 {
			out = append(out, []byte{})
			continue
		}
		if len(result.Rows) != 1 {
			return nil, fmt.Errorf("lookup.Map: unexpected multiple results from vindex %s: %v", lkp.Table, id)
		}
		source, err := getBytes(result.Rows[0][0].ToNative())
		if err != nil {
			return nil, err
		}
		data, err := hex.DecodeString(string(source))
		if err != nil {
			return nil, err
		}
		out = append(out, data)
	}
	return out, nil
}

// MapLookup is for a non-unique vindex.
func (lkp *lookup) MapLookup(vcursor VCursor, ids []interface{}) ([][][]byte, error) {
	out := make([][][]byte, 0, len(ids))
	for _, id := range ids {
		result, err := vcursor.Execute(lkp.sel, map[string]interface{}{
			lkp.From: id,
		})
		if err != nil {
			return nil, fmt.Errorf("lookup.Map: %v", err)
		}
		var ksids [][]byte
		for _, row := range result.Rows {
			source, err := getBytes(row[0].ToNative())
			if err != nil {
				return nil, err
			}
			data, err := hex.DecodeString(string(source))
			if err != nil {
				return nil, fmt.Errorf("lookup.Map: %v", err)
			}
			ksids = append(ksids, data)
		}
		out = append(out, ksids)
	}
	return out, nil
}

// Create creates an association between id and keyspaceid by inserting a row in the vindex table.
func (lkp *lookup) CreateLookup(vcursor VCursor, id interface{}, ksid []byte) error {
	keyspaceID := hex.EncodeToString(ksid)
	if _, err := vcursor.Execute(lkp.ins, map[string]interface{}{
		lkp.From: id,
		lkp.To:   keyspaceID,
	}); err != nil {
		return fmt.Errorf("lookup.Create: %v", err)
	}
	return nil
}

// Delete deletes the association between ids and keyspaceid.
func (lkp *lookup) DeleteLookup(vcursor VCursor, ids []interface{}, ksid []byte) error {
	keyspaceID := hex.EncodeToString(ksid)
	bindvars := map[string]interface{}{
		lkp.To: keyspaceID,
	}
	for _, id := range ids {
		bindvars[lkp.From] = id
		if _, err := vcursor.Execute(lkp.del, bindvars); err != nil {
			return fmt.Errorf("lookup.Delete: %v", err)
		}
	}
	return nil
}

// VerifyLookup returns true if id maps to ksid.
func (lkp *lookup) VerifyLookup(vcursor VCursor, id interface{}, ksid []byte) (bool, error) {
	keyspaceID := hex.EncodeToString(ksid)
	result, err := vcursor.Execute(lkp.ver, map[string]interface{}{
		lkp.From: id,
		lkp.To:   keyspaceID,
	})
	if err != nil {
		return false, fmt.Errorf("lookup.Verify: %v", err)
	}
	if len(result.Rows) == 0 {
		return false, nil
	}
	return true, nil
}
<|endoftext|>


<commit_before>package dry

import (
	"bytes"
	"io"
	"strings"
	"testing"
)

type MyString struct {
	str string
}

func (t MyString) String() string {
	return t.str
}

type MyError struct {
	str string
}

func (t MyError) Error() string {
	return t.str
}

func Test_BytesReader(t *testing.T) {
	expected := []byte("hello")
	testBytesReaderFn := func(input interface{}) {
		result := make([]byte, 5)
		returnedIoReader := BytesReader(input)
		n, _ := returnedIoReader.Read(result)
		if n != 5 {
			t.FailNow()
		}
		for i, _ := range result {
			if result[i] != expected[i] {
				t.FailNow()
			}
		}
		n, err := returnedIoReader.Read(result)
		if n != 0 || err != io.EOF {
			t.FailNow()
		}
	}

	testBytesReaderFn(strings.NewReader("hello"))

	bytesInput := []byte("hello")
	testBytesReaderFn(bytesInput)

	testBytesReaderFn("hello")

	myStr := MyString{"hello"}
	testBytesReaderFn(myStr)

	myErr := MyError{"hello"}
	testBytesReaderFn(myErr)
}

func testCompressDecompress(t *testing.T,
	compressFunc func([]byte) []byte,
	decompressFunc func([]byte) []byte) {
	testFn := func(testData []byte) {
		compressedData := compressFunc(testData)
		uncompressedData := decompressFunc(compressedData)
		if !bytes.Equal(testData, uncompressedData) {
			t.FailNow()
		}
	}

	go testFn([]byte("hello123"))
	go testFn([]byte("gopher456"))
	go testFn([]byte("dry789"))
}

func Test_BytesDeflateInflate(t *testing.T) {
	testCompressDecompress(t, BytesDeflate, BytesInflate)
}

func Test_BytesGzipUnGzip(t *testing.T) {
	testCompressDecompress(t, BytesGzip, BytesUnGzip)
}

func Test_BytesMap(t *testing.T) {
	upper := func(b byte) byte {
		return b - ('a' - 'A')
	}
	result := BytesMap(upper, []byte("hello"))
	correct := []byte("HELLO")
	if len(result) != len(correct) {
		t.Fail()
	}
	for i, _ := range result {
		if result[i] != correct[i] {
			t.Fail()
		}
	}
}

func Test_BytesFilter(t *testing.T) {
	azFunc := func(b byte) bool {
		return b >= 'A' && b <= 'Z'
	}
	result := BytesFilter(azFunc, []byte{1, 2, 3, 'A', 'f', 'R', 123})
	correct := []byte{'A', 'R'}
	if len(result) != len(correct) {
		t.Fail()
	}
	for i, _ := range result {
		if result[i] != correct[i] {
			t.Fail()
		}
	}
}
<commit_msg>Add test cases for some methods in bytes<commit_after>package dry

import (
	"bytes"
	"io"
	"strings"
	"testing"
)

type MyString struct {
	str string
}

func (t MyString) String() string {
	return t.str
}

type MyError struct {
	str string
}

func (t MyError) Error() string {
	return t.str
}

func Test_BytesReader(t *testing.T) {
	expected := []byte("hello")
	testBytesReaderFn := func(input interface{}) {
		result := make([]byte, 5)
		returnedIoReader := BytesReader(input)
		n, _ := returnedIoReader.Read(result)
		if n != 5 {
			t.FailNow()
		}
		for i, _ := range result {
			if result[i] != expected[i] {
				t.FailNow()
			}
		}
		n, err := returnedIoReader.Read(result)
		if n != 0 || err != io.EOF {
			t.FailNow()
		}
	}

	testBytesReaderFn(strings.NewReader("hello"))

	bytesInput := []byte("hello")
	testBytesReaderFn(bytesInput)

	testBytesReaderFn("hello")

	myStr := MyString{"hello"}
	testBytesReaderFn(myStr)

	myErr := MyError{"hello"}
	testBytesReaderFn(myErr)
}

func assertStringsEqual(t *testing.T, str1, str2 string) {
	if str1 != str2 {
		t.FailNow()
	}
}

func Test_BytesMD5(t *testing.T) {
	assertStringsEqual(
		t, "5d41402abc4b2a76b9719d911017c592",
		BytesMD5("hello"))
}

func Test_BytesEncodeBase64(t *testing.T) {
	assertStringsEqual(
		t, "aGVsbG8=",
		BytesEncodeBase64("hello"))
}

func Test_BytesDecodeBase64(t *testing.T) {
	assertStringsEqual(
		t, "hello",
		BytesDecodeBase64("aGVsbG8="))
}

func Test_BytesEncodeHex(t *testing.T) {
	assertStringsEqual(
		t, "68656c6c6f",
		BytesEncodeHex("hello"))
}

func Test_BytesDecodeHex(t *testing.T) {
	assertStringsEqual(
		t, "hello",
		BytesDecodeHex("68656C6C6F"))
}

func testCompressDecompress(t *testing.T,
	compressFunc func([]byte) []byte,
	decompressFunc func([]byte) []byte) {
	testFn := func(testData []byte) {
		compressedData := compressFunc(testData)
		uncompressedData := decompressFunc(compressedData)
		if !bytes.Equal(testData, uncompressedData) {
			t.FailNow()
		}
	}

	go testFn([]byte("hello123"))
	go testFn([]byte("gopher456"))
	go testFn([]byte("dry789"))
}

func Test_BytesDeflateInflate(t *testing.T) {
	testCompressDecompress(t, BytesDeflate, BytesInflate)
}

func Test_BytesGzipUnGzip(t *testing.T) {
	testCompressDecompress(t, BytesGzip, BytesUnGzip)
}

func Test_BytesMap(t *testing.T) {
	upper := func(b byte) byte {
		return b - ('a' - 'A')
	}
	result := BytesMap(upper, []byte("hello"))
	correct := []byte("HELLO")
	if len(result) != len(correct) {
		t.Fail()
	}
	for i, _ := range result {
		if result[i] != correct[i] {
			t.Fail()
		}
	}
}

func Test_BytesFilter(t *testing.T) {
	azFunc := func(b byte) bool {
		return b >= 'A' && b <= 'Z'
	}
	result := BytesFilter(azFunc, []byte{1, 2, 3, 'A', 'f', 'R', 123})
	correct := []byte{'A', 'R'}
	if len(result) != len(correct) {
		t.Fail()
	}
	for i, _ := range result {
		if result[i] != correct[i] {
			t.Fail()
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The Netstack Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package buffer provides the implementation of a buffer view.
package buffer

// View is a slice of a buffer, with convenience methods.
type View []byte

// NewView allocates a new buffer and returns an initialized view that covers
// the whole buffer.
func NewView(size int) View {
	return make(View, size)
}

// TrimFront removes the first "count" bytes from the visible section of the
// buffer.
func (v *View) TrimFront(count int) {
	*v = (*v)[count:]
}

// CapLength irreversibly reduces the length of the visible section of the
// buffer to the value specified.
func (v *View) CapLength(length int) {
	// We also set the slice cap because if we don't, one would be able to
	// expand the view back to include the region just excluded. We want to
	// prevent that to avoid potential data leak if we have uninitialized
	// data in excluded region.
	*v = (*v)[:length:length]
}

// ToVectorisedView transforms a View in a VectorisedView from an
// already-allocated slice of View.
func (v *View) ToVectorisedView(views [1]View) VectorisedView {
	views[0] = *v
	return NewVectorisedView(len(*v), views[:])
}

// VectorisedView is a vectorised version of View using non contigous memory.
// It supports all the convenience methods supported by View.
type VectorisedView struct {
	views []View
	size  int
}

// NewVectorisedView creates a new vectorised view from an already-allocated slice
// of View and sets its size.
func NewVectorisedView(size int, views []View) VectorisedView {
	return VectorisedView{views: views, size: size}
}

// TrimFront removes the first "count" bytes of the vectorised view.
func (vv *VectorisedView) TrimFront(count int) {
	for count > 0 && len(vv.views) > 0 {
		if count < len(vv.views[0]) {
			vv.size -= count
			vv.views[0].TrimFront(count)
			return
		}
		count -= len(vv.views[0])
		vv.RemoveFirst()
	}
}

// CapLength irreversibly reduces the length of the vectorised view.
func (vv *VectorisedView) CapLength(length int) {
	if length < 0 {
		length = 0
	}
	if vv.size < length {
		return
	}
	vv.size = length
	for i := range vv.views {
		v := &vv.views[i]
		if len(*v) >= length {
			if length == 0 {
				vv.views = vv.views[:i]
			} else {
				v.CapLength(length)
				vv.views = vv.views[:i+1]
			}
			return
		}
		length -= len(*v)
	}
}

// Clone returns a clone of this VectorisedView.
// If the buffer argument is large enough to contain all the Views of this VectorisedView,
// the method will avoid allocations and use the buffer to store the Views of the clone.
func (vv *VectorisedView) Clone(buffer []View) VectorisedView {
	var views []View
	if len(buffer) >= len(vv.views) {
		views = buffer[:len(vv.views)]
	} else {
		views = make([]View, len(vv.views))
	}
	for i, v := range vv.views {
		views[i] = v
	}
	return VectorisedView{views: views, size: vv.size}
}

// First returns the first view of the vectorised view.
// It panics if the vectorised view is empty.
func (vv *VectorisedView) First() View {
	if len(vv.views) == 0 {
		panic("vview is empty")
	}
	return vv.views[0]
}

// RemoveFirst removes the first view of the vectorised view.
func (vv *VectorisedView) RemoveFirst() {
	if len(vv.views) == 0 {
		return
	}
	vv.size -= len(vv.views[0])
	vv.views = vv.views[1:]
}

// SetSize unsafely sets the size of the VectorisedView.
func (vv *VectorisedView) SetSize(size int) {
	vv.size = size
}

// SetViews unsafely sets the views of the VectorisedView.
func (vv *VectorisedView) SetViews(views []View) {
	vv.views = views
}

// Size returns the size in bytes of the entire content stored in the vectorised view.
func (vv *VectorisedView) Size() int {
	return vv.size
}

// ToView returns the a single view containing the content of the vectorised view.
func (vv *VectorisedView) ToView() View {
	v := make([]byte, vv.size)
	u := v
	for i := range vv.views {
		n := copy(u, vv.views[i])
		u = u[n:]
	}
	return v
}

// Views returns the slice containing the all views.
func (vv *VectorisedView) Views() []View {
	return vv.views
}

// copy returns a deep-copy of the vectorised view.
// It is an expensive method that should be used only in tests.
func (vv *VectorisedView) copy() *VectorisedView {
	uu := &VectorisedView{
		views: make([]View, len(vv.views)),
		size:  vv.size,
	}
	for i, v := range vv.views {
		uu.views[i] = make(View, len(v))
		copy(uu.views[i], v)
	}
	return uu
}
<commit_msg>buffer: add ByteSlice method<commit_after>// Copyright 2016 The Netstack Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package buffer provides the implementation of a buffer view.
package buffer

// View is a slice of a buffer, with convenience methods.
type View []byte

// NewView allocates a new buffer and returns an initialized view that covers
// the whole buffer.
func NewView(size int) View {
	return make(View, size)
}

// TrimFront removes the first "count" bytes from the visible section of the
// buffer.
func (v *View) TrimFront(count int) {
	*v = (*v)[count:]
}

// CapLength irreversibly reduces the length of the visible section of the
// buffer to the value specified.
func (v *View) CapLength(length int) {
	// We also set the slice cap because if we don't, one would be able to
	// expand the view back to include the region just excluded. We want to
	// prevent that to avoid potential data leak if we have uninitialized
	// data in excluded region.
	*v = (*v)[:length:length]
}

// ToVectorisedView transforms a View in a VectorisedView from an
// already-allocated slice of View.
func (v *View) ToVectorisedView(views [1]View) VectorisedView {
	views[0] = *v
	return NewVectorisedView(len(*v), views[:])
}

// VectorisedView is a vectorised version of View using non contigous memory.
// It supports all the convenience methods supported by View.
type VectorisedView struct {
	views []View
	size  int
}

// NewVectorisedView creates a new vectorised view from an already-allocated slice
// of View and sets its size.
func NewVectorisedView(size int, views []View) VectorisedView {
	return VectorisedView{views: views, size: size}
}

// TrimFront removes the first "count" bytes of the vectorised view.
func (vv *VectorisedView) TrimFront(count int) {
	for count > 0 && len(vv.views) > 0 {
		if count < len(vv.views[0]) {
			vv.size -= count
			vv.views[0].TrimFront(count)
			return
		}
		count -= len(vv.views[0])
		vv.RemoveFirst()
	}
}

// CapLength irreversibly reduces the length of the vectorised view.
func (vv *VectorisedView) CapLength(length int) {
	if length < 0 {
		length = 0
	}
	if vv.size < length {
		return
	}
	vv.size = length
	for i := range vv.views {
		v := &vv.views[i]
		if len(*v) >= length {
			if length == 0 {
				vv.views = vv.views[:i]
			} else {
				v.CapLength(length)
				vv.views = vv.views[:i+1]
			}
			return
		}
		length -= len(*v)
	}
}

// Clone returns a clone of this VectorisedView.
// If the buffer argument is large enough to contain all the Views of this VectorisedView,
// the method will avoid allocations and use the buffer to store the Views of the clone.
func (vv *VectorisedView) Clone(buffer []View) VectorisedView {
	var views []View
	if len(buffer) >= len(vv.views) {
		views = buffer[:len(vv.views)]
	} else {
		views = make([]View, len(vv.views))
	}
	for i, v := range vv.views {
		views[i] = v
	}
	return VectorisedView{views: views, size: vv.size}
}

// First returns the first view of the vectorised view.
// It panics if the vectorised view is empty.
func (vv *VectorisedView) First() View {
	if len(vv.views) == 0 {
		panic("vview is empty")
	}
	return vv.views[0]
}

// RemoveFirst removes the first view of the vectorised view.
func (vv *VectorisedView) RemoveFirst() {
	if len(vv.views) == 0 {
		return
	}
	vv.size -= len(vv.views[0])
	vv.views = vv.views[1:]
}

// SetSize unsafely sets the size of the VectorisedView.
func (vv *VectorisedView) SetSize(size int) {
	vv.size = size
}

// SetViews unsafely sets the views of the VectorisedView.
func (vv *VectorisedView) SetViews(views []View) {
	vv.views = views
}

// Size returns the size in bytes of the entire content stored in the vectorised view.
func (vv *VectorisedView) Size() int {
	return vv.size
}

// ToView returns the a single view containing the content of the vectorised view.
func (vv *VectorisedView) ToView() View {
	v := make([]byte, vv.size)
	u := v
	for i := range vv.views {
		n := copy(u, vv.views[i])
		u = u[n:]
	}
	return v
}

// Views returns the slice containing the all views.
func (vv *VectorisedView) Views() []View {
	return vv.views
}

// ByteSlice returns a slice containing the all views as a []byte.
func (vv *VectorisedView) ByteSlice() [][]byte {
	s := make([][]byte, len(vv.views))
	for i := range vv.views {
		s[i] = []byte(vv.views[i])
	}
	return s
}

// copy returns a deep-copy of the vectorised view.
// It is an expensive method that should be used only in tests.
func (vv *VectorisedView) copy() *VectorisedView {
	uu := &VectorisedView{
		views: make([]View, len(vv.views)),
		size:  vv.size,
	}
	for i, v := range vv.views {
		uu.views[i] = make(View, len(v))
		copy(uu.views[i], v)
	}
	return uu
}
<|endoftext|>


<commit_before>package cache2go

import (
	"testing"
	"time"
)

type myStruct struct {
	XEntry
	data string
}

func TestCache(t *testing.T) {
	a := &myStruct{data: "mama are mere"}
	a.XCache("mama", 1*time.Second, a, nil)
	b, err := GetXCached("mama")
	if err != nil || b == nil || b != a {
		t.Error("Error retrieving data from cache", err)
	}
}

func TestCacheExpire(t *testing.T) {
	a := &myStruct{data: "mama are mere"}
	a.XCache("mama", 1*time.Second, a, nil)
	b, err := GetXCached("mama")
	if err != nil || b == nil || b.(*myStruct).data != "mama are mere" {
		t.Error("Error retrieving data from cache", err)
	}
	time.Sleep(1001 * time.Millisecond)
	b, err = GetXCached("mama")
	if err == nil || b != nil {
		t.Error("Error expiring data")
	}
}

func TestCacheKeepAlive(t *testing.T) {
	a := &myStruct{data: "mama are mere"}
	a.XCache("mama", 1*time.Second, a, nil)
	a = &myStruct{data: "mama are mere2"}
	a.XCache("mama2", 2*time.Second, a, nil)
	b, err := GetXCached("mama")
	if err != nil || b == nil || b.(*myStruct).data != "mama are mere" {
		t.Error("Error retrieving data from cache", err)
	}
	time.Sleep(500 * time.Millisecond)
	b.KeepAlive()
	time.Sleep(1001 * time.Millisecond)
	b, err = GetXCached("mama")
	if err == nil || b != nil {
		t.Error("Error expiring data")
	}
	b, err = GetXCached("mama2")
	if err != nil || b == nil || b.(*myStruct).data != "mama are mere2" {
		t.Error("Error retrieving data from cache", err)
	}
	time.Sleep(2001 * time.Millisecond)
	b, err = GetXCached("mama2")
	if err == nil || b != nil {
		t.Error("Error expiring data")
	}
}

func TestFlush(t *testing.T) {
	a := &myStruct{data: "mama are mere"}
	a.XCache("mama", 10*time.Second, a, nil)
	XFlush()
	b, err := GetXCached("mama")
	if err == nil || b != nil {
		t.Error("Error expiring data")
	}
}

func TestFlushNoTimout(t *testing.T) {
	a := &myStruct{data: "mama are mere"}
	a.XCache("mama", 10*time.Second, a, nil)
	XFlush()
	b, err := GetXCached("mama")
	if err == nil || b != nil {
		t.Error("Error expiring data")
	}
}
<commit_msg>* Be a bit more lax with timer accuracy testing.<commit_after>package cache2go

import (
	"testing"
	"time"
)

type myStruct struct {
	XEntry
	data string
}

func TestCache(t *testing.T) {
	a := &myStruct{data: "mama are mere"}
	a.XCache("mama", 1*time.Second, a, nil)
	b, err := GetXCached("mama")
	if err != nil || b == nil || b != a {
		t.Error("Error retrieving data from cache", err)
	}
}

func TestCacheExpire(t *testing.T) {
	a := &myStruct{data: "mama are mere"}
	a.XCache("mama", 1*time.Second, a, nil)
	b, err := GetXCached("mama")
	if err != nil || b == nil || b.(*myStruct).data != "mama are mere" {
		t.Error("Error retrieving data from cache", err)
	}
	time.Sleep(1001 * time.Millisecond)
	b, err = GetXCached("mama")
	if err == nil || b != nil {
		t.Error("Error expiring data")
	}
}

func TestCacheKeepAlive(t *testing.T) {
	a := &myStruct{data: "mama are mere"}
	a.XCache("mama", 1*time.Second, a, nil)
	a = &myStruct{data: "mama are mere2"}
	a.XCache("mama2", 2*time.Second, a, nil)
	b, err := GetXCached("mama")
	if err != nil || b == nil || b.(*myStruct).data != "mama are mere" {
		t.Error("Error retrieving data from cache", err)
	}
	time.Sleep(500 * time.Millisecond)
	b.KeepAlive()
	time.Sleep(1001 * time.Millisecond)
	b, err = GetXCached("mama")
	if err == nil || b != nil {
		t.Error("Error expiring data")
	}
	b, err = GetXCached("mama2")
	if err != nil || b == nil || b.(*myStruct).data != "mama are mere2" {
		t.Error("Error retrieving data from cache", err)
	}
	time.Sleep(2100 * time.Millisecond)
	b, err = GetXCached("mama2")
	if err == nil || b != nil {
		t.Error("Error expiring data")
	}
}

func TestFlush(t *testing.T) {
	a := &myStruct{data: "mama are mere"}
	a.XCache("mama", 10*time.Second, a, nil)
	XFlush()
	b, err := GetXCached("mama")
	if err == nil || b != nil {
		t.Error("Error expiring data")
	}
}

func TestFlushNoTimout(t *testing.T) {
	a := &myStruct{data: "mama are mere"}
	a.XCache("mama", 10*time.Second, a, nil)
	XFlush()
	b, err := GetXCached("mama")
	if err == nil || b != nil {
		t.Error("Error expiring data")
	}
}
<|endoftext|>


<commit_before>package template

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/signal"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/HeavyHorst/memkv"
	"github.com/HeavyHorst/remco/backends"
	"github.com/HeavyHorst/remco/template/fileutil"
	"github.com/cloudflare/cfssl/log"
	"github.com/flosch/pongo2"
	flag "github.com/spf13/pflag"
)

type StoreConfig struct {
	backends.StoreClient
	Name     string
	Onetime  bool
	Watch    bool
	Prefix   string
	Interval int
	Keys     []string
	store    *memkv.Store
}

// Resource is the representation of a parsed template resource.
type Resource struct {
	storeClients []StoreConfig
	funcMap      map[string]interface{}
	store        memkv.Store
	sources      []*SrcDst
}

var ErrEmptySrc = errors.New("empty src template")

// NewResource creates a Resource.
func NewResource(storeClients []StoreConfig, sources []*SrcDst) (*Resource, error) {
	if len(storeClients) == 0 {
		return nil, errors.New("A valid StoreClient is required.")
	}

	for _, v := range sources {
		if v.Src == "" {
			return nil, ErrEmptySrc
		}
	}

	tr := &Resource{
		storeClients: storeClients,
		store:        memkv.New(),
		funcMap:      newFuncMap(),
		sources:      sources,
	}

	// initialize the inidividual backend memkv Stores
	for i := range tr.storeClients {
		store := memkv.New()
		tr.storeClients[i].store = &store
	}

	addFuncs(tr.funcMap, tr.store.FuncMap)

	return tr, nil
}

// NewResourceFromFlags creates a Resource from the provided commandline flags.
func NewResourceFromFlags(s backends.Store, flags *flag.FlagSet, watch bool) (*Resource, error) {
	src, _ := flags.GetString("src")
	dst, _ := flags.GetString("dst")
	keys, _ := flags.GetStringSlice("keys")
	fileMode, _ := flags.GetString("fileMode")
	prefix, _ := flags.GetString("prefix")
	reloadCmd, _ := flags.GetString("reload_cmd")
	checkCmd, _ := flags.GetString("check_cmd")
	onetime, _ := flags.GetBool("onetime")
	interval, _ := flags.GetInt("interval")
	UID := os.Geteuid()
	GID := os.Getegid()

	sd := &SrcDst{
		Src:       src,
		Dst:       dst,
		Mode:      fileMode,
		ReloadCmd: reloadCmd,
		CheckCmd:  checkCmd,
		GID:       GID,
		UID:       UID,
	}

	b := StoreConfig{
		StoreClient: s.Client,
		Name:        s.Name,
		Onetime:     onetime,
		Prefix:      prefix,
		Watch:       watch,
		Interval:    interval,
		Keys:        keys,
	}

	return NewResource([]StoreConfig{b}, []*SrcDst{sd})
}

// setVars sets the Vars for template resource.
func (t *Resource) setVars(storeClient StoreConfig) error {
	var err error
	log.Debug("Retrieving keys from store: " + storeClient.Name)
	log.Debug("Key prefix set to " + storeClient.Prefix)

	result, err := storeClient.GetValues(appendPrefix(storeClient.Prefix, storeClient.Keys))
	if err != nil {
		return err
	}

	storeClient.store.Purge()

	for k, res := range result {
		storeClient.store.Set(path.Join("/", strings.TrimPrefix(k, storeClient.Prefix)), res)
	}

	//merge all stores
	t.store.Purge()
	for _, v := range t.storeClients {
		for _, kv := range v.store.GetAllKVs() {
			t.store.Set(kv.Key, kv.Value)
		}
	}

	return nil
}

// createStageFile stages the src configuration file by processing the src
// template and setting the desired owner, group, and mode. It also sets the
// StageFile for the template resource.
// It returns an error if any.
func (t *Resource) createStageFileAndSync() error {
	for _, s := range t.sources {
		log.Debug("Using source template " + s.Src)

		if !fileutil.IsFileExist(s.Src) {
			return errors.New("Missing template: " + s.Src)
		}

		log.Debug("Compiling source template(pongo2) " + s.Src)
		tmpl, err := pongo2.FromFile(s.Src)
		if err != nil {
			return fmt.Errorf("Unable to process template %s, %s", s.Src, err)
		}

		// create TempFile in Dest directory to avoid cross-filesystem issues
		temp, err := ioutil.TempFile(filepath.Dir(s.Dst), "."+filepath.Base(s.Dst))
		if err != nil {
			return err
		}

		if err = tmpl.ExecuteWriter(t.funcMap, temp); err != nil {
			temp.Close()
			os.Remove(temp.Name())
			return err
		}

		defer temp.Close()

		// Set the owner, group, and mode on the stage file now to make it easier to
		// compare against the destination configuration file later.
		os.Chmod(temp.Name(), s.fileMode)
		os.Chown(temp.Name(), s.UID, s.GID)
		s.stageFile = temp

		if err = t.sync(s); err != nil {
			return err
		}
	}
	return nil
}

// sync compares the staged and dest config files and attempts to sync them
// if they differ. sync will run a config check command if set before
// overwriting the target config file. Finally, sync will run a reload command
// if set to have the application or service pick up the changes.
// It returns an error if any.
func (t *Resource) sync(s *SrcDst) error {
	staged := s.stageFile.Name()
	defer os.Remove(staged)

	log.Debug("Comparing candidate config to " + s.Dst)
	ok, err := fileutil.SameFile(staged, s.Dst)
	if err != nil {
		log.Error(err.Error())
	}

	if !ok {
		log.Info("Target config " + s.Dst + " out of sync")
		if s.CheckCmd != "" {
			if err := s.check(staged); err != nil {
				return errors.New("Config check failed: " + err.Error())
			}
		}
		log.Debug("Overwriting target config " + s.Dst)
		err := os.Rename(staged, s.Dst)
		if err != nil {
			if strings.Contains(err.Error(), "device or resource busy") {
				log.Debug("Rename failed - target is likely a mount. Trying to write instead")
				// try to open the file and write to it
				var contents []byte
				var rerr error
				contents, rerr = ioutil.ReadFile(staged)
				if rerr != nil {
					return rerr
				}
				err := ioutil.WriteFile(s.Dst, contents, s.fileMode)
				// make sure owner and group match the temp file, in case the file was created with WriteFile
				os.Chown(s.Dst, s.UID, s.GID)
				if err != nil {
					return err
				}
			} else {
				return err
			}
		}
		if s.ReloadCmd != "" {
			if err := s.reload(); err != nil {
				return err
			}
		}
		log.Info("Target config " + s.Dst + " has been updated")
	} else {
		log.Debug("Target config " + s.Dst + " in sync")
	}
	return nil
}

// setFileMode sets the FileMode.
func (t *Resource) setFileMode() error {
	for _, s := range t.sources {
		if err := s.setFileMode(); err != nil {
			return nil
		}
	}
	return nil
}

// Process is a convenience function that wraps calls to the three main tasks
// required to keep local configuration files in sync. First we gather vars
// from the store, then we stage a candidate configuration file, and finally sync
// things up.
// It returns an error if any.
func (t *Resource) process(storeClient StoreConfig) error {
	if err := t.setFileMode(); err != nil {
		return err
	}
	if err := t.setVars(storeClient); err != nil {
		return err
	}
	if err := t.createStageFileAndSync(); err != nil {
		return err
	}
	return nil
}

func (t *Resource) processAll() error {
	//load all KV-pairs the first time
	var err error
	if err = t.setFileMode(); err != nil {
		return err
	}
	for _, storeClient := range t.storeClients {
		if err := t.setVars(storeClient); err != nil {
			return err
		}
	}
	if err = t.createStageFileAndSync(); err != nil {
		return err
	}
	return nil
}

func (s StoreConfig) watch(stopChan chan bool, processChan chan StoreConfig) {
	var lastIndex uint64
	keysPrefix := appendPrefix(s.Prefix, s.Keys)

	for {
		select {
		case <-stopChan:
			return
		default:
			index, err := s.WatchPrefix(s.Prefix, keysPrefix, lastIndex, stopChan)
			if err != nil {
				log.Error(err)
				// Prevent backend errors from consuming all resources.
				time.Sleep(time.Second * 2)
				continue
			}
			processChan <- s
			lastIndex = index
		}
	}
}

func (s StoreConfig) interval(stopChan chan bool, processChan chan StoreConfig) {
	if s.Onetime {
		return
	}
	for {
		select {
		case <-stopChan:
			return
		case <-time.After(time.Duration(s.Interval) * time.Second):
			processChan <- s
		}
	}
}

func (t *Resource) Monitor() {
	wg := &sync.WaitGroup{}
	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
	stopChan := make(chan bool)
	processChan := make(chan StoreConfig)

	defer close(processChan)

	if err := t.processAll(); err != nil {
		log.Error(err)
	}

	for _, sc := range t.storeClients {
		wg.Add(1)
		if sc.Watch {
			go func(s StoreConfig) {
				defer wg.Done()
				//t.watch(s, stopChan, processChan)
				s.watch(stopChan, processChan)
			}(sc)
		} else {
			go func(s StoreConfig) {
				defer wg.Done()
				s.interval(stopChan, processChan)
			}(sc)
		}
	}

	go func() {
		// If there is no goroutine left - quit
		wg.Wait()
		signalChan <- syscall.SIGINT
	}()

	for {
		select {
		case storeClient := <-processChan:
			if err := t.process(storeClient); err != nil {
				log.Error(err)
			}
		case s := <-signalChan:
			log.Info(fmt.Sprintf("Captured %v. Exiting...", s))
			close(stopChan)
			// drain processChan
			go func() {
				for range processChan {
				}
			}()
			wg.Wait()
			return
		}
	}
}
<commit_msg>renamed some variables<commit_after>package template

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/signal"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/HeavyHorst/memkv"
	"github.com/HeavyHorst/remco/backends"
	"github.com/HeavyHorst/remco/template/fileutil"
	"github.com/cloudflare/cfssl/log"
	"github.com/flosch/pongo2"
	flag "github.com/spf13/pflag"
)

type StoreConfig struct {
	backends.StoreClient
	Name     string
	Onetime  bool
	Watch    bool
	Prefix   string
	Interval int
	Keys     []string
	store    *memkv.Store
}

// Resource is the representation of a parsed template resource.
type Resource struct {
	storeClients []StoreConfig
	funcMap      map[string]interface{}
	store        memkv.Store
	sources      []*SrcDst
}

var ErrEmptySrc = errors.New("empty src template")

// NewResource creates a Resource.
func NewResource(storeClients []StoreConfig, sources []*SrcDst) (*Resource, error) {
	if len(storeClients) == 0 {
		return nil, errors.New("A valid StoreClient is required.")
	}

	for _, v := range sources {
		if v.Src == "" {
			return nil, ErrEmptySrc
		}
	}

	tr := &Resource{
		storeClients: storeClients,
		store:        memkv.New(),
		funcMap:      newFuncMap(),
		sources:      sources,
	}

	// initialize the inidividual backend memkv Stores
	for i := range tr.storeClients {
		store := memkv.New()
		tr.storeClients[i].store = &store
	}

	addFuncs(tr.funcMap, tr.store.FuncMap)

	return tr, nil
}

// NewResourceFromFlags creates a Resource from the provided commandline flags.
func NewResourceFromFlags(s backends.Store, flags *flag.FlagSet, watch bool) (*Resource, error) {
	src, _ := flags.GetString("src")
	dst, _ := flags.GetString("dst")
	keys, _ := flags.GetStringSlice("keys")
	fileMode, _ := flags.GetString("fileMode")
	prefix, _ := flags.GetString("prefix")
	reloadCmd, _ := flags.GetString("reload_cmd")
	checkCmd, _ := flags.GetString("check_cmd")
	onetime, _ := flags.GetBool("onetime")
	interval, _ := flags.GetInt("interval")
	UID := os.Geteuid()
	GID := os.Getegid()

	sd := &SrcDst{
		Src:       src,
		Dst:       dst,
		Mode:      fileMode,
		ReloadCmd: reloadCmd,
		CheckCmd:  checkCmd,
		GID:       GID,
		UID:       UID,
	}

	b := StoreConfig{
		StoreClient: s.Client,
		Name:        s.Name,
		Onetime:     onetime,
		Prefix:      prefix,
		Watch:       watch,
		Interval:    interval,
		Keys:        keys,
	}

	return NewResource([]StoreConfig{b}, []*SrcDst{sd})
}

// setVars sets the Vars for template resource.
func (t *Resource) setVars(storeClient StoreConfig) error {
	var err error
	log.Debug("Retrieving keys from store: " + storeClient.Name)
	log.Debug("Key prefix set to " + storeClient.Prefix)

	result, err := storeClient.GetValues(appendPrefix(storeClient.Prefix, storeClient.Keys))
	if err != nil {
		return err
	}

	storeClient.store.Purge()

	for key, value := range result {
		storeClient.store.Set(path.Join("/", strings.TrimPrefix(key, storeClient.Prefix)), value)
	}

	//merge all stores
	t.store.Purge()
	for _, v := range t.storeClients {
		for _, kv := range v.store.GetAllKVs() {
			t.store.Set(kv.Key, kv.Value)
		}
	}

	return nil
}

// createStageFile stages the src configuration file by processing the src
// template and setting the desired owner, group, and mode. It also sets the
// StageFile for the template resource.
// It returns an error if any.
func (t *Resource) createStageFileAndSync() error {
	for _, s := range t.sources {
		log.Debug("Using source template " + s.Src)

		if !fileutil.IsFileExist(s.Src) {
			return errors.New("Missing template: " + s.Src)
		}

		log.Debug("Compiling source template(pongo2) " + s.Src)
		tmpl, err := pongo2.FromFile(s.Src)
		if err != nil {
			return fmt.Errorf("Unable to process template %s, %s", s.Src, err)
		}

		// create TempFile in Dest directory to avoid cross-filesystem issues
		temp, err := ioutil.TempFile(filepath.Dir(s.Dst), "."+filepath.Base(s.Dst))
		if err != nil {
			return err
		}

		if err = tmpl.ExecuteWriter(t.funcMap, temp); err != nil {
			temp.Close()
			os.Remove(temp.Name())
			return err
		}

		defer temp.Close()

		// Set the owner, group, and mode on the stage file now to make it easier to
		// compare against the destination configuration file later.
		os.Chmod(temp.Name(), s.fileMode)
		os.Chown(temp.Name(), s.UID, s.GID)
		s.stageFile = temp

		if err = t.sync(s); err != nil {
			return err
		}
	}
	return nil
}

// sync compares the staged and dest config files and attempts to sync them
// if they differ. sync will run a config check command if set before
// overwriting the target config file. Finally, sync will run a reload command
// if set to have the application or service pick up the changes.
// It returns an error if any.
func (t *Resource) sync(s *SrcDst) error {
	staged := s.stageFile.Name()
	defer os.Remove(staged)

	log.Debug("Comparing candidate config to " + s.Dst)
	ok, err := fileutil.SameFile(staged, s.Dst)
	if err != nil {
		log.Error(err.Error())
	}

	if !ok {
		log.Info("Target config " + s.Dst + " out of sync")
		if s.CheckCmd != "" {
			if err := s.check(staged); err != nil {
				return errors.New("Config check failed: " + err.Error())
			}
		}
		log.Debug("Overwriting target config " + s.Dst)
		err := os.Rename(staged, s.Dst)
		if err != nil {
			if strings.Contains(err.Error(), "device or resource busy") {
				log.Debug("Rename failed - target is likely a mount. Trying to write instead")
				// try to open the file and write to it
				var contents []byte
				var rerr error
				contents, rerr = ioutil.ReadFile(staged)
				if rerr != nil {
					return rerr
				}
				err := ioutil.WriteFile(s.Dst, contents, s.fileMode)
				// make sure owner and group match the temp file, in case the file was created with WriteFile
				os.Chown(s.Dst, s.UID, s.GID)
				if err != nil {
					return err
				}
			} else {
				return err
			}
		}
		if s.ReloadCmd != "" {
			if err := s.reload(); err != nil {
				return err
			}
		}
		log.Info("Target config " + s.Dst + " has been updated")
	} else {
		log.Debug("Target config " + s.Dst + " in sync")
	}
	return nil
}

// setFileMode sets the FileMode.
func (t *Resource) setFileMode() error {
	for _, s := range t.sources {
		if err := s.setFileMode(); err != nil {
			return nil
		}
	}
	return nil
}

// Process is a convenience function that wraps calls to the three main tasks
// required to keep local configuration files in sync. First we gather vars
// from the store, then we stage a candidate configuration file, and finally sync
// things up.
// It returns an error if any.
func (t *Resource) process(storeClient StoreConfig) error {
	if err := t.setFileMode(); err != nil {
		return err
	}
	if err := t.setVars(storeClient); err != nil {
		return err
	}
	if err := t.createStageFileAndSync(); err != nil {
		return err
	}
	return nil
}

func (t *Resource) processAll() error {
	//load all KV-pairs the first time
	var err error
	if err = t.setFileMode(); err != nil {
		return err
	}
	for _, storeClient := range t.storeClients {
		if err := t.setVars(storeClient); err != nil {
			return err
		}
	}
	if err = t.createStageFileAndSync(); err != nil {
		return err
	}
	return nil
}

func (s StoreConfig) watch(stopChan chan bool, processChan chan StoreConfig) {
	var lastIndex uint64
	keysPrefix := appendPrefix(s.Prefix, s.Keys)

	for {
		select {
		case <-stopChan:
			return
		default:
			index, err := s.WatchPrefix(s.Prefix, keysPrefix, lastIndex, stopChan)
			if err != nil {
				log.Error(err)
				// Prevent backend errors from consuming all resources.
				time.Sleep(time.Second * 2)
				continue
			}
			processChan <- s
			lastIndex = index
		}
	}
}

func (s StoreConfig) interval(stopChan chan bool, processChan chan StoreConfig) {
	if s.Onetime {
		return
	}
	for {
		select {
		case <-stopChan:
			return
		case <-time.After(time.Duration(s.Interval) * time.Second):
			processChan <- s
		}
	}
}

func (t *Resource) Monitor() {
	wg := &sync.WaitGroup{}
	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
	stopChan := make(chan bool)
	processChan := make(chan StoreConfig)

	defer close(processChan)

	if err := t.processAll(); err != nil {
		log.Error(err)
	}

	for _, sc := range t.storeClients {
		wg.Add(1)
		if sc.Watch {
			go func(s StoreConfig) {
				defer wg.Done()
				//t.watch(s, stopChan, processChan)
				s.watch(stopChan, processChan)
			}(sc)
		} else {
			go func(s StoreConfig) {
				defer wg.Done()
				s.interval(stopChan, processChan)
			}(sc)
		}
	}

	go func() {
		// If there is no goroutine left - quit
		wg.Wait()
		signalChan <- syscall.SIGINT
	}()

	for {
		select {
		case storeClient := <-processChan:
			if err := t.process(storeClient); err != nil {
				log.Error(err)
			}
		case s := <-signalChan:
			log.Info(fmt.Sprintf("Captured %v. Exiting...", s))
			close(stopChan)
			// drain processChan
			go func() {
				for range processChan {
				}
			}()
			wg.Wait()
			return
		}
	}
}
<|endoftext|>