<commit_before>package varis

import (
	"math/rand"
	"testing"
)

var dataset = Dataset{
	{Vector{0.0, 0.0}, Vector{1.0}},
	{Vector{1.0, 0.0}, Vector{0.0}},
	{Vector{0.0, 1.0}, Vector{0.0}},
	{Vector{1.0, 1.0}, Vector{1.0}},
}

func TestNetworkCreation(t *testing.T) {
	_ = CreatePerceptron(2, 3, 1)
}

func TestNetworkCalculate(t *testing.T) {
	rand.Seed(1488)
	nn := CreatePerceptron(2, 3, 1)

	if nn.Calculate(Vector{0.0, 0.0})[0] != 0.6631222149019123 {
		t.Error("Wrong with calculation")
	}
}

func TestNetworkTrain(t *testing.T) {
	nn := CreatePerceptron(2, 3, 1)

	trainer := PerceptronTrainer{&nn, dataset}
	trainer.BackPropagation(10000)

	if nn.Calculate(Vector{0.0, 0.0})[0] < 0.800000 {
		t.Log("Bad calculation: ", Vector{0.0, 0.0}, " result: ", nn.Calculate(Vector{0.0, 0.0})[0])
	}

	if nn.Calculate(Vector{1.0, 0.0})[0] > 0.200000 {
		t.Log("Bad calculation: ", Vector{1.0, 0.0}, " result: ", nn.Calculate(Vector{1.0, 0.0})[0])
	}

	if nn.Calculate(Vector{0.0, 1.0})[0] > 0.200000 {
		t.Log("Bad calculation: ", Vector{0.0, 1.0}, " result: ", nn.Calculate(Vector{0.0, 1.0})[0])
	}

	if nn.Calculate(Vector{1.0, 1.0})[0] < 0.800000 {
		t.Log("Bad calculation: ", Vector{1.0, 1.0}, " result: ", nn.Calculate(Vector{1.0, 1.0})[0])
	}
}

func TestEncodeJson(t *testing.T) {
	n1 := CreatePerceptron(2, 3, 1)

	trainer := PerceptronTrainer{&n1, dataset}
	trainer.BackPropagation(10000)

	json := ToJSON(n1)
	n2 := FromJSON(json)

	if n1.Calculate(Vector{0.0, 0.0})[0] != n2.Calculate(Vector{0.0, 0.0})[0] {
		t.Error("Result of neural newtwork calculating not a same ")
	}

	if n1.Calculate(Vector{1.0, 0.0})[0] != n2.Calculate(Vector{1.0, 0.0})[0] {
		t.Error("Result of neural newtwork calculating not a same")
	}

	if n1.Calculate(Vector{0.0, 1.0})[0] != n2.Calculate(Vector{0.0, 1.0})[0] {
		t.Error("Result of neural newtwork calculating not a same ")
	}

	if n1.Calculate(Vector{1.0, 1.0})[0] != n2.Calculate(Vector{1.0, 1.0})[0] {
		t.Error("Result of neural newtwork calculating not a same ")
	}
}

func TestModel(t *testing.T) {
	n := CreatePerceptron(2, 3, 1)

	trainer := PerceptronTrainer{&n, dataset}
	trainer.BackPropagation(10000)

	PrintCalculation = true

	n.Calculate(Vector{0.0, 0.0})
	n.Calculate(Vector{1.0, 0.0})
	n.Calculate(Vector{0.0, 1.0})
	n.Calculate(Vector{1.0, 1.0})

	type Model struct {
		Network *Perceptron
		X1      BooleanField `direction:"input"`
		X2      BooleanField `direction:"input"`
		O       BooleanField `direction:"output"`
	}

	f := Model{Network: &n}

	calculate := GenerateRunFunction(f)

	calculate([]interface{}{false, false})
	calculate([]interface{}{true, false})
	calculate([]interface{}{false, true})
	calculate([]interface{}{true, true})
}
<commit_msg>Create example in primary_test<commit_after>package varis

import (
	"math/rand"
	"testing"
)

var dataset = Dataset{
	{Vector{0.0, 0.0}, Vector{1.0}},
	{Vector{1.0, 0.0}, Vector{0.0}},
	{Vector{0.0, 1.0}, Vector{0.0}},
	{Vector{1.0, 1.0}, Vector{1.0}},
}

func TestNetworkCreation(t *testing.T) {
	_ = CreatePerceptron(2, 3, 1)
}

func TestNetworkCalculate(t *testing.T) {
	rand.Seed(1488)
	nn := CreatePerceptron(2, 3, 1)

	if nn.Calculate(Vector{0.0, 0.0})[0] != 0.6631222149019123 {
		t.Error("Wrong with calculation")
	}
}

func TestNetworkTrain(t *testing.T) {
	nn := CreatePerceptron(2, 3, 1)

	trainer := PerceptronTrainer{&nn, dataset}
	trainer.BackPropagation(10000)

	if nn.Calculate(Vector{0.0, 0.0})[0] < 0.800000 {
		t.Log("Bad calculation: ", Vector{0.0, 0.0}, " result: ", nn.Calculate(Vector{0.0, 0.0})[0])
	}

	if nn.Calculate(Vector{1.0, 0.0})[0] > 0.200000 {
		t.Log("Bad calculation: ", Vector{1.0, 0.0}, " result: ", nn.Calculate(Vector{1.0, 0.0})[0])
	}

	if nn.Calculate(Vector{0.0, 1.0})[0] > 0.200000 {
		t.Log("Bad calculation: ", Vector{0.0, 1.0}, " result: ", nn.Calculate(Vector{0.0, 1.0})[0])
	}

	if nn.Calculate(Vector{1.0, 1.0})[0] < 0.800000 {
		t.Log("Bad calculation: ", Vector{1.0, 1.0}, " result: ", nn.Calculate(Vector{1.0, 1.0})[0])
	}
}

func TestEncodeJson(t *testing.T) {
	n1 := CreatePerceptron(2, 3, 1)

	trainer := PerceptronTrainer{&n1, dataset}
	trainer.BackPropagation(10000)

	json := ToJSON(n1)
	n2 := FromJSON(json)

	if n1.Calculate(Vector{0.0, 0.0})[0] != n2.Calculate(Vector{0.0, 0.0})[0] {
		t.Error("Result of neural newtwork calculating not a same ")
	}

	if n1.Calculate(Vector{1.0, 0.0})[0] != n2.Calculate(Vector{1.0, 0.0})[0] {
		t.Error("Result of neural newtwork calculating not a same")
	}

	if n1.Calculate(Vector{0.0, 1.0})[0] != n2.Calculate(Vector{0.0, 1.0})[0] {
		t.Error("Result of neural newtwork calculating not a same ")
	}

	if n1.Calculate(Vector{1.0, 1.0})[0] != n2.Calculate(Vector{1.0, 1.0})[0] {
		t.Error("Result of neural newtwork calculating not a same ")
	}
}

func TestModel(t *testing.T) {
	n := CreatePerceptron(2, 3, 1)

	trainer := PerceptronTrainer{&n, dataset}
	trainer.BackPropagation(10000)

	PrintCalculation = true

	n.Calculate(Vector{0.0, 0.0})
	n.Calculate(Vector{1.0, 0.0})
	n.Calculate(Vector{0.0, 1.0})
	n.Calculate(Vector{1.0, 1.0})

	type Model struct {
		Network *Perceptron
		X1      BooleanField `direction:"input"`
		X2      BooleanField `direction:"input"`
		O       BooleanField `direction:"output"`
	}

	f := Model{Network: &n}

	calculate := GenerateRunFunction(f)

	calculate([]interface{}{false, false})
	calculate([]interface{}{true, false})
	calculate([]interface{}{false, true})
	calculate([]interface{}{true, true})
}

func Example() {
	rand.Seed(1338)
	n := CreatePerceptron(2, 3, 1)

	dataset := Dataset{
		{Vector{0.0, 0.0}, Vector{1.0}},
		{Vector{1.0, 0.0}, Vector{0.0}},
		{Vector{0.0, 1.0}, Vector{0.0}},
		{Vector{1.0, 1.0}, Vector{1.0}},
	}

	trainer := PerceptronTrainer{&n, dataset}
	trainer.BackPropagation(10000)

	PrintCalculation = true

	n.Calculate(Vector{0.0, 0.0})
	n.Calculate(Vector{1.0, 0.0})
	n.Calculate(Vector{0.0, 1.0})
	n.Calculate(Vector{1.0, 1.0})

	// Model example section
	type Model struct {
		Network *Perceptron
		X1      BooleanField `direction:"input"`
		X2      BooleanField `direction:"input"`
		O       BooleanField `direction:"output"`
	}

	f := Model{Network: &n}

	calculate := GenerateRunFunction(f)

	calculate([]interface{}{false, false})
	calculate([]interface{}{true, false})
	calculate([]interface{}{false, true})
	calculate([]interface{}{true, true})

	// Output:
	// Input: [0 0] Output: [0.9816677167418877]
	// Input: [1 0] Output: [0.02076530509106318]
	// Input: [0 1] Output: [0.018253250887023762]
	// Input: [1 1] Output: [0.9847884089930481]
	// Input: [0 0] Output: [0.9816677167418877]
	// Input: [1 0] Output: [0.02076530509106318]
	// Input: [0 1] Output: [0.018253250887023762]
	// Input: [1 1] Output: [0.9847884089930481]
}
<|endoftext|>


<commit_before>//  Author: Kevin Barry
//  Date:   21/09/2017
//  Guessing game
package main

import "fmt"
import "math/rand"//import for random number

func main() {

	var userinput int//variable to take user input
	var randomNum int = rand.Intn(100)//generate random number
	var counter int
	var numFound bool =false
	var numEntered bool = false
	var numsPicked [50]int
	var i int = 0

	fmt.Println()
	fmt.Println("Number Guessing Game")
	fmt.Println("====================")

	for numFound != true{//run while numfound = false

		//take user input
		fmt.Print("Please enter a number(1-100):" )
		fmt.Scan(&userinput)
		//counter = counter +1//increment counter
		fmt.Println(counter)
		i++
		numsPicked[i] = userinput

		for numEntered!=true{
			for y:=0 ;y< 50; y++{

				if numsPicked[y]==userinput{
					fmt.Println(counter)
					counter++
					numEntered=true
				}
			}
		}
		/*
		for y:=0 ;y< 50; y++{

			if numsPicked[y]==userinput{
				fmt.Println(counter)
				//counter--
			}

		}
		*/

		if userinput >randomNum{
			fmt.Println("Number was too large")
		}else if userinput < randomNum{
			fmt.Println("Number was too small")
		}//else if

		if userinput == randomNum{
			fmt.Println()
			fmt.Println("Guess correct Random number was ",randomNum)
			fmt.Println("Took ",counter," attempts")

			numFound = true ;//set bool numFound to true
		}//if
			numFound=false
	}//for

}<commit_msg>problem 5 complete<commit_after>//  Author: Kevin Barry
//  Date:   21/09/2017
//  Guessing game
package main

import "fmt"
import "math/rand"//import for random number

func main() {

	var userinput int//variable to take user input
	var randomNum int = rand.Intn(100)//generate random number
	var prevNum int
	var counter int
	var numFound bool =false

	fmt.Println()
	fmt.Println("Number Guessing Game")
	fmt.Println("====================")

	//loop while numfound = false
	for numFound != true{

		//take user input from console
		fmt.Print("Please enter a number(1-100):" )
		fmt.Scan(&userinput)
		counter++

		if userinput==prevNum{
			counter--
			fmt.Println("Number already chosen")
		}

		//compare user input to random number
		if userinput >randomNum{
			fmt.Println("Number was too large")
		}else if userinput < randomNum{
			fmt.Println("Number was too small")
		}else if  userinput == randomNum{
			fmt.Println()
			fmt.Println("Guess correct Random number was ",randomNum)
			fmt.Println("Took ",counter," attempts")
			numFound = true ;//set bool numFound to true
		}//if

		prevNum = userinput

	}//for
}//main<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"regexp"
	"strings"
)

var lineRegex = regexp.MustCompile("^([A-Za-z]+) => ([A-Za-z]+)$")
var moleculeRegex = regexp.MustCompile("[A-Z][a-z]?")

type replace struct {
	in, out string
}

func process(f io.Reader) (int, error) {
	var replacements []replace
	var molecule []string

	results := make(map[string]bool)

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		s := scanner.Text()
		m := lineRegex.FindStringSubmatch(s)
		if m != nil {
			replacements = append(replacements, replace{m[1], m[2]})
		} else if len(s) > 0 {
			molecule = moleculeRegex.FindAllString(s, -1)
		}
	}

	for k, v := range molecule {
		for _, r := range replacements {
			if r.in == v {
				newMolecule := make([]string, len(molecule))
				copy(newMolecule, molecule)
				newMolecule[k] = r.out
				newMoleculeStr := strings.Join(newMolecule, "")
				results[newMoleculeStr] = true
			}
		}
	}

	return len(results), nil
}

func run() int {
	if len(os.Args) != 2 {
		fmt.Fprintf(os.Stderr, "%s filename\n", os.Args[0])
		return 1
	}

	f, err := os.Open(os.Args[1])
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return 1
	}
	defer f.Close()

	c, err := process(f)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return 1
	}

	fmt.Printf("count: %d\n", c)
	return 0
}

func main() {
	os.Exit(run())
}
<commit_msg>day 19 2015 use regexps for replacements<commit_after>package main

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"regexp"
)

var lineRegex = regexp.MustCompile("^([A-Za-z]+) => ([A-Za-z]+)$")

type replace struct {
	in, out  string
	inRegexp *regexp.Regexp
}

func process(f io.Reader) (int, error) {
	var replacements []replace
	var molecule string

	results := make(map[string]bool)

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		s := scanner.Text()
		m := lineRegex.FindStringSubmatch(s)
		if m != nil {
			replacements = append(replacements, replace{m[1], m[2], regexp.MustCompile(m[1])})
		} else if len(s) > 0 {
			molecule = s
		}
	}

	for _, r := range replacements {
		m := r.inRegexp.FindAllStringIndex(molecule, -1)
		for _, v := range m {
			newMolecule := molecule[:v[0]] + r.out + molecule[v[1]:]
			results[newMolecule] = true
		}
	}

	return len(results), nil
}

func run() int {
	if len(os.Args) != 2 {
		fmt.Fprintf(os.Stderr, "%s filename\n", os.Args[0])
		return 1
	}

	f, err := os.Open(os.Args[1])
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return 1
	}
	defer f.Close()

	c, err := process(f)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return 1
	}

	fmt.Printf("count: %d\n", c)
	return 0
}

func main() {
	os.Exit(run())
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/dekelund/stdres"

	"github.com/dekelund/unbrokenwing/compiler/definition"
	"github.com/dekelund/unbrokenwing/compiler/feature"
	"github.com/dekelund/unbrokenwing/global"
)

// Foreground colors
const (
	reset   = "\033[00m"
	black   = "\033[30m"
	red     = "\033[31m"
	green   = "\033[32m"
	yellow  = "\033[33m"
	blue    = "\033[34m"
	magenta = "\033[35m"
	cyan    = "\033[36m"
	white   = "\033[37m"
)

const (
	PathSeparator = string(os.PathSeparator)
)

var CWD string = "."

func init() {
	cwd, err := os.Getwd()
	if err != nil {
		log.Panic(err)
	}

	CWD = cwd
}

func main() {
	app := cli.NewApp()
	app.Name = "unbrokenwing"
	app.Usage = "Run behaviour driven tests as Gherik features"
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "pretty",
			Usage: "Print colorised result to STDOUT/STDERR",
		},
		cli.BoolFlag{
			Name:  "debug",
			Usage: "Verbose printing (Generated files will be kept)",
		},
		cli.StringFlag{
			Name:  "step-definitions",
			Value: "step_definitions",
			Usage: "Definitions folder name, should be located in features folder",
		},
		cli.StringFlag{
			Name:  "dir",
			Value: ".",
			Usage: "Relative path, to a feature-file or -directory (Current value: " + CWD + ").",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:    "feature-files",
			Aliases: []string{},
			Usage:   "List feature files to STDOUT",
			Flags:   []cli.Flag{},
			Action:  listFeatureFilesCMD,
		},
		{
			Name:    "features",
			Aliases: []string{},
			Usage:   "List features to STDOUT",
			Flags:   []cli.Flag{},
			Action:  listFeaturesCMD,
		},
		{
			Name:    "definitions",
			Aliases: []string{"defs", "code"},
			Usage:   "List behaviours to STDOUT",
			Flags:   []cli.Flag{},
			Action:  printDefinitionsCodeCMD,
		},
		{
			Name:    "test",
			Aliases: []string{"t"},
			Usage:   "Tests either a test directory with features in it, or a .feature file",
			Flags:   []cli.Flag{},
			Action:  testCMD,
		},
	}

	app.Run(os.Args)
}

func setupGlobals(c *cli.Context) {
	global.Debug = c.GlobalBool("debug")
	global.PPrint = c.GlobalBool("pretty")
	global.DefPattern = c.GlobalString("step-definitions")

	global.CWD = CWD

	if global.PPrint {
		stdres.EnableColor()
	} else {
		stdres.DisableColor()
	}
}

func listFeatureFilesCMD(c *cli.Context) {
	setupGlobals(c)
	dir := c.GlobalString("dir")

	_, features := parseDir(dir)

	for i, feature := range features {
		path := CWD + PathSeparator
		fmt.Printf("\t%2d) %s\n", i, strings.TrimPrefix(feature, path))
	}
}

func listFeaturesCMD(c *cli.Context) {
	setupGlobals(c)
	dir := c.GlobalString("dir")

	_, features := parseDir(dir)

	for _, feature := range features {
		fileReader, err := os.Open(feature)
		if err != nil {
			log.Fatal(err)
		}

		bytes, err := ioutil.ReadAll(fileReader)
		if err != nil {
			log.Fatal(err)
		}

		text := string(bytes)
		text = strings.Replace(text, "Feature: ", red+"Feature: "+reset, -1)
		text = strings.Replace(text, "Scenario: ", red+"Scenario: "+reset, -1)
		text = strings.Replace(text, " Given ", green+" Given "+reset, -1)
		text = strings.Replace(text, " And ", green+" And "+reset, -1)
		text = strings.Replace(text, " When ", blue+" When "+reset, -1)
		text = strings.Replace(text, " Then ", yellow+" Then "+reset, -1)

		path := CWD + PathSeparator
		fmt.Print("\n# ", strings.TrimPrefix(feature, path), "\n", text, "\n")
	}
}

func printDefinitionsCodeCMD(c *cli.Context) {
	setupGlobals(c)
	dir := c.GlobalString("dir")

	definitions, _ := parseDir(dir)

	fmt.Println(definitions.Code())
}

// testCMD search, compile and execute features defined in Gherik format where behaviours are defined in Go-Lang based files.
// Behaviours might be undefined, which will end up as red text in stdout if the context c has pretty print enabled.
func testCMD(c *cli.Context) {
	setupGlobals(c)
	dir := c.GlobalString("dir")

	definitions, features := parseDir(dir)

	if !global.Debug {
		defer definitions.Remove()
	}

	for _, file := range features {
		fd, err := os.Open(file)
		if err != nil {
			log.Fatal(err)
		}
		defer fd.Close()

		definitions.Run(fd, global.PPrint, global.Debug)
	}
}

func parseDir(path string) (definition.Definitions, []string) {
	var err error
	var list = feature.List{}
	var defs = []io.Reader{}

	if list, err = feature.ParseDir(path); err != nil {
		log.Fatal(err)
	}

	defFiles := []io.ReadCloser{}

	defer func() {
		for _, file := range defFiles {
			file.Close()
		}
	}() // Make sure to close all open files

	for _, def := range list.Definitions {
		if file, err := os.Open(def); err == nil {
			defFiles = append(defFiles, file)
		} else {
			log.Fatal(err)
		}
	}

	// FIXME figure out if it's possible to ignore convert
	for _, def := range defFiles {
		defs = append(defs, io.Reader(def))
	}

	return definition.NewDefinitions(defs), list.Features
}
<commit_msg>FEATURE: Add syntax highlight to defs subcommand, and respect pprint flag for existing highlights<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/dekelund/stdres"

	"github.com/dekelund/unbrokenwing/compiler/definition"
	"github.com/dekelund/unbrokenwing/compiler/feature"
	"github.com/dekelund/unbrokenwing/global"
)

// Foreground colors
const (
	reset   = "\033[00m"
	black   = "\033[30m"
	red     = "\033[31m"
	green   = "\033[32m"
	yellow  = "\033[33m"
	blue    = "\033[34m"
	magenta = "\033[35m"
	cyan    = "\033[36m"
	white   = "\033[37m"
)

const (
	PathSeparator = string(os.PathSeparator)
)

var CWD string = "."

func init() {
	cwd, err := os.Getwd()
	if err != nil {
		log.Panic(err)
	}

	CWD = cwd
}

func main() {
	app := cli.NewApp()
	app.Name = "unbrokenwing"
	app.Usage = "Run behaviour driven tests as Gherik features"
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "pretty",
			Usage: "Print colorised result to STDOUT/STDERR",
		},
		cli.BoolFlag{
			Name:  "debug",
			Usage: "Verbose printing (Generated files will be kept)",
		},
		cli.StringFlag{
			Name:  "step-definitions",
			Value: "step_definitions",
			Usage: "Definitions folder name, should be located in features folder",
		},
		cli.StringFlag{
			Name:  "dir",
			Value: ".",
			Usage: "Relative path, to a feature-file or -directory (Current value: " + CWD + ").",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:    "feature-files",
			Aliases: []string{},
			Usage:   "List feature files to STDOUT",
			Flags:   []cli.Flag{},
			Action:  listFeatureFilesCMD,
		},
		{
			Name:    "features",
			Aliases: []string{},
			Usage:   "List features to STDOUT",
			Flags:   []cli.Flag{},
			Action:  listFeaturesCMD,
		},
		{
			Name:    "definitions",
			Aliases: []string{"defs", "code"},
			Usage:   "List behaviours to STDOUT",
			Flags:   []cli.Flag{},
			Action:  printDefinitionsCodeCMD,
		},
		{
			Name:    "test",
			Aliases: []string{"t"},
			Usage:   "Tests either a test directory with features in it, or a .feature file",
			Flags:   []cli.Flag{},
			Action:  testCMD,
		},
	}

	app.Run(os.Args)
}

func setupGlobals(c *cli.Context) {
	global.Debug = c.GlobalBool("debug")
	global.PPrint = c.GlobalBool("pretty")
	global.DefPattern = c.GlobalString("step-definitions")

	global.CWD = CWD

	if global.PPrint {
		stdres.EnableColor()
	} else {
		stdres.DisableColor()
	}
}

func listFeatureFilesCMD(c *cli.Context) {
	setupGlobals(c)
	dir := c.GlobalString("dir")

	_, features := parseDir(dir)

	for i, feature := range features {
		path := CWD + PathSeparator
		fmt.Printf("\t%2d) %s\n", i, strings.TrimPrefix(feature, path))
	}
}

func listFeaturesCMD(c *cli.Context) {
	setupGlobals(c)
	dir := c.GlobalString("dir")

	_, features := parseDir(dir)

	for _, feature := range features {
		fileReader, err := os.Open(feature)
		if err != nil {
			log.Fatal(err)
		}

		bytes, err := ioutil.ReadAll(fileReader)
		if err != nil {
			log.Fatal(err)
		}

		text := string(bytes)

		if global.PPrint {
			text = strings.Replace(text, "Feature: ", red+"Feature: "+reset, -1)
			text = strings.Replace(text, "Scenario: ", red+"Scenario: "+reset, -1)
			text = strings.Replace(text, " Given ", green+" Given "+reset, -1)
			text = strings.Replace(text, " And ", green+" And "+reset, -1)
			text = strings.Replace(text, " When ", blue+" When "+reset, -1)
			text = strings.Replace(text, " Then ", yellow+" Then "+reset, -1)
		}

		path := CWD + PathSeparator
		fmt.Print("\n# ", strings.TrimPrefix(feature, path), "\n", text, "\n")
	}
}

func printDefinitionsCodeCMD(c *cli.Context) {
	setupGlobals(c)
	dir := c.GlobalString("dir")

	definitions, _ := parseDir(dir)

	text := string(definitions.Code())

	if global.PPrint {
		text = strings.Replace(text, "package main", yellow+"package "+blue+"main"+reset, -1)
		text = strings.Replace(text, "import ", yellow+"import"+reset+" ", -1)
		text = strings.Replace(text, "func ", blue+"func"+reset+" ", -1)
		text = strings.Replace(text, "func(", blue+"func"+reset+"(", -1)
		text = strings.Replace(text, "defer ", blue+"defer"+reset+" ", -1)
		text = strings.Replace(text, " error ", " "+red+"error"+reset+" ", -1)

		text = strings.Replace(text, "\nFeature(", red+"\nFeature"+reset+"(", -1)
		text = strings.Replace(text, "\nScenario(", red+"\nScenario"+reset+"(", -1)
		text = strings.Replace(text, "\nGiven(", green+"\nGiven"+reset+"(", -1)
		text = strings.Replace(text, "\nAnd(", green+"\nAnd"+reset+"(", -1)
		text = strings.Replace(text, "\nWhen(", blue+"\nWhen"+reset+"(", -1)
		text = strings.Replace(text, "\nThen(", yellow+"\nThen"+reset+"(", -1)

		text = strings.Replace(text, "main()", yellow+"main"+reset+"()", -1)
		text = strings.Replace(text, "setup()", yellow+"setup"+reset+"()", -1)

		text = strings.Replace(text, " Pending(", " "+yellow+"Pending"+reset+"(", -1)
		text = strings.Replace(text, "os.Open(", yellow+"os.Open"+reset+"(", -1)
		text = strings.Replace(text, "fd.Close(", yellow+"fd.Close"+reset+"(", -1)
		text = strings.Replace(text, "suite.Test(", yellow+"suite.Test"+reset+"(", -1)
		text = strings.Replace(text, "ParseBool(", yellow+"ParseBool"+reset+"(", -1)
		text = strings.Replace(text, "stdres.EnableColor(", yellow+"stdres.EnableColor"+reset+"(", -1)
		text = strings.Replace(text, "stdres.DisableColor(", yellow+"stdres.DisableColor"+reset+"(", -1)
		text = strings.Replace(text, "os.Args[", yellow+"os.Args"+reset+"[", -1)
	}

	fmt.Println(text)
}

// testCMD search, compile and execute features defined in Gherik format where behaviours are defined in Go-Lang based files.
// Behaviours might be undefined, which will end up as red text in stdout if the context c has pretty print enabled.
func testCMD(c *cli.Context) {
	setupGlobals(c)
	dir := c.GlobalString("dir")

	definitions, features := parseDir(dir)

	if !global.Debug {
		defer definitions.Remove()
	}

	for _, file := range features {
		fd, err := os.Open(file)
		if err != nil {
			log.Fatal(err)
		}
		defer fd.Close()

		definitions.Run(fd, global.PPrint, global.Debug)
	}
}

func parseDir(path string) (definition.Definitions, []string) {
	var err error
	var list = feature.List{}
	var defs = []io.Reader{}

	if list, err = feature.ParseDir(path); err != nil {
		log.Fatal(err)
	}

	defFiles := []io.ReadCloser{}

	defer func() {
		for _, file := range defFiles {
			file.Close()
		}
	}() // Make sure to close all open files

	for _, def := range list.Definitions {
		if file, err := os.Open(def); err == nil {
			defFiles = append(defFiles, file)
		} else {
			log.Fatal(err)
		}
	}

	// FIXME figure out if it's possible to ignore convert
	for _, def := range defFiles {
		defs = append(defs, io.Reader(def))
	}

	return definition.NewDefinitions(defs), list.Features
}
<|endoftext|>


<commit_before>package url

import (
	"bytes"
	"fmt"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/storage"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"strings"
	"time"
)

//Resource represents a URL based resource, with enriched meta info
type Resource struct {
	URL             string   `description:"resource URL or relative or absolute path" required:"true"` //URL of resource
	Credentials     string   `description:"credentials file"`                                          //name of credential file or credential key depending on implementation
	ParsedURL       *url.URL `json:"-"`                                                                //parsed URL resource
	Cache           string   `description:"local cache path"`                                          //Cache path for the resource, if specified resource will be cached in the specified path
	CacheExpiryMs   int      //CacheExpiryMs expiry time in ms
	modificationTag int64
	init            bool
}

//Clone creates a clone of the resource
func (r *Resource) Clone() *Resource {
	return &Resource{
		init:          r.init,
		URL:           r.URL,
		Credentials:   r.Credentials,
		ParsedURL:     r.ParsedURL,
		Cache:         r.Cache,
		CacheExpiryMs: r.CacheExpiryMs,
	}
}

var defaultSchemePorts = map[string]int{
	"ssh":   22,
	"scp":   22,
	"http":  80,
	"https": 443,
}

//Host returns url's host name with user name if user name is part of url
func (r *Resource) Host() string {
	result := r.ParsedURL.Hostname() + ":" + r.Port()
	if r.ParsedURL.User != nil {
		result = r.ParsedURL.User.Username() + "@" + result
	}
	return result
}

//CredentialURL returns url's with provided credential
func (r *Resource) CredentialURL(username, password string) string {
	var urlCredential = ""
	if username != "" {
		urlCredential = username
		if password != "" {
			urlCredential += ":" + password
		}
		urlCredential += "@"
	}
	result := r.ParsedURL.Scheme + "://" + urlCredential + r.ParsedURL.Hostname() + ":" + r.Port() + r.ParsedURL.Path
	if r.ParsedURL.RawQuery != "" {
		result += "?" + r.ParsedURL.RawQuery
	}

	return result
}

//Path returns url's path  directory, assumption is that directory does not have extension, if path ends with '/' it is being stripped.
func (r *Resource) DirectoryPath() string {
	if r.ParsedURL == nil {
		return ""
	}
	var result = r.ParsedURL.Path

	parent, name := path.Split(result)
	if path.Ext(name) != "" {
		result = parent
	}
	if strings.HasSuffix(result, "/") {
		result = string(result[:len(result)-1])
	}
	return result
}

//Port returns url's port
func (r *Resource) Port() string {
	port := r.ParsedURL.Port()
	if port == "" && r.ParsedURL != nil {
		if value, ok := defaultSchemePorts[r.ParsedURL.Scheme]; ok {
			port = toolbox.AsString(value)
		}
	}
	return port
}

//Download downloads data from URL, it returns data as []byte, or error, if resource is cacheable it first look into cache
func (r *Resource) Download() ([]byte, error) {
	if r == nil {
		return nil, fmt.Errorf("Fail to download content on empty resource")
	}
	if r.Cachable() {
		content := r.readFromCache()
		if content != nil {
			return content, nil
		}
	}
	service, err := storage.NewServiceForURL(r.URL, r.Credentials)
	if err != nil {
		return nil, err
	}
	object, err := service.StorageObject(r.URL)
	if err != nil {
		return nil, err
	}
	reader, err := service.Download(object)
	if err != nil {
		return nil, err
	}
	defer reader.Close()
	content, err := ioutil.ReadAll(reader)
	if err != nil {
		return nil, err
	}
	if r.Cachable() {
		_ = ioutil.WriteFile(r.Cache, content, 0666)
	}
	return content, err
}

//DownloadText returns a text downloaded from url
func (r *Resource) DownloadText() (string, error) {
	var result, err = r.Download()
	if err != nil {
		return "", err
	}
	return string(result), err
}

//Decode decodes url's data into target, it support JSON and YAML exp.
func (r *Resource) Decode(target interface{}) (err error) {
	defer func() {
		if err != nil {
			err = fmt.Errorf("failed to decode: %v, %v", r.URL, err)
		}
	}()
	ext := path.Ext(r.ParsedURL.Path)
	switch ext {
	case ".yaml", ".yml":
		err = r.YAMLDecode(target)
	default:
		err = r.JSONDecode(target)
	}
	return err
}

//DecoderFactory returns new decoder factory for resource
func (r *Resource) DecoderFactory() toolbox.DecoderFactory {
	ext := path.Ext(r.ParsedURL.Path)
	switch ext {
	case ".yaml", ".yml":
		return toolbox.NewYamlDecoderFactory()
	default:
		return toolbox.NewJSONDecoderFactory()
	}
}

//Decode decodes url's data into target, it takes decoderFactory which decodes data into target
func (r *Resource) DecodeWith(target interface{}, decoderFactory toolbox.DecoderFactory) error {
	if r == nil {
		return fmt.Errorf("fail to %T decode on empty resource", decoderFactory)
	}
	if decoderFactory == nil {
		return fmt.Errorf("fail to decode %v, decoderFactory was empty", r.URL)
	}
	var content, err = r.Download()
	if err != nil {
		return err
	}
	err = decoderFactory.Create(bytes.NewReader(content)).Decode(target)
	if err != nil {
		return fmt.Errorf("failed to decode: %v, payload: %s", err, content)
	}
	return err
}

//Rename renames URI name of this resource
func (r *Resource) Rename(name string) (err error) {
	var _, currentName = toolbox.URLSplit(r.URL)
	if currentName == "" && strings.HasSuffix(r.URL, "/") {
		_, currentName = toolbox.URLSplit(r.URL[:len(r.URL)-1])
		currentName += "/"
	}

	r.URL = strings.Replace(r.URL, currentName, name, 1)
	r.ParsedURL, err = url.Parse(r.URL)
	return err
}

//JSONDecode decodes json resource into target
func (r *Resource) JSONDecode(target interface{}) error {
	return r.DecodeWith(target, toolbox.NewJSONDecoderFactory())
}

//JSONDecode decodes yaml resource into target
func (r *Resource) YAMLDecode(target interface{}) error {
	if interfacePrt, ok := target.(*interface{}); ok {
		var data interface{}
		if err := r.DecodeWith(&data, toolbox.NewYamlDecoderFactory()); err != nil {
			return err
		}
		if toolbox.IsSlice(data) {
			*interfacePrt = data
			return nil
		}
	}
	var mapSlice = yaml.MapSlice{}
	if err := r.DecodeWith(&mapSlice, toolbox.NewYamlDecoderFactory()); err != nil {
		return err
	}
	if !toolbox.IsMap(target) {
		return toolbox.DefaultConverter.AssignConverted(target, mapSlice)
	}
	resultMap := toolbox.AsMap(target)
	for _, v := range mapSlice {
		resultMap[toolbox.AsString(v.Key)] = v.Value
	}
	return nil
}

func (r *Resource) readFromCache() []byte {
	if toolbox.FileExists(r.Cache) {
		info, err := os.Stat(r.Cache)
		var isExpired = false
		if err == nil && r.CacheExpiryMs > 0 {
			elapsed := time.Now().Sub(info.ModTime())
			isExpired = elapsed > time.Second*time.Duration(r.CacheExpiryMs)
		}
		content, err := ioutil.ReadFile(r.Cache)
		if err == nil && !isExpired {
			return content
		}
	}
	return nil
}

//Cachable returns true if resource is cachable
func (r *Resource) Cachable() bool {
	return r.Cache != ""
}

func computeResourceModificationTag(resource *Resource) (int64, error) {
	service, err := storage.NewServiceForURL(resource.URL, resource.Credentials)
	if err != nil {
		return 0, err
	}
	object, err := service.StorageObject(resource.URL)
	if err != nil {
		return 0, err
	}
	var fileInfo = object.FileInfo()

	if object.IsContent() {
		return fileInfo.Size() + fileInfo.ModTime().UnixNano(), nil
	}
	var result int64 = 0
	objects, err := service.List(resource.URL)
	if err != nil {
		return 0, err
	}
	for _, object := range objects {
		objectResource := NewResource(object.URL())
		if objectResource.ParsedURL.Path == resource.ParsedURL.Path {
			continue
		}
		modificationTag, err := computeResourceModificationTag(NewResource(object.URL(), resource.Credentials))
		if err != nil {
			return 0, err
		}
		result += modificationTag

	}
	return result, nil
}

func (r *Resource) HasChanged() (changed bool, err error) {
	if r.modificationTag == 0 {
		r.modificationTag, err = computeResourceModificationTag(r)
		return false, err
	}
	var recentModificationTag int64
	recentModificationTag, err = computeResourceModificationTag(r)
	if err != nil {
		return false, err
	}
	if recentModificationTag != r.modificationTag {
		changed = true
		r.modificationTag = recentModificationTag
	}
	return changed, err
}

func normalizeURL(URL string) string {
	if strings.Contains(URL, "://") {
		var protoPosition = strings.Index(URL, "://")
		if protoPosition != -1 {
			var urlSuffix = string(URL[protoPosition+3:])
			urlSuffix = strings.Replace(urlSuffix, "//", "/", len(urlSuffix))
			URL = string(URL[:protoPosition+3]) + urlSuffix
		}
		return URL
	}
	if !strings.HasPrefix(URL, "/") {
		currentDirectory, _ := os.Getwd()

		if strings.Contains(URL, "..") {
			fragments := strings.Split(URL, "/")
			var index = 0
			var offset = 0
			if fragments[0] == "." {
				offset = 1
			}

			for index = offset; index < len(fragments); index++ {
				var fragment = fragments[index]
				if fragment == ".." {
					currentDirectory, _ = path.Split(currentDirectory)
					if strings.HasSuffix(currentDirectory, "/") {
						currentDirectory = string(currentDirectory[:len(currentDirectory)-1])
					}
					continue
				}
				break
			}
			return toolbox.FileSchema + path.Join(currentDirectory, strings.Join(fragments[index:], "/"))
		}

		currentDirectory, err := os.Getwd()
		if err == nil {
			candidate := path.Join(currentDirectory, URL)
			URL = candidate
		}
	}
	return toolbox.FileSchema + URL
}

func (r *Resource) Init() (err error) {
	if r.init {
		return nil
	}
	r.init = true
	r.URL = normalizeURL(r.URL)
	r.ParsedURL, err = url.Parse(r.URL)
	return err
}

//NewResource returns a new resource for provided URL, followed by optional credential, cache and cache expiryMs.
func NewResource(Params ...interface{}) *Resource {
	if len(Params) == 0 {
		return nil
	}
	var URL = toolbox.AsString(Params[0])
	URL = normalizeURL(URL)

	var credential string
	if len(Params) > 1 {
		credential = toolbox.AsString(Params[1])
	}
	var cache string
	if len(Params) > 2 {
		cache = toolbox.AsString(Params[2])
	}
	var cacheExpiryMs int
	if len(Params) > 3 {
		cacheExpiryMs = toolbox.AsInt(Params[3])
	}
	parsedURL, _ := url.Parse(URL)
	return &Resource{
		init:          true,
		ParsedURL:     parsedURL,
		URL:           URL,
		Credentials:   credential,
		Cache:         cache,
		CacheExpiryMs: cacheExpiryMs,
	}
}
<commit_msg>Patched new line delimited json decoding<commit_after>package url

import (
	"bytes"
	"fmt"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/storage"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"strings"
	"time"
)

//Resource represents a URL based resource, with enriched meta info
type Resource struct {
	URL             string   `description:"resource URL or relative or absolute path" required:"true"` //URL of resource
	Credentials     string   `description:"credentials file"`                                          //name of credential file or credential key depending on implementation
	ParsedURL       *url.URL `json:"-"`                                                                //parsed URL resource
	Cache           string   `description:"local cache path"`                                          //Cache path for the resource, if specified resource will be cached in the specified path
	CacheExpiryMs   int      //CacheExpiryMs expiry time in ms
	modificationTag int64
	init            bool
}

//Clone creates a clone of the resource
func (r *Resource) Clone() *Resource {
	return &Resource{
		init:          r.init,
		URL:           r.URL,
		Credentials:   r.Credentials,
		ParsedURL:     r.ParsedURL,
		Cache:         r.Cache,
		CacheExpiryMs: r.CacheExpiryMs,
	}
}

var defaultSchemePorts = map[string]int{
	"ssh":   22,
	"scp":   22,
	"http":  80,
	"https": 443,
}

//Host returns url's host name with user name if user name is part of url
func (r *Resource) Host() string {
	result := r.ParsedURL.Hostname() + ":" + r.Port()
	if r.ParsedURL.User != nil {
		result = r.ParsedURL.User.Username() + "@" + result
	}
	return result
}

//CredentialURL returns url's with provided credential
func (r *Resource) CredentialURL(username, password string) string {
	var urlCredential = ""
	if username != "" {
		urlCredential = username
		if password != "" {
			urlCredential += ":" + password
		}
		urlCredential += "@"
	}
	result := r.ParsedURL.Scheme + "://" + urlCredential + r.ParsedURL.Hostname() + ":" + r.Port() + r.ParsedURL.Path
	if r.ParsedURL.RawQuery != "" {
		result += "?" + r.ParsedURL.RawQuery
	}

	return result
}

//Path returns url's path  directory, assumption is that directory does not have extension, if path ends with '/' it is being stripped.
func (r *Resource) DirectoryPath() string {
	if r.ParsedURL == nil {
		return ""
	}
	var result = r.ParsedURL.Path

	parent, name := path.Split(result)
	if path.Ext(name) != "" {
		result = parent
	}
	if strings.HasSuffix(result, "/") {
		result = string(result[:len(result)-1])
	}
	return result
}

//Port returns url's port
func (r *Resource) Port() string {
	port := r.ParsedURL.Port()
	if port == "" && r.ParsedURL != nil {
		if value, ok := defaultSchemePorts[r.ParsedURL.Scheme]; ok {
			port = toolbox.AsString(value)
		}
	}
	return port
}

//Download downloads data from URL, it returns data as []byte, or error, if resource is cacheable it first look into cache
func (r *Resource) Download() ([]byte, error) {
	if r == nil {
		return nil, fmt.Errorf("Fail to download content on empty resource")
	}
	if r.Cachable() {
		content := r.readFromCache()
		if content != nil {
			return content, nil
		}
	}
	service, err := storage.NewServiceForURL(r.URL, r.Credentials)
	if err != nil {
		return nil, err
	}
	object, err := service.StorageObject(r.URL)
	if err != nil {
		return nil, err
	}
	reader, err := service.Download(object)
	if err != nil {
		return nil, err
	}
	defer reader.Close()
	content, err := ioutil.ReadAll(reader)
	if err != nil {
		return nil, err
	}
	if r.Cachable() {
		_ = ioutil.WriteFile(r.Cache, content, 0666)
	}
	return content, err
}

//DownloadText returns a text downloaded from url
func (r *Resource) DownloadText() (string, error) {
	var result, err = r.Download()
	if err != nil {
		return "", err
	}
	return string(result), err
}

//Decode decodes url's data into target, it support JSON and YAML exp.
func (r *Resource) Decode(target interface{}) (err error) {
	defer func() {
		if err != nil {
			err = fmt.Errorf("failed to decode: %v, %v", r.URL, err)
		}
	}()
	ext := path.Ext(r.ParsedURL.Path)
	switch ext {
	case ".yaml", ".yml":
		err = r.YAMLDecode(target)
	default:
		err = r.JSONDecode(target)
	}
	return err
}

//DecoderFactory returns new decoder factory for resource
func (r *Resource) DecoderFactory() toolbox.DecoderFactory {
	ext := path.Ext(r.ParsedURL.Path)
	switch ext {
	case ".yaml", ".yml":
		return toolbox.NewYamlDecoderFactory()
	default:
		return toolbox.NewJSONDecoderFactory()
	}
}

//Decode decodes url's data into target, it takes decoderFactory which decodes data into target
func (r *Resource) DecodeWith(target interface{}, decoderFactory toolbox.DecoderFactory) error {
	if r == nil {
		return fmt.Errorf("fail to %T decode on empty resource", decoderFactory)
	}
	if decoderFactory == nil {
		return fmt.Errorf("fail to decode %v, decoderFactory was empty", r.URL)
	}
	var content, err = r.Download()
	if err != nil {
		return err
	}

	text := string(content)
	if toolbox.IsNewLineDelimitedJSON(text) {
		if aSlice, err := toolbox.NewLineDelimitedJSON(text); err == nil {
			return toolbox.DefaultConverter.AssignConverted(target, aSlice)
		}
	}
	err = decoderFactory.Create(bytes.NewReader(content)).Decode(target)
	if err != nil {
		return fmt.Errorf("failed to decode: %v, payload: %s", err, content)
	}
	return err
}

//Rename renames URI name of this resource
func (r *Resource) Rename(name string) (err error) {
	var _, currentName = toolbox.URLSplit(r.URL)
	if currentName == "" && strings.HasSuffix(r.URL, "/") {
		_, currentName = toolbox.URLSplit(r.URL[:len(r.URL)-1])
		currentName += "/"
	}

	r.URL = strings.Replace(r.URL, currentName, name, 1)
	r.ParsedURL, err = url.Parse(r.URL)
	return err
}

//JSONDecode decodes json resource into target
func (r *Resource) JSONDecode(target interface{}) error {
	return r.DecodeWith(target, toolbox.NewJSONDecoderFactory())
}

//JSONDecode decodes yaml resource into target
func (r *Resource) YAMLDecode(target interface{}) error {
	if interfacePrt, ok := target.(*interface{}); ok {
		var data interface{}
		if err := r.DecodeWith(&data, toolbox.NewYamlDecoderFactory()); err != nil {
			return err
		}
		if toolbox.IsSlice(data) {
			*interfacePrt = data
			return nil
		}
	}
	var mapSlice = yaml.MapSlice{}
	if err := r.DecodeWith(&mapSlice, toolbox.NewYamlDecoderFactory()); err != nil {
		return err
	}
	if !toolbox.IsMap(target) {
		return toolbox.DefaultConverter.AssignConverted(target, mapSlice)
	}
	resultMap := toolbox.AsMap(target)
	for _, v := range mapSlice {
		resultMap[toolbox.AsString(v.Key)] = v.Value
	}
	return nil
}

func (r *Resource) readFromCache() []byte {
	if toolbox.FileExists(r.Cache) {
		info, err := os.Stat(r.Cache)
		var isExpired = false
		if err == nil && r.CacheExpiryMs > 0 {
			elapsed := time.Now().Sub(info.ModTime())
			isExpired = elapsed > time.Second*time.Duration(r.CacheExpiryMs)
		}
		content, err := ioutil.ReadFile(r.Cache)
		if err == nil && !isExpired {
			return content
		}
	}
	return nil
}

//Cachable returns true if resource is cachable
func (r *Resource) Cachable() bool {
	return r.Cache != ""
}

func computeResourceModificationTag(resource *Resource) (int64, error) {
	service, err := storage.NewServiceForURL(resource.URL, resource.Credentials)
	if err != nil {
		return 0, err
	}
	object, err := service.StorageObject(resource.URL)
	if err != nil {
		return 0, err
	}
	var fileInfo = object.FileInfo()

	if object.IsContent() {
		return fileInfo.Size() + fileInfo.ModTime().UnixNano(), nil
	}
	var result int64 = 0
	objects, err := service.List(resource.URL)
	if err != nil {
		return 0, err
	}
	for _, object := range objects {
		objectResource := NewResource(object.URL())
		if objectResource.ParsedURL.Path == resource.ParsedURL.Path {
			continue
		}
		modificationTag, err := computeResourceModificationTag(NewResource(object.URL(), resource.Credentials))
		if err != nil {
			return 0, err
		}
		result += modificationTag

	}
	return result, nil
}

func (r *Resource) HasChanged() (changed bool, err error) {
	if r.modificationTag == 0 {
		r.modificationTag, err = computeResourceModificationTag(r)
		return false, err
	}
	var recentModificationTag int64
	recentModificationTag, err = computeResourceModificationTag(r)
	if err != nil {
		return false, err
	}
	if recentModificationTag != r.modificationTag {
		changed = true
		r.modificationTag = recentModificationTag
	}
	return changed, err
}

func normalizeURL(URL string) string {
	if strings.Contains(URL, "://") {
		var protoPosition = strings.Index(URL, "://")
		if protoPosition != -1 {
			var urlSuffix = string(URL[protoPosition+3:])
			urlSuffix = strings.Replace(urlSuffix, "//", "/", len(urlSuffix))
			URL = string(URL[:protoPosition+3]) + urlSuffix
		}
		return URL
	}
	if !strings.HasPrefix(URL, "/") {
		currentDirectory, _ := os.Getwd()

		if strings.Contains(URL, "..") {
			fragments := strings.Split(URL, "/")
			var index = 0
			var offset = 0
			if fragments[0] == "." {
				offset = 1
			}

			for index = offset; index < len(fragments); index++ {
				var fragment = fragments[index]
				if fragment == ".." {
					currentDirectory, _ = path.Split(currentDirectory)
					if strings.HasSuffix(currentDirectory, "/") {
						currentDirectory = string(currentDirectory[:len(currentDirectory)-1])
					}
					continue
				}
				break
			}
			return toolbox.FileSchema + path.Join(currentDirectory, strings.Join(fragments[index:], "/"))
		}

		currentDirectory, err := os.Getwd()
		if err == nil {
			candidate := path.Join(currentDirectory, URL)
			URL = candidate
		}
	}
	return toolbox.FileSchema + URL
}

func (r *Resource) Init() (err error) {
	if r.init {
		return nil
	}
	r.init = true
	r.URL = normalizeURL(r.URL)
	r.ParsedURL, err = url.Parse(r.URL)
	return err
}

//NewResource returns a new resource for provided URL, followed by optional credential, cache and cache expiryMs.
func NewResource(Params ...interface{}) *Resource {
	if len(Params) == 0 {
		return nil
	}
	var URL = toolbox.AsString(Params[0])
	URL = normalizeURL(URL)

	var credential string
	if len(Params) > 1 {
		credential = toolbox.AsString(Params[1])
	}
	var cache string
	if len(Params) > 2 {
		cache = toolbox.AsString(Params[2])
	}
	var cacheExpiryMs int
	if len(Params) > 3 {
		cacheExpiryMs = toolbox.AsInt(Params[3])
	}
	parsedURL, _ := url.Parse(URL)
	return &Resource{
		init:          true,
		ParsedURL:     parsedURL,
		URL:           URL,
		Credentials:   credential,
		Cache:         cache,
		CacheExpiryMs: cacheExpiryMs,
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kola

import (
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/coreos/pkg/capnslog"

	"github.com/coreos/mantle/harness"
	"github.com/coreos/mantle/kola/cluster"
	"github.com/coreos/mantle/kola/register"
	"github.com/coreos/mantle/kola/skip"
	"github.com/coreos/mantle/platform"
	awsapi "github.com/coreos/mantle/platform/api/aws"
	gcloudapi "github.com/coreos/mantle/platform/api/gcloud"
	"github.com/coreos/mantle/platform/machine/aws"
	"github.com/coreos/mantle/platform/machine/gcloud"
	"github.com/coreos/mantle/platform/machine/qemu"
	"github.com/coreos/mantle/system"
)

var (
	plog = capnslog.NewPackageLogger("github.com/coreos/mantle", "kola")

	Options     = platform.Options{}
	QEMUOptions = qemu.Options{Options: &Options}      // glue to set platform options from main
	GCEOptions  = gcloudapi.Options{Options: &Options} // glue to set platform options from main
	AWSOptions  = awsapi.Options{Options: &Options}    // glue to set platform options from main

	TestParallelism int    //glue var to set test parallelism from main
	TAPFile         string // if not "", write TAP results here

	testOptions = make(map[string]string, 0)
)

// RegisterTestOption registers any options that need visibility inside
// a Test. Panics if existing option is already registered. Each test
// has global view of options.
func RegisterTestOption(name, option string) {
	_, ok := testOptions[name]
	if ok {
		panic("test option already registered with same name")
	}
	testOptions[name] = option
}

// NativeRunner is a closure passed to all kola test functions and used
// to run native go functions directly on kola machines. It is necessary
// glue until kola does introspection.
type NativeRunner func(funcName string, m platform.Machine) error

func filterTests(tests map[string]*register.Test, pattern, platform string, version semver.Version) (map[string]*register.Test, error) {
	r := make(map[string]*register.Test)

	for name, t := range tests {
		match, err := filepath.Match(pattern, t.Name)
		if err != nil {
			return nil, err
		}
		if !match {
			continue
		}

		// Skip the test if Manual is set and the name doesn't fully match.
		if t.Manual && t.Name != pattern {
			continue
		}

		// Check the test's min and end versions when running more then one test
		if t.Name != pattern && versionOutsideRange(version, t.MinVersion, t.EndVersion) {
			continue
		}

		allowed := true
		for _, p := range t.Platforms {
			if p == platform {
				allowed = true
				break
			} else {
				allowed = false
			}
		}
		if !allowed {
			continue
		}

		r[name] = t
	}

	return r, nil
}

// versionOutsideRange checks to see if version is outside [min, end). If end
// is a zero value, it is ignored and there is no upper bound. If version is a
// zero value, the bounds are ignored.
func versionOutsideRange(version, minVersion, endVersion semver.Version) bool {
	if version == (semver.Version{}) {
		return false
	}

	if version.LessThan(minVersion) {
		return true
	}

	if (endVersion != semver.Version{}) && !version.LessThan(endVersion) {
		return true
	}

	return false
}

// RunTests is a harness for running multiple tests in parallel. Filters
// tests based on a glob pattern and by platform. Has access to all
// tests either registered in this package or by imported packages that
// register tests in their init() function.
// outputDir is where various test logs and data will be written for
// analysis after the test run. If it already exists it will be erased!
func RunTests(pattern, pltfrm, outputDir string) error {
	// Avoid incurring cost of starting machine in getClusterSemver when
	// either:
	// 1) we already know 0 tests will run
	// 2) glob is an exact match which means minVersion will be ignored
	//    either way
	tests, err := filterTests(register.Tests, pattern, pltfrm, semver.Version{})
	if err != nil {
		plog.Fatal(err)
	}

	var skipGetVersion bool
	if len(tests) == 0 {
		skipGetVersion = true
	} else if len(tests) == 1 {
		for name := range tests {
			if name == pattern {
				skipGetVersion = true
			}
		}
	}

	if !skipGetVersion {
		version, err := getClusterSemver(pltfrm, outputDir)
		if err != nil {
			plog.Fatal(err)
		}

		// one more filter pass now that we know real version
		tests, err = filterTests(tests, pattern, pltfrm, *version)
		if err != nil {
			plog.Fatal(err)
		}
	}

	opts := harness.Options{
		OutputDir: outputDir,
		Parallel:  TestParallelism,
		Verbose:   true,
	}
	htests := []harness.InternalTest{}
	for _, test := range tests {
		test := test // for the closure
		run := func(h *harness.H) {
			// don't go too fast, in case we're talking to a rate limiting api like AWS EC2.
			time.Sleep(2 * time.Second)

			err := RunTest(test, pltfrm, outputDir)
			if _, ok := err.(skip.Skip); ok {
				h.Skip(err)
			} else if err != nil {
				h.Error(err)
			}
		}
		htests = append(htests, harness.InternalTest{Name: test.Name, F: run})
	}

	suite := harness.NewSuite(opts, htests)
	err = suite.Run()

	if TAPFile != "" {
		src := filepath.Join(outputDir, "test.tap")
		if err2 := system.CopyRegularFile(src, TAPFile); err == nil && err2 != nil {
			err = err2
		}
	}

	if err != nil {
		fmt.Println("FAIL")
	} else {
		fmt.Println("PASS")
	}

	return err
}

// getClusterSemVer returns the CoreOS semantic version via starting a
// machine and checking
func getClusterSemver(pltfrm, outputDir string) (*semver.Version, error) {
	var err error
	var cluster platform.Cluster

	testDir := filepath.Join(outputDir, "get_cluster_semver")
	if err := os.MkdirAll(testDir, 0777); err != nil {
		return nil, err
	}

	switch pltfrm {
	case "qemu":
		cluster, err = qemu.NewCluster(&QEMUOptions, testDir)
	case "gce":
		cluster, err = gcloud.NewCluster(&GCEOptions, testDir)
	case "aws":
		cluster, err = aws.NewCluster(&AWSOptions, testDir)
	default:
		err = fmt.Errorf("invalid platform %q", pltfrm)
	}

	if err != nil {
		return nil, fmt.Errorf("creating cluster for semver check: %v", err)
	}
	defer func() {
		if err := cluster.Destroy(); err != nil {
			plog.Errorf("cluster.Destroy(): %v", err)
		}
	}()

	m, err := cluster.NewMachine("#cloud-config")
	if err != nil {
		return nil, fmt.Errorf("creating new machine for semver check: %v", err)
	}

	out, err := m.SSH("grep ^VERSION_ID= /etc/os-release")
	if err != nil {
		return nil, fmt.Errorf("parsing /etc/os-release: %v", err)
	}

	version, err := semver.NewVersion(strings.Split(string(out), "=")[1])
	if err != nil {
		return nil, fmt.Errorf("parsing os-release semver: %v", err)
	}

	return version, nil
}

// RunTest is a harness for running a single test. It is used by
// RunTests but can also be used directly by binaries that aim to run a
// single test. Using RunTest directly means that TestCluster flags used
// to filter out tests such as 'Platforms', 'Manual', or 'MinVersion'
// are not respected.
// outputDir is where various test logs and data will be written for
// analysis after the test run. It should already exist.
func RunTest(t *register.Test, pltfrm, outputDir string) (err error) {
	var c platform.Cluster

	testDir := filepath.Join(outputDir, t.Name)
	if err := os.MkdirAll(testDir, 0777); err != nil {
		return err
	}

	switch pltfrm {
	case "qemu":
		c, err = qemu.NewCluster(&QEMUOptions, testDir)
	case "gce":
		c, err = gcloud.NewCluster(&GCEOptions, testDir)
	case "aws":
		c, err = aws.NewCluster(&AWSOptions, testDir)
	default:
		err = fmt.Errorf("invalid platform %q", pltfrm)
	}

	if err != nil {
		return fmt.Errorf("Cluster failed: %v", err)
	}
	defer func() {
		if err := c.Destroy(); err != nil {
			plog.Errorf("cluster.Destroy(): %v", err)
		}
	}()

	url, err := c.GetDiscoveryURL(t.ClusterSize)
	if err != nil {
		return fmt.Errorf("Failed to create discovery endpoint: %v", err)
	}

	cfgs := MakeConfigs(url, t.UserData, t.ClusterSize)

	if t.ClusterSize > 0 {
		_, err := platform.NewMachines(c, cfgs)
		if err != nil {
			return fmt.Errorf("Cluster failed starting machines: %v", err)
		}
	}

	// pass along all registered native functions
	var names []string
	for k := range t.NativeFuncs {
		names = append(names, k)
	}

	// prevent unsafe access if tests ever become parallel and access
	tempTestOptions := make(map[string]string, 0)
	for k, v := range testOptions {
		tempTestOptions[k] = v
	}

	// Cluster -> TestCluster
	tcluster := cluster.TestCluster{
		Name:        t.Name,
		NativeFuncs: names,
		Options:     tempTestOptions,
		Cluster:     c,
	}

	// drop kolet binary on machines
	if t.NativeFuncs != nil {
		nativeArch := "amd64"
		if pltfrm == "qemu" && QEMUOptions.Board != "" {
			nativeArch = strings.SplitN(QEMUOptions.Board, "-", 2)[0]
		}

		err = scpKolet(tcluster, nativeArch)
		if err != nil {
			return fmt.Errorf("dropping kolet binary: %v", err)
		}
	}

	defer func() {
		r := recover()
		switch r := r.(type) {
		case nil:
			// no-op
		case error:
			err = r
		default:
			err = fmt.Errorf("test panicked: %v", r)
		}

		// give some time for the remote journal to be flushed so it can be read
		// before we run the deferred machine destruction
		if err != nil {
			time.Sleep(2 * time.Second)
		}
	}()

	// run test
	return t.Run(tcluster)
}

// scpKolet searches for a kolet binary and copies it to the machine.
func scpKolet(t cluster.TestCluster, mArch string) error {
	for _, d := range []string{
		".",
		filepath.Dir(os.Args[0]),
		filepath.Join(filepath.Dir(os.Args[0]), mArch),
		filepath.Join("/usr/lib/kola", mArch),
	} {
		kolet := filepath.Join(d, "kolet")
		if _, err := os.Stat(kolet); err == nil {
			return t.DropFile(kolet)
		}
	}
	return fmt.Errorf("Unable to locate kolet binary for %s", mArch)
}

// replaces $discovery with discover url in etcd cloud config and
// replaces $name with a unique name
func MakeConfigs(url, cfg string, csize int) []string {
	cfg = strings.Replace(cfg, "$discovery", url, -1)

	var cfgs []string
	for i := 0; i < csize; i++ {
		cfgs = append(cfgs, strings.Replace(cfg, "$name", "instance"+strconv.Itoa(i), -1))
	}
	return cfgs
}

// CleanOutputDir creates an empty directory, any existing data will be wiped!
func CleanOutputDir(outputDir string) (string, error) {
	outputDir = filepath.Clean(outputDir)
	if outputDir == "." {
		return "", fmt.Errorf("kola: missing output directory path")
	}
	if err := os.RemoveAll(outputDir); err != nil {
		return "", err
	}
	if err := os.MkdirAll(outputDir, 0777); err != nil {
		return "", err
	}
	return outputDir, nil
}
<commit_msg>kola: restore parallel test execution<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kola

import (
	"fmt"
	"math/rand"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/coreos/pkg/capnslog"

	"github.com/coreos/mantle/harness"
	"github.com/coreos/mantle/kola/cluster"
	"github.com/coreos/mantle/kola/register"
	"github.com/coreos/mantle/kola/skip"
	"github.com/coreos/mantle/platform"
	awsapi "github.com/coreos/mantle/platform/api/aws"
	gcloudapi "github.com/coreos/mantle/platform/api/gcloud"
	"github.com/coreos/mantle/platform/machine/aws"
	"github.com/coreos/mantle/platform/machine/gcloud"
	"github.com/coreos/mantle/platform/machine/qemu"
	"github.com/coreos/mantle/system"
)

var (
	plog = capnslog.NewPackageLogger("github.com/coreos/mantle", "kola")

	Options     = platform.Options{}
	QEMUOptions = qemu.Options{Options: &Options}      // glue to set platform options from main
	GCEOptions  = gcloudapi.Options{Options: &Options} // glue to set platform options from main
	AWSOptions  = awsapi.Options{Options: &Options}    // glue to set platform options from main

	TestParallelism int    //glue var to set test parallelism from main
	TAPFile         string // if not "", write TAP results here

	testOptions = make(map[string]string, 0)
)

// RegisterTestOption registers any options that need visibility inside
// a Test. Panics if existing option is already registered. Each test
// has global view of options.
func RegisterTestOption(name, option string) {
	_, ok := testOptions[name]
	if ok {
		panic("test option already registered with same name")
	}
	testOptions[name] = option
}

// NativeRunner is a closure passed to all kola test functions and used
// to run native go functions directly on kola machines. It is necessary
// glue until kola does introspection.
type NativeRunner func(funcName string, m platform.Machine) error

func filterTests(tests map[string]*register.Test, pattern, platform string, version semver.Version) (map[string]*register.Test, error) {
	r := make(map[string]*register.Test)

	for name, t := range tests {
		match, err := filepath.Match(pattern, t.Name)
		if err != nil {
			return nil, err
		}
		if !match {
			continue
		}

		// Skip the test if Manual is set and the name doesn't fully match.
		if t.Manual && t.Name != pattern {
			continue
		}

		// Check the test's min and end versions when running more then one test
		if t.Name != pattern && versionOutsideRange(version, t.MinVersion, t.EndVersion) {
			continue
		}

		allowed := true
		for _, p := range t.Platforms {
			if p == platform {
				allowed = true
				break
			} else {
				allowed = false
			}
		}
		if !allowed {
			continue
		}

		r[name] = t
	}

	return r, nil
}

// versionOutsideRange checks to see if version is outside [min, end). If end
// is a zero value, it is ignored and there is no upper bound. If version is a
// zero value, the bounds are ignored.
func versionOutsideRange(version, minVersion, endVersion semver.Version) bool {
	if version == (semver.Version{}) {
		return false
	}

	if version.LessThan(minVersion) {
		return true
	}

	if (endVersion != semver.Version{}) && !version.LessThan(endVersion) {
		return true
	}

	return false
}

// RunTests is a harness for running multiple tests in parallel. Filters
// tests based on a glob pattern and by platform. Has access to all
// tests either registered in this package or by imported packages that
// register tests in their init() function.
// outputDir is where various test logs and data will be written for
// analysis after the test run. If it already exists it will be erased!
func RunTests(pattern, pltfrm, outputDir string) error {
	// Avoid incurring cost of starting machine in getClusterSemver when
	// either:
	// 1) we already know 0 tests will run
	// 2) glob is an exact match which means minVersion will be ignored
	//    either way
	tests, err := filterTests(register.Tests, pattern, pltfrm, semver.Version{})
	if err != nil {
		plog.Fatal(err)
	}

	var skipGetVersion bool
	if len(tests) == 0 {
		skipGetVersion = true
	} else if len(tests) == 1 {
		for name := range tests {
			if name == pattern {
				skipGetVersion = true
			}
		}
	}

	if !skipGetVersion {
		version, err := getClusterSemver(pltfrm, outputDir)
		if err != nil {
			plog.Fatal(err)
		}

		// one more filter pass now that we know real version
		tests, err = filterTests(tests, pattern, pltfrm, *version)
		if err != nil {
			plog.Fatal(err)
		}
	}

	opts := harness.Options{
		OutputDir: outputDir,
		Parallel:  TestParallelism,
		Verbose:   true,
	}
	htests := []harness.InternalTest{}
	for _, test := range tests {
		test := test // for the closure
		run := func(h *harness.H) {
			h.Parallel()

			// don't go too fast, in case we're talking to a rate limiting api like AWS EC2.
			// FIXME(marineam): API requests must do their own
			// backoff due to rate limiting, this is unreliable.
			max := int64(2 * time.Second)
			splay := time.Duration(rand.Int63n(max))
			time.Sleep(splay)

			err := RunTest(test, pltfrm, outputDir)
			if _, ok := err.(skip.Skip); ok {
				h.Skip(err)
			} else if err != nil {
				h.Error(err)
			}
		}
		htests = append(htests, harness.InternalTest{Name: test.Name, F: run})
	}

	suite := harness.NewSuite(opts, htests)
	err = suite.Run()

	if TAPFile != "" {
		src := filepath.Join(outputDir, "test.tap")
		if err2 := system.CopyRegularFile(src, TAPFile); err == nil && err2 != nil {
			err = err2
		}
	}

	if err != nil {
		fmt.Println("FAIL")
	} else {
		fmt.Println("PASS")
	}

	return err
}

// getClusterSemVer returns the CoreOS semantic version via starting a
// machine and checking
func getClusterSemver(pltfrm, outputDir string) (*semver.Version, error) {
	var err error
	var cluster platform.Cluster

	testDir := filepath.Join(outputDir, "get_cluster_semver")
	if err := os.MkdirAll(testDir, 0777); err != nil {
		return nil, err
	}

	switch pltfrm {
	case "qemu":
		cluster, err = qemu.NewCluster(&QEMUOptions, testDir)
	case "gce":
		cluster, err = gcloud.NewCluster(&GCEOptions, testDir)
	case "aws":
		cluster, err = aws.NewCluster(&AWSOptions, testDir)
	default:
		err = fmt.Errorf("invalid platform %q", pltfrm)
	}

	if err != nil {
		return nil, fmt.Errorf("creating cluster for semver check: %v", err)
	}
	defer func() {
		if err := cluster.Destroy(); err != nil {
			plog.Errorf("cluster.Destroy(): %v", err)
		}
	}()

	m, err := cluster.NewMachine("#cloud-config")
	if err != nil {
		return nil, fmt.Errorf("creating new machine for semver check: %v", err)
	}

	out, err := m.SSH("grep ^VERSION_ID= /etc/os-release")
	if err != nil {
		return nil, fmt.Errorf("parsing /etc/os-release: %v", err)
	}

	version, err := semver.NewVersion(strings.Split(string(out), "=")[1])
	if err != nil {
		return nil, fmt.Errorf("parsing os-release semver: %v", err)
	}

	return version, nil
}

// RunTest is a harness for running a single test. It is used by
// RunTests but can also be used directly by binaries that aim to run a
// single test. Using RunTest directly means that TestCluster flags used
// to filter out tests such as 'Platforms', 'Manual', or 'MinVersion'
// are not respected.
// outputDir is where various test logs and data will be written for
// analysis after the test run. It should already exist.
func RunTest(t *register.Test, pltfrm, outputDir string) (err error) {
	var c platform.Cluster

	testDir := filepath.Join(outputDir, t.Name)
	if err := os.MkdirAll(testDir, 0777); err != nil {
		return err
	}

	switch pltfrm {
	case "qemu":
		c, err = qemu.NewCluster(&QEMUOptions, testDir)
	case "gce":
		c, err = gcloud.NewCluster(&GCEOptions, testDir)
	case "aws":
		c, err = aws.NewCluster(&AWSOptions, testDir)
	default:
		err = fmt.Errorf("invalid platform %q", pltfrm)
	}

	if err != nil {
		return fmt.Errorf("Cluster failed: %v", err)
	}
	defer func() {
		if err := c.Destroy(); err != nil {
			plog.Errorf("cluster.Destroy(): %v", err)
		}
	}()

	url, err := c.GetDiscoveryURL(t.ClusterSize)
	if err != nil {
		return fmt.Errorf("Failed to create discovery endpoint: %v", err)
	}

	cfgs := MakeConfigs(url, t.UserData, t.ClusterSize)

	if t.ClusterSize > 0 {
		_, err := platform.NewMachines(c, cfgs)
		if err != nil {
			return fmt.Errorf("Cluster failed starting machines: %v", err)
		}
	}

	// pass along all registered native functions
	var names []string
	for k := range t.NativeFuncs {
		names = append(names, k)
	}

	// prevent unsafe access if tests ever become parallel and access
	tempTestOptions := make(map[string]string, 0)
	for k, v := range testOptions {
		tempTestOptions[k] = v
	}

	// Cluster -> TestCluster
	tcluster := cluster.TestCluster{
		Name:        t.Name,
		NativeFuncs: names,
		Options:     tempTestOptions,
		Cluster:     c,
	}

	// drop kolet binary on machines
	if t.NativeFuncs != nil {
		nativeArch := "amd64"
		if pltfrm == "qemu" && QEMUOptions.Board != "" {
			nativeArch = strings.SplitN(QEMUOptions.Board, "-", 2)[0]
		}

		err = scpKolet(tcluster, nativeArch)
		if err != nil {
			return fmt.Errorf("dropping kolet binary: %v", err)
		}
	}

	defer func() {
		r := recover()
		switch r := r.(type) {
		case nil:
			// no-op
		case error:
			err = r
		default:
			err = fmt.Errorf("test panicked: %v", r)
		}

		// give some time for the remote journal to be flushed so it can be read
		// before we run the deferred machine destruction
		if err != nil {
			time.Sleep(2 * time.Second)
		}
	}()

	// run test
	return t.Run(tcluster)
}

// scpKolet searches for a kolet binary and copies it to the machine.
func scpKolet(t cluster.TestCluster, mArch string) error {
	for _, d := range []string{
		".",
		filepath.Dir(os.Args[0]),
		filepath.Join(filepath.Dir(os.Args[0]), mArch),
		filepath.Join("/usr/lib/kola", mArch),
	} {
		kolet := filepath.Join(d, "kolet")
		if _, err := os.Stat(kolet); err == nil {
			return t.DropFile(kolet)
		}
	}
	return fmt.Errorf("Unable to locate kolet binary for %s", mArch)
}

// replaces $discovery with discover url in etcd cloud config and
// replaces $name with a unique name
func MakeConfigs(url, cfg string, csize int) []string {
	cfg = strings.Replace(cfg, "$discovery", url, -1)

	var cfgs []string
	for i := 0; i < csize; i++ {
		cfgs = append(cfgs, strings.Replace(cfg, "$name", "instance"+strconv.Itoa(i), -1))
	}
	return cfgs
}

// CleanOutputDir creates an empty directory, any existing data will be wiped!
func CleanOutputDir(outputDir string) (string, error) {
	outputDir = filepath.Clean(outputDir)
	if outputDir == "." {
		return "", fmt.Errorf("kola: missing output directory path")
	}
	if err := os.RemoveAll(outputDir); err != nil {
		return "", err
	}
	if err := os.MkdirAll(outputDir, 0777); err != nil {
		return "", err
	}
	return outputDir, nil
}
<|endoftext|>


<commit_before>// Copyright 2014, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package planbuilder

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path"
	"strings"
	"testing"

	"github.com/youtube/vitess/go/testfiles"
	"github.com/youtube/vitess/go/vt/sqlparser"
	"github.com/youtube/vitess/go/vt/vtgate/vindexes"
)

// hashIndex satisfies Functional, Unique.
type hashIndex struct{ name string }

func (v *hashIndex) String() string { return v.name }
func (*hashIndex) Cost() int        { return 1 }
func (*hashIndex) Verify(vindexes.VCursor, []interface{}, [][]byte) (bool, error) {
	return false, nil
}
func (*hashIndex) Map(vindexes.VCursor, []interface{}) ([][]byte, error) { return nil, nil }
func (*hashIndex) Create(vindexes.VCursor, []interface{}) error          { return nil }
func (*hashIndex) Delete(vindexes.VCursor, []interface{}, []byte) error  { return nil }

func newHashIndex(name string, _ map[string]string) (vindexes.Vindex, error) {
	return &hashIndex{name: name}, nil
}

// lookupIndex satisfies Lookup, Unique.
type lookupIndex struct{ name string }

func (v *lookupIndex) String() string { return v.name }
func (*lookupIndex) Cost() int        { return 2 }
func (*lookupIndex) Verify(vindexes.VCursor, []interface{}, [][]byte) (bool, error) {
	return false, nil
}
func (*lookupIndex) Map(vindexes.VCursor, []interface{}) ([][]byte, error)  { return nil, nil }
func (*lookupIndex) Create(vindexes.VCursor, []interface{}, [][]byte) error { return nil }
func (*lookupIndex) Delete(vindexes.VCursor, []interface{}, []byte) error   { return nil }

func newLookupIndex(name string, _ map[string]string) (vindexes.Vindex, error) {
	return &lookupIndex{name: name}, nil
}

// multiIndex satisfies Lookup, NonUnique.
type multiIndex struct{ name string }

func (v *multiIndex) String() string { return v.name }
func (*multiIndex) Cost() int        { return 3 }
func (*multiIndex) Verify(vindexes.VCursor, []interface{}, [][]byte) (bool, error) {
	return false, nil
}
func (*multiIndex) Map(vindexes.VCursor, []interface{}) ([][][]byte, error) { return nil, nil }
func (*multiIndex) Create(vindexes.VCursor, []interface{}, [][]byte) error  { return nil }
func (*multiIndex) Delete(vindexes.VCursor, []interface{}, []byte) error    { return nil }

func newMultiIndex(name string, _ map[string]string) (vindexes.Vindex, error) {
	return &multiIndex{name: name}, nil
}

// costlyIndex satisfies Lookup, NonUnique.
type costlyIndex struct{ name string }

func (v *costlyIndex) String() string { return v.name }
func (*costlyIndex) Cost() int        { return 10 }
func (*costlyIndex) Verify(vindexes.VCursor, []interface{}, [][]byte) (bool, error) {
	return false, nil
}
func (*costlyIndex) Map(vindexes.VCursor, []interface{}) ([][][]byte, error) { return nil, nil }
func (*costlyIndex) Create(vindexes.VCursor, []interface{}, [][]byte) error  { return nil }
func (*costlyIndex) Delete(vindexes.VCursor, []interface{}, []byte) error    { return nil }

func newCostlyIndex(name string, _ map[string]string) (vindexes.Vindex, error) {
	return &costlyIndex{name: name}, nil
}

func init() {
	vindexes.Register("hash_test", newHashIndex)
	vindexes.Register("lookup_test", newLookupIndex)
	vindexes.Register("multi", newMultiIndex)
	vindexes.Register("costly", newCostlyIndex)
}

func TestPlan(t *testing.T) {
	vschema := loadSchema(t, "schema_test.json")

	testFile(t, "from_cases.txt", vschema)
	testFile(t, "filter_cases.txt", vschema)
	testFile(t, "select_cases.txt", vschema)
	testFile(t, "postprocess_cases.txt", vschema)
	testFile(t, "wireup_cases.txt", vschema)
	testFile(t, "dml_cases.txt", vschema)
	testFile(t, "unsupported_cases.txt", vschema)
}

func TestOne(t *testing.T) {
	vschema := loadSchema(t, "schema_test.json")
	testFile(t, "onecase.txt", vschema)
}

func loadSchema(t *testing.T, filename string) *vindexes.VSchema {
	formal, err := vindexes.LoadFormal(locateFile(filename))
	if err != nil {
		t.Fatal(err)
	}
	vschema, err := vindexes.BuildVSchema(formal)
	if err != nil {
		t.Fatal(err)
	}
	return vschema
}

type vschemaWrapper struct {
	v *vindexes.VSchema
}

func (vw *vschemaWrapper) Find(ks, tab sqlparser.TableIdent) (*vindexes.Table, error) {
	return vw.v.Find(ks.String(), tab.String())
}

func testFile(t *testing.T, filename string, vschema *vindexes.VSchema) {
	for tcase := range iterateExecFile(filename) {
		plan, err := Build(tcase.input, &vschemaWrapper{
			v: vschema,
		})
		var out string
		if err != nil {
			out = err.Error()
		} else {
			bout, _ := json.Marshal(plan)
			out = string(bout)
		}
		if out != tcase.output {
			t.Errorf("File: %s, Line:%v\n%s\n%s", filename, tcase.lineno, tcase.output, out)
			// Uncomment these lines to re-generate input files
			if err != nil {
				out = fmt.Sprintf("\"%s\"", out)
			} else {
				bout, _ := json.MarshalIndent(plan, "", "  ")
				out = string(bout)
			}
			fmt.Printf("%s\"%s\"\n%s\n\n", tcase.comments, tcase.input, out)
		}
	}
}

type testCase struct {
	file     string
	lineno   int
	input    string
	output   string
	comments string
}

func iterateExecFile(name string) (testCaseIterator chan testCase) {
	name = locateFile(name)
	fd, err := os.OpenFile(name, os.O_RDONLY, 0)
	if err != nil {
		panic(fmt.Sprintf("Could not open file %s", name))
	}
	testCaseIterator = make(chan testCase)
	var comments string
	go func() {
		defer close(testCaseIterator)

		r := bufio.NewReader(fd)
		lineno := 0
		for {
			binput, err := r.ReadBytes('\n')
			if err != nil {
				if err != io.EOF {
					fmt.Printf("Line: %d\n", lineno)
					panic(fmt.Errorf("error reading file %s: %s", name, err.Error()))
				}
				break
			}
			lineno++
			input := string(binput)
			if input == "" || input == "\n" || strings.HasPrefix(input, "Length:") {
				continue
			}
			if input[0] == '#' {
				comments = comments + input
				continue
			}
			err = json.Unmarshal(binput, &input)
			if err != nil {
				fmt.Printf("Line: %d, input: %s\n", lineno, binput)
				panic(err)
			}
			input = strings.Trim(input, "\"")
			var output []byte
			for {
				l, err := r.ReadBytes('\n')
				lineno++
				if err != nil {
					fmt.Printf("Line: %d\n", lineno)
					panic(fmt.Errorf("error reading file %s: %s", name, err.Error()))
				}
				output = append(output, l...)
				if l[0] == '}' {
					output = output[:len(output)-1]
					b := bytes.NewBuffer(make([]byte, 0, 64))
					if err := json.Compact(b, output); err == nil {
						output = b.Bytes()
					}
					break
				}
				if l[0] == '"' {
					output = output[1 : len(output)-2]
					break
				}
			}
			testCaseIterator <- testCase{
				file:     name,
				lineno:   lineno,
				input:    input,
				output:   string(output),
				comments: comments,
			}
			comments = ""
		}
	}()
	return testCaseIterator
}

func locateFile(name string) string {
	if path.IsAbs(name) {
		return name
	}
	return testfiles.Locate("vtgate/" + name)
}
<commit_msg>panic if a planbuilder test input contains invalid JSON (#2648)<commit_after>// Copyright 2014, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package planbuilder

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path"
	"strings"
	"testing"

	"github.com/youtube/vitess/go/testfiles"
	"github.com/youtube/vitess/go/vt/sqlparser"
	"github.com/youtube/vitess/go/vt/vtgate/vindexes"
)

// hashIndex satisfies Functional, Unique.
type hashIndex struct{ name string }

func (v *hashIndex) String() string { return v.name }
func (*hashIndex) Cost() int        { return 1 }
func (*hashIndex) Verify(vindexes.VCursor, []interface{}, [][]byte) (bool, error) {
	return false, nil
}
func (*hashIndex) Map(vindexes.VCursor, []interface{}) ([][]byte, error) { return nil, nil }
func (*hashIndex) Create(vindexes.VCursor, []interface{}) error          { return nil }
func (*hashIndex) Delete(vindexes.VCursor, []interface{}, []byte) error  { return nil }

func newHashIndex(name string, _ map[string]string) (vindexes.Vindex, error) {
	return &hashIndex{name: name}, nil
}

// lookupIndex satisfies Lookup, Unique.
type lookupIndex struct{ name string }

func (v *lookupIndex) String() string { return v.name }
func (*lookupIndex) Cost() int        { return 2 }
func (*lookupIndex) Verify(vindexes.VCursor, []interface{}, [][]byte) (bool, error) {
	return false, nil
}
func (*lookupIndex) Map(vindexes.VCursor, []interface{}) ([][]byte, error)  { return nil, nil }
func (*lookupIndex) Create(vindexes.VCursor, []interface{}, [][]byte) error { return nil }
func (*lookupIndex) Delete(vindexes.VCursor, []interface{}, []byte) error   { return nil }

func newLookupIndex(name string, _ map[string]string) (vindexes.Vindex, error) {
	return &lookupIndex{name: name}, nil
}

// multiIndex satisfies Lookup, NonUnique.
type multiIndex struct{ name string }

func (v *multiIndex) String() string { return v.name }
func (*multiIndex) Cost() int        { return 3 }
func (*multiIndex) Verify(vindexes.VCursor, []interface{}, [][]byte) (bool, error) {
	return false, nil
}
func (*multiIndex) Map(vindexes.VCursor, []interface{}) ([][][]byte, error) { return nil, nil }
func (*multiIndex) Create(vindexes.VCursor, []interface{}, [][]byte) error  { return nil }
func (*multiIndex) Delete(vindexes.VCursor, []interface{}, []byte) error    { return nil }

func newMultiIndex(name string, _ map[string]string) (vindexes.Vindex, error) {
	return &multiIndex{name: name}, nil
}

// costlyIndex satisfies Lookup, NonUnique.
type costlyIndex struct{ name string }

func (v *costlyIndex) String() string { return v.name }
func (*costlyIndex) Cost() int        { return 10 }
func (*costlyIndex) Verify(vindexes.VCursor, []interface{}, [][]byte) (bool, error) {
	return false, nil
}
func (*costlyIndex) Map(vindexes.VCursor, []interface{}) ([][][]byte, error) { return nil, nil }
func (*costlyIndex) Create(vindexes.VCursor, []interface{}, [][]byte) error  { return nil }
func (*costlyIndex) Delete(vindexes.VCursor, []interface{}, []byte) error    { return nil }

func newCostlyIndex(name string, _ map[string]string) (vindexes.Vindex, error) {
	return &costlyIndex{name: name}, nil
}

func init() {
	vindexes.Register("hash_test", newHashIndex)
	vindexes.Register("lookup_test", newLookupIndex)
	vindexes.Register("multi", newMultiIndex)
	vindexes.Register("costly", newCostlyIndex)
}

func TestPlan(t *testing.T) {
	vschema := loadSchema(t, "schema_test.json")

	testFile(t, "from_cases.txt", vschema)
	testFile(t, "filter_cases.txt", vschema)
	testFile(t, "select_cases.txt", vschema)
	testFile(t, "postprocess_cases.txt", vschema)
	testFile(t, "wireup_cases.txt", vschema)
	testFile(t, "dml_cases.txt", vschema)
	testFile(t, "unsupported_cases.txt", vschema)
}

func TestOne(t *testing.T) {
	vschema := loadSchema(t, "schema_test.json")
	testFile(t, "onecase.txt", vschema)
}

func loadSchema(t *testing.T, filename string) *vindexes.VSchema {
	formal, err := vindexes.LoadFormal(locateFile(filename))
	if err != nil {
		t.Fatal(err)
	}
	vschema, err := vindexes.BuildVSchema(formal)
	if err != nil {
		t.Fatal(err)
	}
	return vschema
}

type vschemaWrapper struct {
	v *vindexes.VSchema
}

func (vw *vschemaWrapper) Find(ks, tab sqlparser.TableIdent) (*vindexes.Table, error) {
	return vw.v.Find(ks.String(), tab.String())
}

func testFile(t *testing.T, filename string, vschema *vindexes.VSchema) {
	for tcase := range iterateExecFile(filename) {
		plan, err := Build(tcase.input, &vschemaWrapper{
			v: vschema,
		})
		var out string
		if err != nil {
			out = err.Error()
		} else {
			bout, _ := json.Marshal(plan)
			out = string(bout)
		}
		if out != tcase.output {
			t.Errorf("File: %s, Line:%v\n%s\n%s", filename, tcase.lineno, tcase.output, out)
			// Uncomment these lines to re-generate input files
			if err != nil {
				out = fmt.Sprintf("\"%s\"", out)
			} else {
				bout, _ := json.MarshalIndent(plan, "", "  ")
				out = string(bout)
			}
			fmt.Printf("%s\"%s\"\n%s\n\n", tcase.comments, tcase.input, out)
		}
	}
}

type testCase struct {
	file     string
	lineno   int
	input    string
	output   string
	comments string
}

func iterateExecFile(name string) (testCaseIterator chan testCase) {
	name = locateFile(name)
	fd, err := os.OpenFile(name, os.O_RDONLY, 0)
	if err != nil {
		panic(fmt.Sprintf("Could not open file %s", name))
	}
	testCaseIterator = make(chan testCase)
	var comments string
	go func() {
		defer close(testCaseIterator)

		r := bufio.NewReader(fd)
		lineno := 0
		for {
			binput, err := r.ReadBytes('\n')
			if err != nil {
				if err != io.EOF {
					fmt.Printf("Line: %d\n", lineno)
					panic(fmt.Errorf("error reading file %s: %s", name, err.Error()))
				}
				break
			}
			lineno++
			input := string(binput)
			if input == "" || input == "\n" || strings.HasPrefix(input, "Length:") {
				continue
			}
			if input[0] == '#' {
				comments = comments + input
				continue
			}
			err = json.Unmarshal(binput, &input)
			if err != nil {
				fmt.Printf("Line: %d, input: %s\n", lineno, binput)
				panic(err)
			}
			input = strings.Trim(input, "\"")
			var output []byte
			for {
				l, err := r.ReadBytes('\n')
				lineno++
				if err != nil {
					fmt.Printf("Line: %d\n", lineno)
					panic(fmt.Errorf("error reading file %s: %s", name, err.Error()))
				}
				output = append(output, l...)
				if l[0] == '}' {
					output = output[:len(output)-1]
					b := bytes.NewBuffer(make([]byte, 0, 64))
					err := json.Compact(b, output)
					if err == nil {
						output = b.Bytes()
					} else {
						panic("Invalid JSON " + string(output) + err.Error())
					}
					break
				}
				if l[0] == '"' {
					output = output[1 : len(output)-2]
					break
				}
			}
			testCaseIterator <- testCase{
				file:     name,
				lineno:   lineno,
				input:    input,
				output:   string(output),
				comments: comments,
			}
			comments = ""
		}
	}()
	return testCaseIterator
}

func locateFile(name string) string {
	if path.IsAbs(name) {
		return name
	}
	return testfiles.Locate("vtgate/" + name)
}
<|endoftext|>