<commit_before>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/dekelund/stdres"

	"github.com/dekelund/unbrokenwing/compiler/definition"
	"github.com/dekelund/unbrokenwing/compiler/feature"
	"github.com/dekelund/unbrokenwing/global"
)

// Foreground colors
const (
	reset   = "\033[00m"
	black   = "\033[30m"
	red     = "\033[31m"
	green   = "\033[32m"
	yellow  = "\033[33m"
	blue    = "\033[34m"
	magenta = "\033[35m"
	cyan    = "\033[36m"
	white   = "\033[37m"
)

const (
	PathSeparator = string(os.PathSeparator)
)

var CWD string = "."

func init() {
	cwd, err := os.Getwd()
	if err != nil {
		log.Panic(err)
	}

	CWD = cwd
}

func main() {
	app := cli.NewApp()
	app.Name = "unbrokenwing"
	app.Usage = "Run behaviour driven tests as Gherik features"
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "pretty",
			Usage: "Print colorised result to STDOUT/STDERR",
		},
		cli.BoolFlag{
			Name:  "debug",
			Usage: "Verbose printing (Generated files will be kept)",
		},
		cli.StringFlag{
			Name:  "step-definitions",
			Value: "step_definitions",
			Usage: "Definitions folder name, should be located in features folder",
		},
		cli.StringFlag{
			Name:  "dir",
			Value: ".",
			Usage: "Relative path, to a feature-file or -directory (Current value: " + CWD + ").",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:    "feature-files",
			Aliases: []string{},
			Usage:   "List feature files to STDOUT",
			Flags:   []cli.Flag{},
			Action:  listFeatureFilesCMD,
		},
		{
			Name:    "features",
			Aliases: []string{},
			Usage:   "List features to STDOUT",
			Flags:   []cli.Flag{},
			Action:  listFeaturesCMD,
		},
		{
			Name:    "definitions",
			Aliases: []string{"defs", "code"},
			Usage:   "List behaviours to STDOUT",
			Flags:   []cli.Flag{},
			Action:  printDefinitionsCodeCMD,
		},
		{
			Name:    "test",
			Aliases: []string{"t"},
			Usage:   "Tests either a test directory with features in it, or a .feature file",
			Flags:   []cli.Flag{},
			Action:  testCMD,
		},
	}

	app.Run(os.Args)
}

func setupGlobals(c *cli.Context) {
	global.Debug = c.GlobalBool("debug")
	global.PPrint = c.GlobalBool("pretty")
	global.DefPattern = c.GlobalString("step-definitions")

	global.CWD = CWD

	if global.PPrint {
		stdres.EnableColor()
	} else {
		stdres.DisableColor()
	}
}

func listFeatureFilesCMD(c *cli.Context) {
	setupGlobals(c)
	dir := c.GlobalString("dir")

	_, features := parseDir(dir)

	for i, feature := range features {
		path := CWD + PathSeparator
		fmt.Printf("\t%2d) %s\n", i, strings.TrimPrefix(feature, path))
	}
}

func listFeaturesCMD(c *cli.Context) {
	setupGlobals(c)
	dir := c.GlobalString("dir")

	_, features := parseDir(dir)

	for _, feature := range features {
		fileReader, err := os.Open(feature)
		if err != nil {
			log.Fatal(err)
		}

		bytes, err := ioutil.ReadAll(fileReader)
		if err != nil {
			log.Fatal(err)
		}

		text := string(bytes)

		if global.PPrint {
			text = strings.Replace(text, "Feature: ", red+"Feature: "+reset, -1)
			text = strings.Replace(text, "Scenario: ", red+"Scenario: "+reset, -1)
			text = strings.Replace(text, " Given ", green+" Given "+reset, -1)
			text = strings.Replace(text, " And ", green+" And "+reset, -1)
			text = strings.Replace(text, " When ", blue+" When "+reset, -1)
			text = strings.Replace(text, " Then ", yellow+" Then "+reset, -1)
		}

		path := CWD + PathSeparator
		fmt.Print("\n# ", strings.TrimPrefix(feature, path), "\n", text, "\n")
	}
}

func printDefinitionsCodeCMD(c *cli.Context) {
	setupGlobals(c)
	dir := c.GlobalString("dir")

	definitions, _ := parseDir(dir)

	text := string(definitions.Code())

	if global.PPrint {
		text = strings.Replace(text, "package main", yellow+"package "+blue+"main"+reset, -1)
		text = strings.Replace(text, "import ", yellow+"import"+reset+" ", -1)
		text = strings.Replace(text, "func ", blue+"func"+reset+" ", -1)
		text = strings.Replace(text, "func(", blue+"func"+reset+"(", -1)
		text = strings.Replace(text, "defer ", blue+"defer"+reset+" ", -1)
		text = strings.Replace(text, " error ", " "+red+"error"+reset+" ", -1)

		text = strings.Replace(text, "\nFeature(", red+"\nFeature"+reset+"(", -1)
		text = strings.Replace(text, "\nScenario(", red+"\nScenario"+reset+"(", -1)
		text = strings.Replace(text, "\nGiven(", green+"\nGiven"+reset+"(", -1)
		text = strings.Replace(text, "\nAnd(", green+"\nAnd"+reset+"(", -1)
		text = strings.Replace(text, "\nWhen(", blue+"\nWhen"+reset+"(", -1)
		text = strings.Replace(text, "\nThen(", yellow+"\nThen"+reset+"(", -1)

		text = strings.Replace(text, "main()", yellow+"main"+reset+"()", -1)
		text = strings.Replace(text, "setup()", yellow+"setup"+reset+"()", -1)

		text = strings.Replace(text, " Pending(", " "+yellow+"Pending"+reset+"(", -1)
		text = strings.Replace(text, "os.Open(", yellow+"os.Open"+reset+"(", -1)
		text = strings.Replace(text, "fd.Close(", yellow+"fd.Close"+reset+"(", -1)
		text = strings.Replace(text, "suite.Test(", yellow+"suite.Test"+reset+"(", -1)
		text = strings.Replace(text, "ParseBool(", yellow+"ParseBool"+reset+"(", -1)
		text = strings.Replace(text, "stdres.EnableColor(", yellow+"stdres.EnableColor"+reset+"(", -1)
		text = strings.Replace(text, "stdres.DisableColor(", yellow+"stdres.DisableColor"+reset+"(", -1)
		text = strings.Replace(text, "os.Args[", yellow+"os.Args"+reset+"[", -1)
	}

	fmt.Println(text)
}

// testCMD search, compile and execute features defined in Gherik format where behaviours are defined in Go-Lang based files.
// Behaviours might be undefined, which will end up as red text in stdout if the context c has pretty print enabled.
func testCMD(c *cli.Context) {
	setupGlobals(c)
	dir := c.GlobalString("dir")

	definitions, features := parseDir(dir)

	if !global.Debug {
		defer definitions.Remove()
	}

	for _, file := range features {
		fd, err := os.Open(file)
		if err != nil {
			log.Fatal(err)
		}
		defer fd.Close()

		definitions.Run(fd)
	}
}

func parseDir(path string) (definition.Definitions, []string) {
	var err error
	var list = feature.List{}
	var defs = []io.Reader{}

	if list, err = feature.ParseDir(path); err != nil {
		log.Fatal(err)
	}

	defFiles := []io.ReadCloser{}

	defer func() {
		for _, file := range defFiles {
			file.Close()
		}
	}() // Make sure to close all open files

	for _, def := range list.Definitions {
		if file, err := os.Open(def); err == nil {
			defFiles = append(defFiles, file)
		} else {
			log.Fatal(err)
		}
	}

	// FIXME figure out if it's possible to ignore convert
	for _, def := range defFiles {
		defs = append(defs, io.Reader(def))
	}

	return definition.NewDefinitions(defs), list.Features
}
<commit_msg>REFACTOR: Remove rotten code from unbrokenwing.go/parseDir method<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/dekelund/stdres"

	"github.com/dekelund/unbrokenwing/compiler/definition"
	"github.com/dekelund/unbrokenwing/compiler/feature"
	"github.com/dekelund/unbrokenwing/global"
)

// Foreground colors
const (
	reset   = "\033[00m"
	black   = "\033[30m"
	red     = "\033[31m"
	green   = "\033[32m"
	yellow  = "\033[33m"
	blue    = "\033[34m"
	magenta = "\033[35m"
	cyan    = "\033[36m"
	white   = "\033[37m"
)

const (
	PathSeparator = string(os.PathSeparator)
)

var CWD string = "."

func init() {
	cwd, err := os.Getwd()
	if err != nil {
		log.Panic(err)
	}

	CWD = cwd
}

func main() {
	app := cli.NewApp()
	app.Name = "unbrokenwing"
	app.Usage = "Run behaviour driven tests as Gherik features"
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "pretty",
			Usage: "Print colorised result to STDOUT/STDERR",
		},
		cli.BoolFlag{
			Name:  "debug",
			Usage: "Verbose printing (Generated files will be kept)",
		},
		cli.StringFlag{
			Name:  "step-definitions",
			Value: "step_definitions",
			Usage: "Definitions folder name, should be located in features folder",
		},
		cli.StringFlag{
			Name:  "dir",
			Value: ".",
			Usage: "Relative path, to a feature-file or -directory (Current value: " + CWD + ").",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:    "feature-files",
			Aliases: []string{},
			Usage:   "List feature files to STDOUT",
			Flags:   []cli.Flag{},
			Action:  listFeatureFilesCMD,
		},
		{
			Name:    "features",
			Aliases: []string{},
			Usage:   "List features to STDOUT",
			Flags:   []cli.Flag{},
			Action:  listFeaturesCMD,
		},
		{
			Name:    "definitions",
			Aliases: []string{"defs", "code"},
			Usage:   "List behaviours to STDOUT",
			Flags:   []cli.Flag{},
			Action:  printDefinitionsCodeCMD,
		},
		{
			Name:    "test",
			Aliases: []string{"t"},
			Usage:   "Tests either a test directory with features in it, or a .feature file",
			Flags:   []cli.Flag{},
			Action:  testCMD,
		},
	}

	app.Run(os.Args)
}

func setupGlobals(c *cli.Context) {
	global.Debug = c.GlobalBool("debug")
	global.PPrint = c.GlobalBool("pretty")
	global.DefPattern = c.GlobalString("step-definitions")

	global.CWD = CWD

	if global.PPrint {
		stdres.EnableColor()
	} else {
		stdres.DisableColor()
	}
}

func listFeatureFilesCMD(c *cli.Context) {
	setupGlobals(c)
	dir := c.GlobalString("dir")

	_, features := parseDir(dir)

	for i, feature := range features {
		path := CWD + PathSeparator
		fmt.Printf("\t%2d) %s\n", i, strings.TrimPrefix(feature, path))
	}
}

func listFeaturesCMD(c *cli.Context) {
	setupGlobals(c)
	dir := c.GlobalString("dir")

	_, features := parseDir(dir)

	for _, feature := range features {
		fileReader, err := os.Open(feature)
		if err != nil {
			log.Fatal(err)
		}

		bytes, err := ioutil.ReadAll(fileReader)
		if err != nil {
			log.Fatal(err)
		}

		text := string(bytes)

		if global.PPrint {
			text = strings.Replace(text, "Feature: ", red+"Feature: "+reset, -1)
			text = strings.Replace(text, "Scenario: ", red+"Scenario: "+reset, -1)
			text = strings.Replace(text, " Given ", green+" Given "+reset, -1)
			text = strings.Replace(text, " And ", green+" And "+reset, -1)
			text = strings.Replace(text, " When ", blue+" When "+reset, -1)
			text = strings.Replace(text, " Then ", yellow+" Then "+reset, -1)
		}

		path := CWD + PathSeparator
		fmt.Print("\n# ", strings.TrimPrefix(feature, path), "\n", text, "\n")
	}
}

func printDefinitionsCodeCMD(c *cli.Context) {
	setupGlobals(c)
	dir := c.GlobalString("dir")

	definitions, _ := parseDir(dir)

	text := string(definitions.Code())

	if global.PPrint {
		text = strings.Replace(text, "package main", yellow+"package "+blue+"main"+reset, -1)
		text = strings.Replace(text, "import ", yellow+"import"+reset+" ", -1)
		text = strings.Replace(text, "func ", blue+"func"+reset+" ", -1)
		text = strings.Replace(text, "func(", blue+"func"+reset+"(", -1)
		text = strings.Replace(text, "defer ", blue+"defer"+reset+" ", -1)
		text = strings.Replace(text, " error ", " "+red+"error"+reset+" ", -1)

		text = strings.Replace(text, "\nFeature(", red+"\nFeature"+reset+"(", -1)
		text = strings.Replace(text, "\nScenario(", red+"\nScenario"+reset+"(", -1)
		text = strings.Replace(text, "\nGiven(", green+"\nGiven"+reset+"(", -1)
		text = strings.Replace(text, "\nAnd(", green+"\nAnd"+reset+"(", -1)
		text = strings.Replace(text, "\nWhen(", blue+"\nWhen"+reset+"(", -1)
		text = strings.Replace(text, "\nThen(", yellow+"\nThen"+reset+"(", -1)

		text = strings.Replace(text, "main()", yellow+"main"+reset+"()", -1)
		text = strings.Replace(text, "setup()", yellow+"setup"+reset+"()", -1)

		text = strings.Replace(text, " Pending(", " "+yellow+"Pending"+reset+"(", -1)
		text = strings.Replace(text, "os.Open(", yellow+"os.Open"+reset+"(", -1)
		text = strings.Replace(text, "fd.Close(", yellow+"fd.Close"+reset+"(", -1)
		text = strings.Replace(text, "suite.Test(", yellow+"suite.Test"+reset+"(", -1)
		text = strings.Replace(text, "ParseBool(", yellow+"ParseBool"+reset+"(", -1)
		text = strings.Replace(text, "stdres.EnableColor(", yellow+"stdres.EnableColor"+reset+"(", -1)
		text = strings.Replace(text, "stdres.DisableColor(", yellow+"stdres.DisableColor"+reset+"(", -1)
		text = strings.Replace(text, "os.Args[", yellow+"os.Args"+reset+"[", -1)
	}

	fmt.Println(text)
}

// testCMD search, compile and execute features defined in Gherik format where behaviours are defined in Go-Lang based files.
// Behaviours might be undefined, which will end up as red text in stdout if the context c has pretty print enabled.
func testCMD(c *cli.Context) {
	setupGlobals(c)
	dir := c.GlobalString("dir")

	definitions, features := parseDir(dir)

	if !global.Debug {
		defer definitions.Remove()
	}

	for _, file := range features {
		fd, err := os.Open(file)
		if err != nil {
			log.Fatal(err)
		}
		defer fd.Close()

		definitions.Run(fd)
	}
}

func parseDir(path string) (definition.Definitions, []string) {
	var err error
	var list = feature.List{}
	var defs = []io.Reader{}

	if list, err = feature.ParseDir(path); err != nil {
		log.Fatal(err)
	}

	for _, def := range list.Definitions {
		file, err := os.Open(def)
		if err != nil {
			log.Fatal(err)
		}

		defs = append(defs, io.Reader(file))
		defer file.Close()
	}

	return definition.NewDefinitions(defs), list.Features
}
<|endoftext|>


<commit_before>package mailgun

import (
	"strconv"
)

type Unsubscription struct {
	CreatedAt string `json:"created_at"`
	Tag       string `json:"tag"`
	ID        string `json:"id"`
	Address   string `json:"address"`
}

// GetUnsubscribes retrieves a list of unsubscriptions issued by recipients of mail from your domain.
// Zero is a valid list length.
func (mg *MailgunImpl) GetUnsubscribes(limit, skip int) (int, []Unsubscription, error) {
	r := newHTTPRequest(generateApiUrl(mg, unsubscribesEndpoint))
	if limit != DefaultLimit {
		r.addParameter("limit", strconv.Itoa(limit))
	}
	if skip != DefaultSkip {
		r.addParameter("skip", strconv.Itoa(skip))
	}
	r.setClient(mg.Client())
	r.setBasicAuth(basicAuthUser, mg.ApiKey())
	var envelope struct {
		TotalCount int              `json:"total_count"`
		Items      []Unsubscription `json:"items"`
	}
	err := getResponseFromJSON(r, &envelope)
	return envelope.TotalCount, envelope.Items, err
}

// GetUnsubscribesByAddress retrieves a list of unsubscriptions by recipient address.
// Zero is a valid list length.
func (mg *MailgunImpl) GetUnsubscribesByAddress(a string) (int, []Unsubscription, error) {
	r := newHTTPRequest(generateApiUrlWithTarget(mg, unsubscribesEndpoint, a))
	r.setClient(mg.Client())
	r.setBasicAuth(basicAuthUser, mg.ApiKey())
	var envelope struct {
		TotalCount int              `json:"total_count"`
		Items      []Unsubscription `json:"items"`
	}
	err := getResponseFromJSON(r, &envelope)
	return envelope.TotalCount, envelope.Items, err
}

// Unsubscribe adds an e-mail address to the domain's unsubscription table.
func (mg *MailgunImpl) Unsubscribe(a, t string) error {
	r := newHTTPRequest(generateApiUrl(mg, unsubscribesEndpoint))
	r.setClient(mg.Client())
	r.setBasicAuth(basicAuthUser, mg.ApiKey())
	p := newUrlEncodedPayload()
	p.addValue("address", a)
	p.addValue("tag", t)
	_, err := makePostRequest(r, p)
	return err
}

// RemoveUnsubscribe removes the e-mail address given from the domain's unsubscription table.
// If passing in an ID (discoverable from, e.g., GetUnsubscribes()), the e-mail address associated
// with the given ID will be removed.
func (mg *MailgunImpl) RemoveUnsubscribe(a string) error {
	r := newHTTPRequest(generateApiUrlWithTarget(mg, unsubscribesEndpoint, a))
	r.setClient(mg.Client())
	r.setBasicAuth(basicAuthUser, mg.ApiKey())
	_, err := makeDeleteRequest(r)
	return err
}

// RemoveUnsubscribe removes the e-mail address given from the domain's unsubscription table with a matching tag.
// If passing in an ID (discoverable from, e.g., GetUnsubscribes()), the e-mail address associated
// with the given ID will be removed.
func (mg *MailgunImpl) RemoveUnsubscribeWithTag(a, t string) error {
	r := newHTTPRequest(generateApiUrlWithTarget(mg, unsubscribesEndpoint, a))
	r.setClient(mg.Client())
	r.setBasicAuth(basicAuthUser, mg.ApiKey())
	r.addParameter("tag", t)
	_, err := makeDeleteRequest(r)
	return err
}
<commit_msg>GetUnsubscribes response returns 'tags', not 'tag'<commit_after>package mailgun

import (
	"strconv"
)

type Unsubscription struct {
	CreatedAt string   `json:"created_at"`
	Tags      []string `json:"tags"`
	ID        string   `json:"id"`
	Address   string   `json:"address"`
}

// GetUnsubscribes retrieves a list of unsubscriptions issued by recipients of mail from your domain.
// Zero is a valid list length.
func (mg *MailgunImpl) GetUnsubscribes(limit, skip int) (int, []Unsubscription, error) {
	r := newHTTPRequest(generateApiUrl(mg, unsubscribesEndpoint))
	if limit != DefaultLimit {
		r.addParameter("limit", strconv.Itoa(limit))
	}
	if skip != DefaultSkip {
		r.addParameter("skip", strconv.Itoa(skip))
	}
	r.setClient(mg.Client())
	r.setBasicAuth(basicAuthUser, mg.ApiKey())
	var envelope struct {
		TotalCount int              `json:"total_count"`
		Items      []Unsubscription `json:"items"`
	}
	err := getResponseFromJSON(r, &envelope)
	return envelope.TotalCount, envelope.Items, err
}

// GetUnsubscribesByAddress retrieves a list of unsubscriptions by recipient address.
// Zero is a valid list length.
func (mg *MailgunImpl) GetUnsubscribesByAddress(a string) (int, []Unsubscription, error) {
	r := newHTTPRequest(generateApiUrlWithTarget(mg, unsubscribesEndpoint, a))
	r.setClient(mg.Client())
	r.setBasicAuth(basicAuthUser, mg.ApiKey())
	var envelope struct {
		TotalCount int              `json:"total_count"`
		Items      []Unsubscription `json:"items"`
	}
	err := getResponseFromJSON(r, &envelope)
	return envelope.TotalCount, envelope.Items, err
}

// Unsubscribe adds an e-mail address to the domain's unsubscription table.
func (mg *MailgunImpl) Unsubscribe(a, t string) error {
	r := newHTTPRequest(generateApiUrl(mg, unsubscribesEndpoint))
	r.setClient(mg.Client())
	r.setBasicAuth(basicAuthUser, mg.ApiKey())
	p := newUrlEncodedPayload()
	p.addValue("address", a)
	p.addValue("tag", t)
	_, err := makePostRequest(r, p)
	return err
}

// RemoveUnsubscribe removes the e-mail address given from the domain's unsubscription table.
// If passing in an ID (discoverable from, e.g., GetUnsubscribes()), the e-mail address associated
// with the given ID will be removed.
func (mg *MailgunImpl) RemoveUnsubscribe(a string) error {
	r := newHTTPRequest(generateApiUrlWithTarget(mg, unsubscribesEndpoint, a))
	r.setClient(mg.Client())
	r.setBasicAuth(basicAuthUser, mg.ApiKey())
	_, err := makeDeleteRequest(r)
	return err
}

// RemoveUnsubscribe removes the e-mail address given from the domain's unsubscription table with a matching tag.
// If passing in an ID (discoverable from, e.g., GetUnsubscribes()), the e-mail address associated
// with the given ID will be removed.
func (mg *MailgunImpl) RemoveUnsubscribeWithTag(a, t string) error {
	r := newHTTPRequest(generateApiUrlWithTarget(mg, unsubscribesEndpoint, a))
	r.setClient(mg.Client())
	r.setBasicAuth(basicAuthUser, mg.ApiKey())
	r.addParameter("tag", t)
	_, err := makeDeleteRequest(r)
	return err
}
<|endoftext|>


<commit_before>// Copyright 2013 Joshua Tacoma. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package uritemplates is a level 4 implementation of RFC 6570 (URI
// Template, http://tools.ietf.org/html/rfc6570).
//
// To use uritemplates, parse a template string and expand it with a value
// map:
//
//	template, _ := uritemplates.Parse("https://api.github.com/repos{/user,repo}")
//	values := make(map[string]interface{})
//	values["user"] = "jtacoma"
//	values["repo"] = "uritemplates"
//	expanded, _ := template.ExpandString(values)
//	fmt.Printf(expanded)
//
package uritemplates

import (
	"bytes"
	"errors"
	"fmt"
	"reflect"
	"regexp"
	"strconv"
	"strings"
)

var (
	unreserved = regexp.MustCompile("[^A-Za-z0-9\\-._~]")
	reserved   = regexp.MustCompile("[^A-Za-z0-9\\-._~:/?#[\\]@!$&'()*+,;=]")
	validname  = regexp.MustCompile("^([A-Za-z0-9_\\.]|%[0-9A-Fa-f][0-9A-Fa-f])+$")
	hex        = []byte("0123456789ABCDEF")
)

func pctEncode(src []byte) []byte {
	dst := make([]byte, len(src)*3)
	for i, b := range src {
		buf := dst[i*3 : i*3+3]
		buf[0] = 0x25
		buf[1] = hex[b/16]
		buf[2] = hex[b%16]
	}
	return dst
}

func escape(s string, allowReserved bool) (escaped string) {
	if allowReserved {
		escaped = string(reserved.ReplaceAllFunc([]byte(s), pctEncode))
	} else {
		escaped = string(unreserved.ReplaceAllFunc([]byte(s), pctEncode))
	}
	return escaped
}

// A UriTemplate is a parsed representation of a URI template.
type UriTemplate struct {
	raw   string
	parts []templatePart
}

// Parse parses a URI template string into a UriTemplate object.
func Parse(rawtemplate string) (template *UriTemplate, err error) {
	template = new(UriTemplate)
	template.raw = rawtemplate
	split := strings.Split(rawtemplate, "{")
	template.parts = make([]templatePart, len(split)*2-1)
	for i, s := range split {
		if i == 0 {
			if strings.Contains(s, "}") {
				err = errors.New("unexpected }")
				break
			} else {
				subsplit := strings.Split(s, ":")
				if len(subsplit) > 1 && strings.Contains(subsplit[0], "/") {
					err = errors.New("unexpected :")
					break
				}
			}
			template.parts[i].raw = s
		} else {
			subsplit := strings.Split(s, "}")
			if len(subsplit) != 2 {
				err = errors.New("malformed template")
				break
			}
			expression := subsplit[0]
			template.parts[i*2-1], err = parseExpression(expression)
			if err != nil {
				break
			}
			if strings.Contains(subsplit[1], "}") {
				err = errors.New("unexpected }")
				break
			}
			template.parts[i*2].raw = subsplit[1]
		}
	}
	if err != nil {
		template = nil
	}
	return template, err
}

type templatePart struct {
	raw           string
	terms         []templateTerm
	first         string
	sep           string
	named         bool
	ifemp         string
	allowReserved bool
}

type templateTerm struct {
	name     string
	explode  bool
	truncate int
}

func parseExpression(expression string) (result templatePart, err error) {
	switch expression[0] {
	case '+':
		result.sep = ","
		result.allowReserved = true
		expression = expression[1:]
	case '.':
		result.first = "."
		result.sep = "."
		expression = expression[1:]
	case '/':
		result.first = "/"
		result.sep = "/"
		expression = expression[1:]
	case ';':
		result.first = ";"
		result.sep = ";"
		result.named = true
		expression = expression[1:]
	case '?':
		result.first = "?"
		result.sep = "&"
		result.named = true
		result.ifemp = "="
		expression = expression[1:]
	case '&':
		result.first = "&"
		result.sep = "&"
		result.named = true
		result.ifemp = "="
		expression = expression[1:]
	case '#':
		result.first = "#"
		result.sep = ","
		result.allowReserved = true
		expression = expression[1:]
	default:
		result.sep = ","
	}
	rawterms := strings.Split(expression, ",")
	result.terms = make([]templateTerm, len(rawterms))
	for i, raw := range rawterms {
		result.terms[i], err = parseTerm(raw)
		if err != nil {
			break
		}
	}
	return result, err
}

func parseTerm(term string) (result templateTerm, err error) {
	if strings.HasSuffix(term, "*") {
		result.explode = true
		term = term[:len(term)-1]
	}
	split := strings.Split(term, ":")
	if len(split) == 1 {
		result.name = term
	} else if len(split) == 2 {
		result.name = split[0]
		var parsed int64
		parsed, err = strconv.ParseInt(split[1], 10, 0)
		result.truncate = int(parsed)
	} else {
		err = errors.New("multiple colons in same term")
	}
	if !validname.MatchString(result.name) {
		err = errors.New("not a valid name: " + result.name)
	}
	if result.explode && result.truncate > 0 {
		err = errors.New("both explode and prefix modifers on same term")
	}
	return result, err
}

// Expand expands a URI template with a set of values to produce a string.
func (self *UriTemplate) Expand(value interface{}) (string, error) {
	values, ismap := value.(map[string]interface{})
	if !ismap {
		if m, ismap := struct2map(value); !ismap {
			return "", errors.New("expected map[string]interface{}, struct, or pointer to struct.")
		} else {
			return self.Expand(m)
		}
	}
	var buf bytes.Buffer
	for _, p := range self.parts {
		err := p.expand(&buf, values)
		if err != nil {
			return "", err
		}
	}
	return buf.String(), nil
}

func (self *templatePart) expand(buf *bytes.Buffer, values map[string]interface{}) error {
	if len(self.raw) > 0 {
		buf.WriteString(self.raw)
		return nil
	}
	var zeroLen = buf.Len()
	buf.WriteString(self.first)
	var firstLen = buf.Len()
	for _, term := range self.terms {
		value, exists := values[term.name]
		if !exists {
			continue
		}
		if buf.Len() != firstLen {
			buf.WriteString(self.sep)
		}
		switch v := value.(type) {
		case string:
			self.expandString(buf, term, v)
		case []interface{}:
			self.expandArray(buf, term, v)
		case map[string]interface{}:
			if term.truncate > 0 {
				return errors.New("cannot truncate a map expansion")
			}
			self.expandMap(buf, term, v)
		default:
			if m, ismap := struct2map(value); ismap {
				if term.truncate > 0 {
					return errors.New("cannot truncate a map expansion")
				}
				self.expandMap(buf, term, m)
			} else {
				str := fmt.Sprintf("%v", value)
				self.expandString(buf, term, str)
			}
		}
	}
	if buf.Len() == firstLen {
		original := buf.Bytes()[:zeroLen]
		buf.Reset()
		buf.Write(original)
	}
	return nil
}

func (self *templatePart) expandName(buf *bytes.Buffer, name string, empty bool) {
	if self.named {
		buf.WriteString(name)
		if empty {
			buf.WriteString(self.ifemp)
		} else {
			buf.WriteString("=")
		}
	}
}

func (self *templatePart) expandString(buf *bytes.Buffer, t templateTerm, s string) {
	if len(s) > t.truncate && t.truncate > 0 {
		s = s[:t.truncate]
	}
	self.expandName(buf, t.name, len(s) == 0)
	buf.WriteString(escape(s, self.allowReserved))
}

func (self *templatePart) expandArray(buf *bytes.Buffer, t templateTerm, a []interface{}) {
	if len(a) == 0 {
		return
	} else if !t.explode {
		self.expandName(buf, t.name, false)
	}
	for i, value := range a {
		if t.explode && i > 0 {
			buf.WriteString(self.sep)
		} else if i > 0 {
			buf.WriteString(",")
		}
		var s string
		switch v := value.(type) {
		case string:
			s = v
		default:
			s = fmt.Sprintf("%v", v)
		}
		if len(s) > t.truncate && t.truncate > 0 {
			s = s[:t.truncate]
		}
		if self.named && t.explode {
			self.expandName(buf, t.name, len(s) == 0)
		}
		buf.WriteString(escape(s, self.allowReserved))
	}
}

func (self *templatePart) expandMap(buf *bytes.Buffer, t templateTerm, m map[string]interface{}) {
	if len(m) == 0 {
		return
	}
	if !t.explode {
		self.expandName(buf, t.name, len(m) == 0)
	}
	var firstLen = buf.Len()
	for k, value := range m {
		if firstLen != buf.Len() {
			if t.explode {
				buf.WriteString(self.sep)
			} else {
				buf.WriteString(",")
			}
		}
		var s string
		switch v := value.(type) {
		case string:
			s = v
		default:
			s = fmt.Sprintf("%v", v)
		}
		if len(s) > t.truncate && t.truncate > 0 {
			s = s[:t.truncate]
		}
		if t.explode {
			buf.WriteString(escape(k, self.allowReserved))
			buf.WriteRune('=')
			buf.WriteString(escape(s, self.allowReserved))
		} else {
			buf.WriteString(escape(k, self.allowReserved))
			buf.WriteRune(',')
			buf.WriteString(escape(s, self.allowReserved))
		}
	}
}

func struct2map(v interface{}) (map[string]interface{}, bool) {
	value := reflect.ValueOf(v)
	switch value.Type().Kind() {
	case reflect.Ptr:
		return struct2map(value.Elem().Interface())
	case reflect.Struct:
		m := make(map[string]interface{})
		for i := 0; i < value.NumField(); i++ {
			tag := value.Type().Field(i).Tag
			var name string
			if strings.Contains(string(tag), ":") {
				name = tag.Get("uri")
			} else {
				name = strings.TrimSpace(string(tag))
			}
			if len(name) == 0 {
				name = value.Type().Field(i).Name
			}
			m[name] = value.Field(i).Interface()
		}
		return m, true
	default:
		return nil, false
	}
}
<commit_msg>Fix build error for Go 1.0.<commit_after>// Copyright 2013 Joshua Tacoma. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package uritemplates is a level 4 implementation of RFC 6570 (URI
// Template, http://tools.ietf.org/html/rfc6570).
//
// To use uritemplates, parse a template string and expand it with a value
// map:
//
//	template, _ := uritemplates.Parse("https://api.github.com/repos{/user,repo}")
//	values := make(map[string]interface{})
//	values["user"] = "jtacoma"
//	values["repo"] = "uritemplates"
//	expanded, _ := template.ExpandString(values)
//	fmt.Printf(expanded)
//
package uritemplates

import (
	"bytes"
	"errors"
	"fmt"
	"reflect"
	"regexp"
	"strconv"
	"strings"
)

var (
	unreserved = regexp.MustCompile("[^A-Za-z0-9\\-._~]")
	reserved   = regexp.MustCompile("[^A-Za-z0-9\\-._~:/?#[\\]@!$&'()*+,;=]")
	validname  = regexp.MustCompile("^([A-Za-z0-9_\\.]|%[0-9A-Fa-f][0-9A-Fa-f])+$")
	hex        = []byte("0123456789ABCDEF")
)

func pctEncode(src []byte) []byte {
	dst := make([]byte, len(src)*3)
	for i, b := range src {
		buf := dst[i*3 : i*3+3]
		buf[0] = 0x25
		buf[1] = hex[b/16]
		buf[2] = hex[b%16]
	}
	return dst
}

func escape(s string, allowReserved bool) (escaped string) {
	if allowReserved {
		escaped = string(reserved.ReplaceAllFunc([]byte(s), pctEncode))
	} else {
		escaped = string(unreserved.ReplaceAllFunc([]byte(s), pctEncode))
	}
	return escaped
}

// A UriTemplate is a parsed representation of a URI template.
type UriTemplate struct {
	raw   string
	parts []templatePart
}

// Parse parses a URI template string into a UriTemplate object.
func Parse(rawtemplate string) (template *UriTemplate, err error) {
	template = new(UriTemplate)
	template.raw = rawtemplate
	split := strings.Split(rawtemplate, "{")
	template.parts = make([]templatePart, len(split)*2-1)
	for i, s := range split {
		if i == 0 {
			if strings.Contains(s, "}") {
				err = errors.New("unexpected }")
				break
			} else {
				subsplit := strings.Split(s, ":")
				if len(subsplit) > 1 && strings.Contains(subsplit[0], "/") {
					err = errors.New("unexpected :")
					break
				}
			}
			template.parts[i].raw = s
		} else {
			subsplit := strings.Split(s, "}")
			if len(subsplit) != 2 {
				err = errors.New("malformed template")
				break
			}
			expression := subsplit[0]
			template.parts[i*2-1], err = parseExpression(expression)
			if err != nil {
				break
			}
			if strings.Contains(subsplit[1], "}") {
				err = errors.New("unexpected }")
				break
			}
			template.parts[i*2].raw = subsplit[1]
		}
	}
	if err != nil {
		template = nil
	}
	return template, err
}

type templatePart struct {
	raw           string
	terms         []templateTerm
	first         string
	sep           string
	named         bool
	ifemp         string
	allowReserved bool
}

type templateTerm struct {
	name     string
	explode  bool
	truncate int
}

func parseExpression(expression string) (result templatePart, err error) {
	switch expression[0] {
	case '+':
		result.sep = ","
		result.allowReserved = true
		expression = expression[1:]
	case '.':
		result.first = "."
		result.sep = "."
		expression = expression[1:]
	case '/':
		result.first = "/"
		result.sep = "/"
		expression = expression[1:]
	case ';':
		result.first = ";"
		result.sep = ";"
		result.named = true
		expression = expression[1:]
	case '?':
		result.first = "?"
		result.sep = "&"
		result.named = true
		result.ifemp = "="
		expression = expression[1:]
	case '&':
		result.first = "&"
		result.sep = "&"
		result.named = true
		result.ifemp = "="
		expression = expression[1:]
	case '#':
		result.first = "#"
		result.sep = ","
		result.allowReserved = true
		expression = expression[1:]
	default:
		result.sep = ","
	}
	rawterms := strings.Split(expression, ",")
	result.terms = make([]templateTerm, len(rawterms))
	for i, raw := range rawterms {
		result.terms[i], err = parseTerm(raw)
		if err != nil {
			break
		}
	}
	return result, err
}

func parseTerm(term string) (result templateTerm, err error) {
	if strings.HasSuffix(term, "*") {
		result.explode = true
		term = term[:len(term)-1]
	}
	split := strings.Split(term, ":")
	if len(split) == 1 {
		result.name = term
	} else if len(split) == 2 {
		result.name = split[0]
		var parsed int64
		parsed, err = strconv.ParseInt(split[1], 10, 0)
		result.truncate = int(parsed)
	} else {
		err = errors.New("multiple colons in same term")
	}
	if !validname.MatchString(result.name) {
		err = errors.New("not a valid name: " + result.name)
	}
	if result.explode && result.truncate > 0 {
		err = errors.New("both explode and prefix modifers on same term")
	}
	return result, err
}

// Expand expands a URI template with a set of values to produce a string.
func (self *UriTemplate) Expand(value interface{}) (string, error) {
	values, ismap := value.(map[string]interface{})
	if !ismap {
		if m, ismap := struct2map(value); !ismap {
			return "", errors.New("expected map[string]interface{}, struct, or pointer to struct.")
		} else {
			return self.Expand(m)
		}
	}
	var buf bytes.Buffer
	for _, p := range self.parts {
		err := p.expand(&buf, values)
		if err != nil {
			return "", err
		}
	}
	return buf.String(), nil
}

func (self *templatePart) expand(buf *bytes.Buffer, values map[string]interface{}) error {
	if len(self.raw) > 0 {
		buf.WriteString(self.raw)
		return nil
	}
	var zeroLen = buf.Len()
	buf.WriteString(self.first)
	var firstLen = buf.Len()
	for _, term := range self.terms {
		value, exists := values[term.name]
		if !exists {
			continue
		}
		if buf.Len() != firstLen {
			buf.WriteString(self.sep)
		}
		switch v := value.(type) {
		case string:
			self.expandString(buf, term, v)
		case []interface{}:
			self.expandArray(buf, term, v)
		case map[string]interface{}:
			if term.truncate > 0 {
				return errors.New("cannot truncate a map expansion")
			}
			self.expandMap(buf, term, v)
		default:
			if m, ismap := struct2map(value); ismap {
				if term.truncate > 0 {
					return errors.New("cannot truncate a map expansion")
				}
				self.expandMap(buf, term, m)
			} else {
				str := fmt.Sprintf("%v", value)
				self.expandString(buf, term, str)
			}
		}
	}
	if buf.Len() == firstLen {
		original := buf.Bytes()[:zeroLen]
		buf.Reset()
		buf.Write(original)
	}
	return nil
}

func (self *templatePart) expandName(buf *bytes.Buffer, name string, empty bool) {
	if self.named {
		buf.WriteString(name)
		if empty {
			buf.WriteString(self.ifemp)
		} else {
			buf.WriteString("=")
		}
	}
}

func (self *templatePart) expandString(buf *bytes.Buffer, t templateTerm, s string) {
	if len(s) > t.truncate && t.truncate > 0 {
		s = s[:t.truncate]
	}
	self.expandName(buf, t.name, len(s) == 0)
	buf.WriteString(escape(s, self.allowReserved))
}

func (self *templatePart) expandArray(buf *bytes.Buffer, t templateTerm, a []interface{}) {
	if len(a) == 0 {
		return
	} else if !t.explode {
		self.expandName(buf, t.name, false)
	}
	for i, value := range a {
		if t.explode && i > 0 {
			buf.WriteString(self.sep)
		} else if i > 0 {
			buf.WriteString(",")
		}
		var s string
		switch v := value.(type) {
		case string:
			s = v
		default:
			s = fmt.Sprintf("%v", v)
		}
		if len(s) > t.truncate && t.truncate > 0 {
			s = s[:t.truncate]
		}
		if self.named && t.explode {
			self.expandName(buf, t.name, len(s) == 0)
		}
		buf.WriteString(escape(s, self.allowReserved))
	}
}

func (self *templatePart) expandMap(buf *bytes.Buffer, t templateTerm, m map[string]interface{}) {
	if len(m) == 0 {
		return
	}
	if !t.explode {
		self.expandName(buf, t.name, len(m) == 0)
	}
	var firstLen = buf.Len()
	for k, value := range m {
		if firstLen != buf.Len() {
			if t.explode {
				buf.WriteString(self.sep)
			} else {
				buf.WriteString(",")
			}
		}
		var s string
		switch v := value.(type) {
		case string:
			s = v
		default:
			s = fmt.Sprintf("%v", v)
		}
		if len(s) > t.truncate && t.truncate > 0 {
			s = s[:t.truncate]
		}
		if t.explode {
			buf.WriteString(escape(k, self.allowReserved))
			buf.WriteRune('=')
			buf.WriteString(escape(s, self.allowReserved))
		} else {
			buf.WriteString(escape(k, self.allowReserved))
			buf.WriteRune(',')
			buf.WriteString(escape(s, self.allowReserved))
		}
	}
}

func struct2map(v interface{}) (map[string]interface{}, bool) {
	value := reflect.ValueOf(v)
	switch value.Type().Kind() {
	case reflect.Ptr:
		return struct2map(value.Elem().Interface())
	case reflect.Struct:
		m := make(map[string]interface{})
		for i := 0; i < value.NumField(); i++ {
			tag := value.Type().Field(i).Tag
			var name string
			if strings.Contains(string(tag), ":") {
				name = tag.Get("uri")
			} else {
				name = strings.TrimSpace(string(tag))
			}
			if len(name) == 0 {
				name = value.Type().Field(i).Name
			}
			m[name] = value.Field(i).Interface()
		}
		return m, true
	}
	return nil, false
}
<|endoftext|>


<commit_before>package http

import (
	"bytes"
	"io"
	"net/http"
	"strings"

	"github.com/viant/endly"
	"github.com/viant/toolbox"
)

// Build a native http request from SendRequest
type RequestBuilder interface {
	setContext(*endly.Context) RequestBuilder
	setRequest(*Request) RequestBuilder
	build() (*http.Request, error)
}
type requestBuilder struct {
	context *endly.Context
	request *Request
}

func newRequestBuilder() RequestBuilder {
	return &requestBuilder{}
}
func (b *requestBuilder) setContext(c *endly.Context) RequestBuilder {
	b.context = c
	return b
}
func (b *requestBuilder) setRequest(r *Request) RequestBuilder {
	b.request = r
	return b
}
func (b *requestBuilder) build() (*http.Request, error) {
	var err error

	//Expand request
	b.request = b.request.Expand(b.context)

	//Build request body
	requestBody, isBase64Encoded, err := b.buildRequestBody()
	if err != nil {
		return nil, err
	}

	//Create http request
	httpRequest, err := http.NewRequest(strings.ToUpper(b.request.Method), b.request.URL, requestBody)
	if err != nil {
		return nil, err
	}

	//Set transfer encoding
	if isBase64Encoded {
		setTransferEncoding(httpRequest, "")
	}

	//Initialize http request with headers and cookies from SendRequest
	copyHeaders(b.request.Header, httpRequest.Header)
	addCookies(httpRequest.Header, b.request.Cookies)

	//Add session cookies
	b.buildSessionCookies(httpRequest)

	return httpRequest, nil
}
func (b *requestBuilder) buildRequestBody() (io.Reader, bool, error) {
	var isBase64Encoded bool
	var body []byte
	var err error
	var reader io.Reader
	var ok bool
	if len(b.request.Body) > 0 {
		body = []byte(b.request.Body)
		if b.request.RequestUdf != "" {
			transformed, err := endly.TransformWithUDF(b.context, b.request.RequestUdf, b.request.URL, string(body))
			if err != nil {
				return nil, isBase64Encoded, err
			}
			if body, ok = transformed.([]byte); !ok {
				body = []byte(toolbox.AsString(transformed))
			}
		}
		isBase64Encoded = strings.HasPrefix(string(body), "base64:")
		body, err = endly.FromPayload(string(body))
		if err != nil {
			return nil, isBase64Encoded, err
		}
		reader = bytes.NewReader(body)
	}

	return reader, isBase64Encoded, nil
}
func (b *requestBuilder) buildSessionCookies(r *http.Request) {
	sessionCookies := b.context.State().GetMap("cookies")
	if sessionCookies != nil && len(sessionCookies) > 0 {
		contextCookies := make([]*http.Cookie, len(sessionCookies))
		for _, v := range sessionCookies {
			if cookie, ok := v.(*http.Cookie); ok {
				contextCookies = append(contextCookies, cookie)
			}
		}
		addCookies(r.Header, contextCookies)
	}
}
func copyHeaders(source http.Header, target http.Header) {
	for key, values := range source {
		if _, has := target[key]; !has {
			target[key] = make([]string, 0)
		}
		if len(values) == 1 {
			target.Set(key, values[0])
		} else {

			for _, value := range values {
				target.Add(key, value)
			}
		}
	}
}
func addCookies(h http.Header, c Cookies) {
	if c == nil || len(c) == 0 {
		return
	}
	for _, cookie := range c {
		if v := cookie.String(); v != "" {
			h.Add("Cookie", v)
		}
	}
}

func setTransferEncoding(r *http.Request, encoding string) {
	if r.TransferEncoding == nil {
		r.TransferEncoding = make([]string, 0)
	}
	r.TransferEncoding = append(r.TransferEncoding, encoding)
}
<commit_msg>Changes to reflect endly refactoring. TransformWithUDF is now in udf package.<commit_after>package http

import (
	"bytes"
	"io"
	"net/http"
	"strings"

	"github.com/viant/endly"
	"github.com/viant/endly/udf"
	"github.com/viant/endly/util"
	"github.com/viant/toolbox"
)

// Build a native http request from SendRequest
type RequestBuilder interface {
	setContext(*endly.Context) RequestBuilder
	setRequest(*Request) RequestBuilder
	build() (*http.Request, error)
}
type requestBuilder struct {
	context *endly.Context
	request *Request
}

func newRequestBuilder() RequestBuilder {
	return &requestBuilder{}
}
func (b *requestBuilder) setContext(c *endly.Context) RequestBuilder {
	b.context = c
	return b
}
func (b *requestBuilder) setRequest(r *Request) RequestBuilder {
	b.request = r
	return b
}
func (b *requestBuilder) build() (*http.Request, error) {
	var err error

	//Expand request
	b.request = b.request.Expand(b.context)

	//Build request body
	requestBody, isBase64Encoded, err := b.buildRequestBody()
	if err != nil {
		return nil, err
	}

	//Create http request
	httpRequest, err := http.NewRequest(strings.ToUpper(b.request.Method), b.request.URL, requestBody)
	if err != nil {
		return nil, err
	}

	//Set transfer encoding
	if isBase64Encoded {
		setTransferEncoding(httpRequest, "")
	}

	//Initialize http request with headers and cookies from SendRequest
	copyHeaders(b.request.Header, httpRequest.Header)
	addCookies(httpRequest.Header, b.request.Cookies)

	//Add session cookies
	b.buildSessionCookies(httpRequest)

	return httpRequest, nil
}
func (b *requestBuilder) buildRequestBody() (io.Reader, bool, error) {
	var isBase64Encoded bool
	var body []byte
	var err error
	var reader io.Reader
	var ok bool
	if len(b.request.Body) > 0 {
		body = []byte(b.request.Body)
		if b.request.RequestUdf != "" {
			transformed, err := udf.TransformWithUDF(b.context, b.request.RequestUdf, b.request.URL, string(body))
			if err != nil {
				return nil, isBase64Encoded, err
			}
			if body, ok = transformed.([]byte); !ok {
				body = []byte(toolbox.AsString(transformed))
			}
		}
		isBase64Encoded = strings.HasPrefix(string(body), "base64:")
		body, err = util.FromPayload(string(body))
		if err != nil {
			return nil, isBase64Encoded, err
		}
		reader = bytes.NewReader(body)
	}

	return reader, isBase64Encoded, nil
}
func (b *requestBuilder) buildSessionCookies(r *http.Request) {
	sessionCookies := b.context.State().GetMap("cookies")
	if sessionCookies != nil && len(sessionCookies) > 0 {
		contextCookies := make([]*http.Cookie, len(sessionCookies))
		for _, v := range sessionCookies {
			if cookie, ok := v.(*http.Cookie); ok {
				contextCookies = append(contextCookies, cookie)
			}
		}
		addCookies(r.Header, contextCookies)
	}
}
func copyHeaders(source http.Header, target http.Header) {
	for key, values := range source {
		if _, has := target[key]; !has {
			target[key] = make([]string, 0)
		}
		if len(values) == 1 {
			target.Set(key, values[0])
		} else {

			for _, value := range values {
				target.Add(key, value)
			}
		}
	}
}
func addCookies(h http.Header, c Cookies) {
	if c == nil || len(c) == 0 {
		return
	}
	for _, cookie := range c {
		if v := cookie.String(); v != "" {
			h.Add("Cookie", v)
		}
	}
}

func setTransferEncoding(r *http.Request, encoding string) {
	if r.TransferEncoding == nil {
		r.TransferEncoding = make([]string, 0)
	}
	r.TransferEncoding = append(r.TransferEncoding, encoding)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
)

type Fetcher interface {
	// Fetch returns the body of URL and
	// a slice of URLs found on that page.
	Fetch(url string) (body string, urls []string, err error)
}

// Crawl uses fetcher to recursively crawl
// pages starting with url, to a maximum of depth.
func Crawl(url string, depth int, fetcher Fetcher) {
	// TODO: Fetch URLs in parallel.
	// TODO: Don't fetch the same URL twice.
	// This implementation doesn't do either:
	if depth <= 0 {
		return
	}
	body, urls, err := fetcher.Fetch(url)
	if err != nil {
		fmt.Println(err)
		return
	}
	fmt.Printf("found: %s %q\n", url, body)
	for _, u := range urls {
		Crawl(u, depth-1, fetcher)
	}
	return
}

func main() {
	Crawl("http://golang.org/", 4, fetcher)
}

// fakeFetcher is Fetcher that returns canned results.
type fakeFetcher map[string]*fakeResult

type fakeResult struct {
	body string
	urls []string
}

func (f fakeFetcher) Fetch(url string) (string, []string, error) {
	if res, ok := f[url]; ok {
		return res.body, res.urls, nil
	}
	return "", nil, fmt.Errorf("not found: %s", url)
}

// fetcher is a populated fakeFetcher.
var fetcher = fakeFetcher{
	"http://golang.org/": &fakeResult{
		"The Go Programming Language",
		[]string{
			"http://golang.org/pkg/",
			"http://golang.org/cmd/",
		},
	},
	"http://golang.org/pkg/": &fakeResult{
		"Packages",
		[]string{
			"http://golang.org/",
			"http://golang.org/cmd/",
			"http://golang.org/pkg/fmt/",
			"http://golang.org/pkg/os/",
		},
	},
	"http://golang.org/pkg/fmt/": &fakeResult{
		"Package fmt",
		[]string{
			"http://golang.org/",
			"http://golang.org/pkg/",
		},
	},
	"http://golang.org/pkg/os/": &fakeResult{
		"Package os",
		[]string{
			"http://golang.org/",
			"http://golang.org/pkg/",
		},
	},
}
<commit_msg>Prevent urls from being crawled twice<commit_after>package main

import (
	"fmt"
)

var urlsFetched map[string]bool = make(map[string]bool)

type Fetcher interface {
	// Fetch returns the body of URL and
	// a slice of URLs found on that page.
	Fetch(url string) (body string, urls []string, err error)
}

// Crawl uses fetcher to recursively crawl
// pages starting with url, to a maximum of depth.
func Crawl(url string, depth int, fetcher Fetcher) {
	// TODO: Fetch URLs in parallel.
	// This implementation doesn't do either:
	if depth <= 0 {
		return
	}

	if urlsFetched[url] {
		fmt.Printf("already crawled: %s\n", url)
		return
	} else {
		urlsFetched[url] = true
	}

	body, urls, err := fetcher.Fetch(url)
	if err != nil {
		fmt.Println(err)
		return
	}
	fmt.Printf("found: %s %q\n", url, body)
	for _, u := range urls {
		Crawl(u, depth-1, fetcher)
	}
	return
}

func main() {
	Crawl("http://golang.org/", 4, fetcher)
}

// fakeFetcher is Fetcher that returns canned results.
type fakeFetcher map[string]*fakeResult

type fakeResult struct {
	body string
	urls []string
}

func (f fakeFetcher) Fetch(url string) (string, []string, error) {
	if res, ok := f[url]; ok {
		return res.body, res.urls, nil
	}
	return "", nil, fmt.Errorf("not found: %s", url)
}

// fetcher is a populated fakeFetcher.
var fetcher = fakeFetcher{
	"http://golang.org/": &fakeResult{
		"The Go Programming Language",
		[]string{
			"http://golang.org/pkg/",
			"http://golang.org/cmd/",
		},
	},
	"http://golang.org/pkg/": &fakeResult{
		"Packages",
		[]string{
			"http://golang.org/",
			"http://golang.org/cmd/",
			"http://golang.org/pkg/fmt/",
			"http://golang.org/pkg/os/",
		},
	},
	"http://golang.org/pkg/fmt/": &fakeResult{
		"Package fmt",
		[]string{
			"http://golang.org/",
			"http://golang.org/pkg/",
		},
	},
	"http://golang.org/pkg/os/": &fakeResult{
		"Package os",
		[]string{
			"http://golang.org/",
			"http://golang.org/pkg/",
		},
	},
}
<|endoftext|>


<commit_before>// dnsseed simply gathers the list of seed servers, stores them. and deteermines if we are one
// of them.

package util

import (
	"github.com/goarchit/archit/log"
	"github.com/valyala/gorpc"
	"math/rand"
	"net"
	"time"
)

var aliveDNSes []string

func Dnsseed() {
	var err error

	DNSSeeds, err = net.LookupHost("dnsseed.goarchit.online")
	aliveDNSes = make([]string, 0)
	if err != nil {
		log.Critical("Failure to lookup dnsseed.goarchit.online")
	}
	log.Debug("DNSseed resolved to", len(DNSSeeds), "seeds:", DNSSeeds)
	if len(PublicIP) < 9 { // len(1.2.3.4:1) == 9
		log.Critical("Dnsseed - PublicIP is obviously wrong:", PublicIP)
	}
	WG.Add(len(DNSSeeds))
	for i, v := range DNSSeeds {
		ip := v + SeedPortBase
		if ip == PublicIP {
			IAmASeed = true
			log.Console("We are a registered seed node!")
			WG.Done()
		} else {
			go dnsalive(i, v)
		}
	}
	if SeedMode && !IAmASeed {
		log.Critical("Sorry, your public IP", PublicIP, "is not a registered seed node!")
	}
	WG.Wait()

	log.Debug("IAmASeed =",IAmASeed,"len(aliveDNSes) =",len(aliveDNSes)

	if !IAmASeed || len(aliveDNSes) > 0 {
		log.Debug(len(aliveDNSes), "DNSes found alive")
		rand.Seed(time.Now().UnixNano())
		MyDNSServerIP = aliveDNSes[rand.Intn(len(aliveDNSes)-1)] + SeedPortBase
		log.Console("Associating ourselves with DNSSeed", MyDNSServerIP)
	}
	// Nil MyDNSServerIP is used as flag for 1st DNSSeed
}

func dnsalive(i int, v string) {
	var found bool
	defer WG.Done()
	//  Don't call yourself!
	pip, _, err := net.SplitHostPort(PublicIP)
	if err != nil {
		log.Critical("Error splitting PublicIP", err)
	}
	log.Trace("Checking if IP", v, "is our PublicIP", pip, "?")
	// Don't call yourself
	if v == pip {
		log.Trace("Skipping talking to ourselves as unhealthy")
		return
	}
	serverIP := v + SeedPortBase
	c := gorpc.NewTCPClient(serverIP)
	c.Start()
	defer c.Stop()
	d := gorpc.NewDispatcher()
	d.AddFunc("Ping", func() {})
	dc := d.NewFuncClient(c)
	_, err = dc.CallTimeout("Ping", nil, time.Second*5)
	if err == nil {
		// Add mutexes just to be safe
		Mutex.Lock()
		// Add to current up list
		aliveDNSes = append(aliveDNSes, v)
		Mutex.Unlock()
		found = true
		log.Info("DNSSeed", v, "is alive.")
	}
	if !found && !IAmASeed {
		log.Critical("No DNSSeeds are apparently active.  Sorry.")
	}
}
<commit_msg>Forget to check if it would compile<commit_after>// dnsseed simply gathers the list of seed servers, stores them. and deteermines if we are one
// of them.

package util

import (
	"github.com/goarchit/archit/log"
	"github.com/valyala/gorpc"
	"math/rand"
	"net"
	"time"
)

var aliveDNSes []string

func Dnsseed() {
	var err error

	DNSSeeds, err = net.LookupHost("dnsseed.goarchit.online")
	aliveDNSes = make([]string, 0)
	if err != nil {
		log.Critical("Failure to lookup dnsseed.goarchit.online")
	}
	log.Debug("DNSseed resolved to", len(DNSSeeds), "seeds:", DNSSeeds)
	if len(PublicIP) < 9 { // len(1.2.3.4:1) == 9
		log.Critical("Dnsseed - PublicIP is obviously wrong:", PublicIP)
	}
	WG.Add(len(DNSSeeds))
	for i, v := range DNSSeeds {
		ip := v + SeedPortBase
		if ip == PublicIP {
			IAmASeed = true
			log.Console("We are a registered seed node!")
			WG.Done()
		} else {
			go dnsalive(i, v)
		}
	}
	if SeedMode && !IAmASeed {
		log.Critical("Sorry, your public IP", PublicIP, "is not a registered seed node!")
	}
	WG.Wait()

	log.Debug("IAmASeed =",IAmASeed,"len(aliveDNSes) =",len(aliveDNSes))

	if !IAmASeed || len(aliveDNSes) > 0 {
		log.Debug(len(aliveDNSes), "DNSes found alive")
		rand.Seed(time.Now().UnixNano())
		MyDNSServerIP = aliveDNSes[rand.Intn(len(aliveDNSes)-1)] + SeedPortBase
		log.Console("Associating ourselves with DNSSeed", MyDNSServerIP)
	}
	// Nil MyDNSServerIP is used as flag for 1st DNSSeed
}

func dnsalive(i int, v string) {
	var found bool
	defer WG.Done()
	//  Don't call yourself!
	pip, _, err := net.SplitHostPort(PublicIP)
	if err != nil {
		log.Critical("Error splitting PublicIP", err)
	}
	log.Trace("Checking if IP", v, "is our PublicIP", pip, "?")
	// Don't call yourself
	if v == pip {
		log.Trace("Skipping talking to ourselves as unhealthy")
		return
	}
	serverIP := v + SeedPortBase
	c := gorpc.NewTCPClient(serverIP)
	c.Start()
	defer c.Stop()
	d := gorpc.NewDispatcher()
	d.AddFunc("Ping", func() {})
	dc := d.NewFuncClient(c)
	_, err = dc.CallTimeout("Ping", nil, time.Second*5)
	if err == nil {
		// Add mutexes just to be safe
		Mutex.Lock()
		// Add to current up list
		aliveDNSes = append(aliveDNSes, v)
		Mutex.Unlock()
		found = true
		log.Info("DNSSeed", v, "is alive.")
	}
	if !found && !IAmASeed {
		log.Critical("No DNSSeeds are apparently active.  Sorry.")
	}
}
<|endoftext|>


<commit_before>package util

import (
	"container/list"
	"log"
	"reflect"
	"time"
)

// Map of strings.
type StrMap map[string]interface{}

// Function that enumerates a list of strings.
type EnumFunc func() (StrMap, error)

// StrEnum provides a simple mechanism for periodically invoking a function
// that enumerates strings and indicating when items are added or removed.
type StrEnum struct {
	StringAdded   chan string // notify when a string is added
	StringRemoved chan string // notify when a string is removed
}

// Create a new enumerator with the specified enumeration function. The
// enumeration process will run each time a value is received from enumChan
// until it is closed. Note that enumFunc must not modify the map it returns.
func NewStrEnum(enumChan <-chan time.Time, enumFunc EnumFunc) *StrEnum {

	// Create a new enumerator
	s := &StrEnum{
		StringAdded:   make(chan string),
		StringRemoved: make(chan string),
	}

	// Launch a separate goroutine to perform the enumeration
	go s.run(enumChan, enumFunc)

	return s
}

// Continually invoke the enumerator until stopped.
func (s *StrEnum) run(enumChan <-chan time.Time, enumFunc EnumFunc) {

	// Map of strings from the previous enumeration and lists of items that
	// need to be sent on one of the notification channels
	oldStrings := StrMap{}
	stringsAdded, stringsRemoved := list.New(), list.New()

	for {

		// Use constants to make it a bit easier to see what's going on.
		const (
			enumCase = iota
			addCase
			removeCase
			numCases
		)

		// A runtime select is needed here in order to avoid a deadlock.
		// Whenever enumeration indicates that notifications should be sent,
		// this needs to take place within the same select as enumChan. Since
		// sending on these channels can block, there needs to be a way to
		// abort the send if the enumChan is closed.
		cases := make([]reflect.SelectCase, numCases)
		cases[enumCase].Dir = reflect.SelectRecv
		cases[enumCase].Chan = reflect.ValueOf(enumChan)
		cases[addCase].Dir = reflect.SelectSend
		cases[removeCase].Dir = reflect.SelectSend

		// If there are values waiting to be sent on either of the two channels
		// then fill in the appropriate fields in each of the cases.
		if stringsAdded.Len() != 0 {
			cases[addCase].Chan = reflect.ValueOf(s.StringAdded)
			cases[addCase].Send = reflect.ValueOf(stringsAdded.Front().Value.(string))
		}
		if stringsRemoved.Len() != 0 {
			cases[removeCase].Chan = reflect.ValueOf(s.StringRemoved)
			cases[removeCase].Send = reflect.ValueOf(stringsRemoved.Front().Value.(string))
		}

		// Perform the select. The value received is ignored.
		i, _, ok := reflect.Select(cases)

		// Perform the appropriate action based on the case index.
		switch i {
		case enumCase:

			// If the receive was successful, perform another enumeration.
			// Otherwise, the channel was closed and the loop should quit.
			if ok {

				// Log and ignore any errors
				if newStrings, err := enumFunc(); err != nil {
					log.Println("[ERR]", err)
				} else {
					stringsAdded.PushBackList(s.compare(oldStrings, newStrings))
					stringsRemoved.PushBackList(s.compare(newStrings, oldStrings))
					oldStrings = newStrings
				}
			} else {
				break
			}

		case addCase:
			stringsAdded.Remove(stringsAdded.Front())
		case removeCase:
			stringsRemoved.Remove(stringsRemoved.Front())
		}
	}

	close(s.StringAdded)
	close(s.StringRemoved)
}

// Compare two maps and return a list of any changes.
func (s *StrEnum) compare(a, b StrMap) *list.List {

	// Create an empty list for new items.
	l := list.New()

	// Check each of the items in map b to see if they exist in map a. If not,
	// add them to the list.
	for item, _ := range b {
		if _, exists := a[item]; !exists {
			l.PushBack(item)
		}
	}

	return l
}
<commit_msg>Adjusted StrEnum class to always invoke enumFunc() once before the loop.<commit_after>package util

import (
	"container/list"
	"log"
	"reflect"
	"time"
)

// Map of strings.
type StrMap map[string]interface{}

// Function that enumerates a list of strings.
type EnumFunc func() (StrMap, error)

// StrEnum provides a simple mechanism for periodically invoking a function
// that enumerates strings and indicating when items are added or removed.
type StrEnum struct {
	StringAdded   chan string // notify when a string is added
	StringRemoved chan string // notify when a string is removed
}

// Create a new enumerator with the specified enumeration function. The
// enumeration process will run each time a value is received from enumChan
// until it is closed. Note that enumFunc must not modify the map it returns.
func NewStrEnum(enumChan <-chan time.Time, enumFunc EnumFunc) *StrEnum {

	// Create a new enumerator
	s := &StrEnum{
		StringAdded:   make(chan string),
		StringRemoved: make(chan string),
	}

	// Launch a separate goroutine to perform the enumeration
	go s.run(enumChan, enumFunc)

	return s
}

// Continually invoke the enumerator until stopped.
func (s *StrEnum) run(enumChan <-chan time.Time, enumFunc EnumFunc) {

	// Lists of items that need to be sent on one of the notification channels.
	stringsAdded, stringsRemoved := list.New(), list.New()

	// Load the initial values into the map.
	oldStrings, err := enumFunc()
	if err != nil {
		log.Println("[ERR]", err)
	} else {
		for item, _ := range oldStrings {
			stringsAdded.PushBack(item)
		}
	}

	for {

		// Use constants to make it a bit easier to see what's going on.
		const (
			enumCase = iota
			addCase
			removeCase
			numCases
		)

		// A runtime select is needed here in order to avoid a deadlock.
		// Whenever enumeration indicates that notifications should be sent,
		// this needs to take place within the same select as enumChan. Since
		// sending on these channels can block, there needs to be a way to
		// abort the send if the enumChan is closed.
		cases := make([]reflect.SelectCase, numCases)
		cases[enumCase].Dir = reflect.SelectRecv
		cases[enumCase].Chan = reflect.ValueOf(enumChan)
		cases[addCase].Dir = reflect.SelectSend
		cases[removeCase].Dir = reflect.SelectSend

		// If there are values waiting to be sent on either of the two channels
		// then fill in the appropriate fields in each of the cases.
		if stringsAdded.Len() != 0 {
			cases[addCase].Chan = reflect.ValueOf(s.StringAdded)
			cases[addCase].Send = reflect.ValueOf(stringsAdded.Front().Value.(string))
		}
		if stringsRemoved.Len() != 0 {
			cases[removeCase].Chan = reflect.ValueOf(s.StringRemoved)
			cases[removeCase].Send = reflect.ValueOf(stringsRemoved.Front().Value.(string))
		}

		// Perform the select. The value received is ignored.
		i, _, ok := reflect.Select(cases)

		// Perform the appropriate action based on the case index.
		switch i {
		case enumCase:

			// If the receive was successful, perform another enumeration.
			// Otherwise, the channel was closed and the loop should quit.
			if ok {

				// Log and ignore any errors
				if newStrings, err := enumFunc(); err != nil {
					log.Println("[ERR]", err)
				} else {
					stringsAdded.PushBackList(s.compare(oldStrings, newStrings))
					stringsRemoved.PushBackList(s.compare(newStrings, oldStrings))
					oldStrings = newStrings
				}
			} else {
				break
			}

		case addCase:
			stringsAdded.Remove(stringsAdded.Front())
		case removeCase:
			stringsRemoved.Remove(stringsRemoved.Front())
		}
	}

	close(s.StringAdded)
	close(s.StringRemoved)
}

// Compare two maps and return a list of any changes.
func (s *StrEnum) compare(a, b StrMap) *list.List {

	// Create an empty list for new items.
	l := list.New()

	// Check each of the items in map b to see if they exist in map a. If not,
	// add them to the list.
	for item, _ := range b {
		if _, exists := a[item]; !exists {
			l.PushBack(item)
		}
	}

	return l
}
<|endoftext|>


<commit_before>// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//     http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.
//

// Code generated by MockGen. DO NOT EDIT.
// Source: github.com/aws/amazon-ecs-agent/agent/utils/ioutilwrapper (interfaces: IOUtil)

// Package mock_ioutilwrapper is a generated GoMock package.
package mock_ioutilwrapper

import (
	os "os"
	reflect "reflect"

	oswrapper "github.com/aws/amazon-ecs-agent/agent/utils/oswrapper"
	gomock "github.com/golang/mock/gomock"
)

// MockIOUtil is a mock of IOUtil interface
type MockIOUtil struct {
	ctrl     *gomock.Controller
	recorder *MockIOUtilMockRecorder
}

// MockIOUtilMockRecorder is the mock recorder for MockIOUtil
type MockIOUtilMockRecorder struct {
	mock *MockIOUtil
}

// NewMockIOUtil creates a new mock instance
func NewMockIOUtil(ctrl *gomock.Controller) *MockIOUtil {
	mock := &MockIOUtil{ctrl: ctrl}
	mock.recorder = &MockIOUtilMockRecorder{mock}
	return mock
}

// EXPECT returns an object that allows the caller to indicate expected use
func (m *MockIOUtil) EXPECT() *MockIOUtilMockRecorder {
	return m.recorder
}

// TempFile mocks base method
func (m *MockIOUtil) TempFile(arg0, arg1 string) (oswrapper.File, error) {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "TempFile", arg0, arg1)
	ret0, _ := ret[0].(oswrapper.File)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// TempFile indicates an expected call of TempFile
func (mr *MockIOUtilMockRecorder) TempFile(arg0, arg1 interface{}) *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TempFile", reflect.TypeOf((*MockIOUtil)(nil).TempFile), arg0, arg1)
}

// WriteFile mocks base method
func (m *MockIOUtil) WriteFile(arg0 string, arg1 []byte, arg2 os.FileMode) error {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "WriteFile", arg0, arg1, arg2)
	ret0, _ := ret[0].(error)
	return ret0
}

// WriteFile indicates an expected call of WriteFile
func (mr *MockIOUtilMockRecorder) WriteFile(arg0, arg1, arg2 interface{}) *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteFile", reflect.TypeOf((*MockIOUtil)(nil).WriteFile), arg0, arg1, arg2)
}
<commit_msg>run make gogenerate<commit_after>// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//     http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.
//

// Code generated by MockGen. DO NOT EDIT.
// Source: github.com/aws/amazon-ecs-agent/agent/utils/ioutilwrapper (interfaces: IOUtil)

// Package mock_ioutilwrapper is a generated GoMock package.
package mock_ioutilwrapper

import (
	fs "io/fs"
	reflect "reflect"

	oswrapper "github.com/aws/amazon-ecs-agent/agent/utils/oswrapper"
	gomock "github.com/golang/mock/gomock"
)

// MockIOUtil is a mock of IOUtil interface
type MockIOUtil struct {
	ctrl     *gomock.Controller
	recorder *MockIOUtilMockRecorder
}

// MockIOUtilMockRecorder is the mock recorder for MockIOUtil
type MockIOUtilMockRecorder struct {
	mock *MockIOUtil
}

// NewMockIOUtil creates a new mock instance
func NewMockIOUtil(ctrl *gomock.Controller) *MockIOUtil {
	mock := &MockIOUtil{ctrl: ctrl}
	mock.recorder = &MockIOUtilMockRecorder{mock}
	return mock
}

// EXPECT returns an object that allows the caller to indicate expected use
func (m *MockIOUtil) EXPECT() *MockIOUtilMockRecorder {
	return m.recorder
}

// TempFile mocks base method
func (m *MockIOUtil) TempFile(arg0, arg1 string) (oswrapper.File, error) {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "TempFile", arg0, arg1)
	ret0, _ := ret[0].(oswrapper.File)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// TempFile indicates an expected call of TempFile
func (mr *MockIOUtilMockRecorder) TempFile(arg0, arg1 interface{}) *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TempFile", reflect.TypeOf((*MockIOUtil)(nil).TempFile), arg0, arg1)
}

// WriteFile mocks base method
func (m *MockIOUtil) WriteFile(arg0 string, arg1 []byte, arg2 fs.FileMode) error {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "WriteFile", arg0, arg1, arg2)
	ret0, _ := ret[0].(error)
	return ret0
}

// WriteFile indicates an expected call of WriteFile
func (mr *MockIOUtilMockRecorder) WriteFile(arg0, arg1, arg2 interface{}) *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteFile", reflect.TypeOf((*MockIOUtil)(nil).WriteFile), arg0, arg1, arg2)
}
<|endoftext|>


<commit_before>// Copyright 2019 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pilot

import (
	"fmt"
	"math"
	"strings"
	"sync"
	"testing"
	"time"

	"istio.io/istio/pkg/test/util/retry"
	"istio.io/istio/tests/util"

	envoyAdmin "github.com/envoyproxy/go-control-plane/envoy/admin/v2alpha"
	"github.com/hashicorp/go-multierror"

	"istio.io/istio/pkg/test/framework"
	"istio.io/istio/pkg/test/framework/components/echo"
	"istio.io/istio/pkg/test/framework/components/echo/echoboot"
	"istio.io/istio/pkg/test/framework/components/environment"
	"istio.io/istio/pkg/test/framework/components/namespace"
	"istio.io/istio/pkg/test/util/file"
	"istio.io/istio/pkg/test/util/structpath"
	"istio.io/istio/pkg/test/util/tmpl"
	"istio.io/istio/tests/integration/pilot/outboundtrafficpolicy"
)

//	Virtual service topology
//
//	    a                      b                     c
//	|-------|             |-------|    mirror   |-------|
//	| Host0 | ----------> | Host1 | ----------> | Host2 |
//	|-------|             |-------|             |-------|
//

type VirtualServiceMirrorConfig struct {
	Name       string
	Namespace  string
	Absent     bool
	Percent    float64
	MirrorHost string
}

type testCaseMirror struct {
	name       string
	absent     bool
	percentage float64
	threshold  float64
}

func TestMirroring(t *testing.T) {
	cases := []testCaseMirror{
		{
			name:       "mirror-percent-absent",
			absent:     true,
			percentage: 100.0,
			threshold:  0.0,
		},
		{
			name:       "mirror-50",
			percentage: 50.0,
			threshold:  10.0,
		},
		{
			name:       "mirror-10",
			percentage: 10.0,
			threshold:  5.0,
		},
		{
			name:       "mirror-80",
			percentage: 80.0,
			threshold:  10.0,
		},
		{
			name:       "mirror-0",
			percentage: 0.0,
			threshold:  0.0,
		},
	}

	framework.
		NewTest(t).
		RequiresEnvironment(environment.Kube).
		Run(func(ctx framework.TestContext) {
			ns := namespace.NewOrFail(t, ctx, namespace.Config{
				Prefix: "mirroring",
				Inject: true,
			})

			var instances [3]echo.Instance
			echoboot.NewBuilderOrFail(t, ctx).
				With(&instances[0], echoConfig(ns, "a")). // client
				With(&instances[1], echoConfig(ns, "b")). // target
				With(&instances[2], echoConfig(ns, "c")). // receives mirrored requests
				BuildOrFail(t)

			for _, c := range cases {
				t.Run(c.name, func(t *testing.T) {
					vsc := VirtualServiceMirrorConfig{
						c.name,
						ns.Name(),
						c.absent,
						c.percentage,
						instances[2].Config().Service,
					}

					deployment := tmpl.EvaluateOrFail(t,
						file.AsStringOrFail(t, "testdata/traffic-mirroring-template.yaml"), vsc)
					g.ApplyConfigOrFail(t, ns, deployment)
					defer g.DeleteConfigOrFail(t, ns, deployment)

					workloads, err := instances[0].Workloads()
					if err != nil {
						t.Fatalf("Failed to get workloads. Error: %v", err)
					}
					mirrorClusterName := fmt.Sprintf("%s.%s.svc.%s", instances[2].Config().Service, instances[2].Config().Namespace.Name(), instances[2].Config().Domain)

					for _, w := range workloads {
						if err = w.Sidecar().WaitForConfig(func(cfg *envoyAdmin.ConfigDump) (bool, error) {
							validator := structpath.ForProto(cfg)
							if err = checkIfMirrorWasApplied(instances[1], mirrorClusterName, c, validator); err != nil {
								return false, err
							}
							return true, nil
						}); err != nil {
							t.Fatalf("Failed to apply configuration. Error: %v", err)
						}
					}

					testID := util.RandomString(16)
					sendTrafficMirror(t, instances, testID)
					verifyTrafficMirror(t, instances, c, testID)
				})
			}
		})
}

func checkIfMirrorWasApplied(target echo.Instance, mirrorClusterName string, tc testCaseMirror, validator *structpath.Instance) error {
	for _, port := range target.Config().Ports {
		vsName := vsName(target, port)
		instance := validator.Select(
			"{.configs[*].dynamicRouteConfigs[*].routeConfig.virtualHosts[?(@.name == %q)].routes[*].route}",
			vsName)

		if tc.percentage > 0 {
			instance.Exists("{.requestMirrorPolicy}")

			clusterName := fmt.Sprintf("outbound|%d||%s", port.ServicePort, mirrorClusterName)
			instance.Equals(clusterName, "{.requestMirrorPolicy.cluster}")

			instance.Equals(tc.percentage, "{.requestMirrorPolicy.runtimeFraction.defaultValue.numerator}")
		} else {
			instance.NotExists("{.requestMirrorPolicy}")
		}

		if err := instance.Check(); err != nil {
			return err
		}
	}
	return nil
}

func vsName(target echo.Instance, port echo.Port) string {
	cfg := target.Config()
	return fmt.Sprintf("%s.%s.svc.%s:%d", cfg.Service, cfg.Namespace.Name(), cfg.Domain, port.ServicePort)
}

func sendTrafficMirror(t *testing.T, instances [3]echo.Instance, testID string) {
	const totalThreads = 10
	errs := make(chan error, totalThreads)

	wg := sync.WaitGroup{}
	wg.Add(totalThreads)

	for i := 0; i < totalThreads; i++ {
		go func() {
			_, err := instances[0].Call(echo.CallOptions{
				Target:   instances[1],
				PortName: "http",
				Path:     "/" + testID,
				Count:    50,
			})
			if err != nil {
				errs <- err
			}
			wg.Done()
		}()
	}

	wg.Wait()
	close(errs)

	var callerrors error
	for err := range errs {
		callerrors = multierror.Append(err, callerrors)
	}
	if callerrors != nil {
		t.Fatalf("Error occurred during call: %v", callerrors)
	}
}

func verifyTrafficMirror(t *testing.T, instances [3]echo.Instance, tc testCaseMirror, testID string) {
	_, err := retry.Do(func() (interface{}, bool, error) {
		countB := logCount(t, instances[1], testID)
		countC := logCount(t, instances[2], testID)
		actualPercent := (countC / countB) * 100
		deltaFromExpected := math.Abs(actualPercent - tc.percentage)

		if tc.threshold-deltaFromExpected < 0 {
			err := fmt.Errorf("unexpected mirror traffic. Expected %g%%, got %.1f%% (threshold: %g%%, testID: %s)",
				tc.percentage, actualPercent, tc.threshold, testID)
			t.Logf("%v", err)
			return nil, false, err
		}

		t.Logf("Got expected mirror traffic. Expected %g%%, got %.1f%% (threshold: %g%%, , testID: %s)",
			tc.percentage, actualPercent, tc.threshold, testID)
		return nil, true, nil
	}, retry.Delay(time.Second))

	if err != nil {
		t.Fatalf("%v", err)
	}
}

func logCount(t *testing.T, instance echo.Instance, testID string) float64 {
	workloads, err := instance.Workloads()
	if err != nil {
		t.Fatalf("Failed to get workloads. Error: %v", err)
	}

	var logs string
	for _, w := range workloads {
		logs += w.Sidecar().LogsOrFail(t)
	}

	return float64(strings.Count(logs, testID))
}

// Tests mirroring to an external service. Uses same topology as the test above, a -> b -> c, with "c" being external.
//
// Since we don't want to rely on actual external websites, we simulate that by using a Sidecar to limit connectivity
// from "a" so that it cannot reach "c" directly, and we use a ServiceEntry to define our "external" website, which
// is static and points to the service "c" ip.

// Thus when "a" tries to mirror to the external service, it is actually connecting to "c" (which is not part of the
// mesh because of the Sidecar), then we can inspect "c" logs to verify the requests were properly mirrored.

const (
	fakeExternalURL = "external-website-url-just-for-testing.extension"

	serviceEntry = `
apiVersion: networking.istio.io/v1alpha3
kind: ServiceEntry
metadata:
  name: external-service
spec:
  hosts:
  - %s
  location: MESH_EXTERNAL
  ports:
  - name: http
    number: 80
    protocol: HTTP
  resolution: STATIC
  endpoints:
  - address: %s
`

	sidecar = `
apiVersion: networking.istio.io/v1alpha3
kind: Sidecar
metadata:
  name: restrict-to-service-entry
spec:
  egress:
  - hosts:
    - "istio-system/*"
    - "./b.%s.svc.%s"
    - "*/%s"
  outboundTrafficPolicy:
    mode: REGISTRY_ONLY
`
)

func TestMirroringExternalService(t *testing.T) {
	framework.
		NewTest(t).
		RequiresEnvironment(environment.Kube).
		Run(func(ctx framework.TestContext) {
			ns := namespace.NewOrFail(t, ctx, namespace.Config{
				Prefix: "external",
				Inject: true,
			})

			var instances [3]echo.Instance
			echoboot.NewBuilderOrFail(t, ctx).
				With(&instances[0], echoConfig(ns, "a")). // client
				With(&instances[1], echoConfig(ns, "b")). // target
				With(&instances[2], echoConfig(ns, "c")). // receives mirrored requests
				BuildOrFail(t)

			g.ApplyConfigOrFail(t, ns, fmt.Sprintf(sidecar, ns.Name(), instances[1].Config().Domain, fakeExternalURL))
			g.ApplyConfigOrFail(t, ns, fmt.Sprintf(serviceEntry, fakeExternalURL, instances[2].Address()))
			if err := outboundtrafficpolicy.WaitUntilNotCallable(instances[0], instances[2]); err != nil {
				t.Fatalf("failed to apply sidecar, %v", err)
			}

			c := testCaseMirror{
				name:       "mirror-external",
				absent:     true,
				percentage: 100.0,
				threshold:  0.0,
			}
			vsc := VirtualServiceMirrorConfig{
				"mirror-external",
				ns.Name(),
				true,
				100,
				fakeExternalURL,
			}

			deployment := tmpl.EvaluateOrFail(t,
				file.AsStringOrFail(t, "testdata/traffic-mirroring-template.yaml"), vsc)
			g.ApplyConfigOrFail(t, ns, deployment)

			workloads, err := instances[0].Workloads()
			if err != nil {
				t.Fatalf("Failed to get workloads. Error: %v", err)
			}

			for _, w := range workloads {
				if err = w.Sidecar().WaitForConfig(func(cfg *envoyAdmin.ConfigDump) (bool, error) {
					validator := structpath.ForProto(cfg)
					if err = checkIfMirrorWasApplied(instances[1], fakeExternalURL, c, validator); err != nil {
						return false, err
					}
					return true, nil
				}); err != nil {
					t.Fatalf("Failed to apply configuration. Error: %v", err)
				}
			}

			testID := util.RandomString(16)
			sendTrafficMirror(t, instances, testID)
			verifyTrafficMirror(t, instances, c, testID)
		})
}
<commit_msg>Also make GRPC requests on mirroring test (#17459)<commit_after>// Copyright 2019 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pilot

import (
	"fmt"
	"math"
	"strings"
	"sync"
	"testing"
	"time"

	"istio.io/istio/pkg/test/util/retry"
	"istio.io/istio/tests/util"

	envoyAdmin "github.com/envoyproxy/go-control-plane/envoy/admin/v2alpha"
	"github.com/hashicorp/go-multierror"

	"istio.io/istio/pkg/config/protocol"
	"istio.io/istio/pkg/test/framework"
	"istio.io/istio/pkg/test/framework/components/echo"
	"istio.io/istio/pkg/test/framework/components/echo/echoboot"
	"istio.io/istio/pkg/test/framework/components/environment"
	"istio.io/istio/pkg/test/framework/components/namespace"
	"istio.io/istio/pkg/test/util/file"
	"istio.io/istio/pkg/test/util/structpath"
	"istio.io/istio/pkg/test/util/tmpl"
	"istio.io/istio/tests/integration/pilot/outboundtrafficpolicy"
)

//	Virtual service topology
//
//	    a                      b                     c
//	|-------|             |-------|    mirror   |-------|
//	| Host0 | ----------> | Host1 | ----------> | Host2 |
//	|-------|             |-------|             |-------|
//

type VirtualServiceMirrorConfig struct {
	Name       string
	Namespace  string
	Absent     bool
	Percent    float64
	MirrorHost string
}

type testCaseMirror struct {
	name       string
	absent     bool
	percentage float64
	threshold  float64
}

var (
	mirrorProtocols = []protocol.Instance{protocol.HTTP, protocol.GRPC}
)

func TestMirroring(t *testing.T) {
	cases := []testCaseMirror{
		{
			name:       "mirror-percent-absent",
			absent:     true,
			percentage: 100.0,
			threshold:  0.0,
		},
		{
			name:       "mirror-50",
			percentage: 50.0,
			threshold:  10.0,
		},
		{
			name:       "mirror-10",
			percentage: 10.0,
			threshold:  5.0,
		},
		{
			name:       "mirror-80",
			percentage: 80.0,
			threshold:  10.0,
		},
		{
			name:       "mirror-0",
			percentage: 0.0,
			threshold:  0.0,
		},
	}

	framework.
		NewTest(t).
		RequiresEnvironment(environment.Kube).
		Run(func(ctx framework.TestContext) {
			ns := namespace.NewOrFail(t, ctx, namespace.Config{
				Prefix: "mirroring",
				Inject: true,
			})

			var instances [3]echo.Instance
			echoboot.NewBuilderOrFail(t, ctx).
				With(&instances[0], echoConfig(ns, "a")). // client
				With(&instances[1], echoConfig(ns, "b")). // target
				With(&instances[2], echoConfig(ns, "c")). // receives mirrored requests
				BuildOrFail(t)

			for _, c := range cases {
				t.Run(c.name, func(t *testing.T) {
					vsc := VirtualServiceMirrorConfig{
						c.name,
						ns.Name(),
						c.absent,
						c.percentage,
						instances[2].Config().Service,
					}

					deployment := tmpl.EvaluateOrFail(t,
						file.AsStringOrFail(t, "testdata/traffic-mirroring-template.yaml"), vsc)
					g.ApplyConfigOrFail(t, ns, deployment)
					defer g.DeleteConfigOrFail(t, ns, deployment)

					workloads, err := instances[0].Workloads()
					if err != nil {
						t.Fatalf("Failed to get workloads. Error: %v", err)
					}
					mirrorClusterName := fmt.Sprintf("%s.%s.svc.%s", instances[2].Config().Service, instances[2].Config().Namespace.Name(), instances[2].Config().Domain)

					for _, w := range workloads {
						if err = w.Sidecar().WaitForConfig(func(cfg *envoyAdmin.ConfigDump) (bool, error) {
							validator := structpath.ForProto(cfg)
							if err = checkIfMirrorWasApplied(instances[1], mirrorClusterName, c, validator); err != nil {
								return false, err
							}
							return true, nil
						}); err != nil {
							t.Fatalf("Failed to apply configuration. Error: %v", err)
						}
					}

					for _, proto := range mirrorProtocols {
						t.Run(string(proto), func(t *testing.T) {
							testID := util.RandomString(16)
							sendTrafficMirror(t, instances, proto, testID)
							verifyTrafficMirror(t, instances, c, testID)
						})
					}
				})
			}
		})
}

func checkIfMirrorWasApplied(target echo.Instance, mirrorClusterName string, tc testCaseMirror, validator *structpath.Instance) error {
	for _, port := range target.Config().Ports {
		vsName := vsName(target, port)
		instance := validator.Select(
			"{.configs[*].dynamicRouteConfigs[*].routeConfig.virtualHosts[?(@.name == %q)].routes[*].route}",
			vsName)

		if tc.percentage > 0 {
			instance.Exists("{.requestMirrorPolicy}")

			clusterName := fmt.Sprintf("outbound|%d||%s", port.ServicePort, mirrorClusterName)
			instance.Equals(clusterName, "{.requestMirrorPolicy.cluster}")

			instance.Equals(tc.percentage, "{.requestMirrorPolicy.runtimeFraction.defaultValue.numerator}")
		} else {
			instance.NotExists("{.requestMirrorPolicy}")
		}

		if err := instance.Check(); err != nil {
			return err
		}
	}
	return nil
}

func vsName(target echo.Instance, port echo.Port) string {
	cfg := target.Config()
	return fmt.Sprintf("%s.%s.svc.%s:%d", cfg.Service, cfg.Namespace.Name(), cfg.Domain, port.ServicePort)
}

func sendTrafficMirror(t *testing.T, instances [3]echo.Instance, proto protocol.Instance, testID string) {
	options := echo.CallOptions{
		Target:   instances[1],
		Count:    50,
		PortName: strings.ToLower(string(proto)),
	}
	switch proto {
	case protocol.HTTP:
		options.Path = "/" + testID
	case protocol.GRPC:
		options.Message = testID
	default:
		t.Fatalf("protocol not supported in mirror testing: %s", proto)
	}
	const totalThreads = 10
	errs := make(chan error, totalThreads)

	wg := sync.WaitGroup{}
	wg.Add(totalThreads)

	for i := 0; i < totalThreads; i++ {
		go func() {
			_, err := instances[0].Call(options)
			if err != nil {
				errs <- err
			}
			wg.Done()
		}()
	}

	wg.Wait()
	close(errs)

	var callerrors error
	for err := range errs {
		callerrors = multierror.Append(err, callerrors)
	}
	if callerrors != nil {
		t.Fatalf("Error occurred during call: %v", callerrors)
	}
}

func verifyTrafficMirror(t *testing.T, instances [3]echo.Instance, tc testCaseMirror, testID string) {
	_, err := retry.Do(func() (interface{}, bool, error) {
		countB := logCount(t, instances[1], testID)
		countC := logCount(t, instances[2], testID)
		actualPercent := (countC / countB) * 100
		deltaFromExpected := math.Abs(actualPercent - tc.percentage)

		if tc.threshold-deltaFromExpected < 0 {
			err := fmt.Errorf("unexpected mirror traffic. Expected %g%%, got %.1f%% (threshold: %g%%, testID: %s)",
				tc.percentage, actualPercent, tc.threshold, testID)
			t.Logf("%v", err)
			return nil, false, err
		}

		t.Logf("Got expected mirror traffic. Expected %g%%, got %.1f%% (threshold: %g%%, , testID: %s)",
			tc.percentage, actualPercent, tc.threshold, testID)
		return nil, true, nil
	}, retry.Delay(time.Second))

	if err != nil {
		t.Fatalf("%v", err)
	}
}

func logCount(t *testing.T, instance echo.Instance, testID string) float64 {
	workloads, err := instance.Workloads()
	if err != nil {
		t.Fatalf("Failed to get workloads. Error: %v", err)
	}

	var logs string
	for _, w := range workloads {
		logs += w.LogsOrFail(t)
	}

	return float64(strings.Count(logs, testID))
}

// Tests mirroring to an external service. Uses same topology as the test above, a -> b -> c, with "c" being external.
//
// Since we don't want to rely on actual external websites, we simulate that by using a Sidecar to limit connectivity
// from "a" so that it cannot reach "c" directly, and we use a ServiceEntry to define our "external" website, which
// is static and points to the service "c" ip.

// Thus when "a" tries to mirror to the external service, it is actually connecting to "c" (which is not part of the
// mesh because of the Sidecar), then we can inspect "c" logs to verify the requests were properly mirrored.

const (
	fakeExternalURL = "external-website-url-just-for-testing.extension"

	serviceEntry = `
apiVersion: networking.istio.io/v1alpha3
kind: ServiceEntry
metadata:
  name: external-service
spec:
  hosts:
  - %s
  location: MESH_EXTERNAL
  ports:
  - name: http
    number: 80
    protocol: HTTP
  - name: grpc
    number: 7070
    protocol: GRPC
  resolution: STATIC
  endpoints:
  - address: %s
`

	sidecar = `
apiVersion: networking.istio.io/v1alpha3
kind: Sidecar
metadata:
  name: restrict-to-service-entry
spec:
  egress:
  - hosts:
    - "istio-system/*"
    - "./b.%s.svc.%s"
    - "*/%s"
  outboundTrafficPolicy:
    mode: REGISTRY_ONLY
`
)

func TestMirroringExternalService(t *testing.T) {
	framework.
		NewTest(t).
		RequiresEnvironment(environment.Kube).
		Run(func(ctx framework.TestContext) {
			ns := namespace.NewOrFail(t, ctx, namespace.Config{
				Prefix: "external",
				Inject: true,
			})

			var instances [3]echo.Instance
			echoboot.NewBuilderOrFail(t, ctx).
				With(&instances[0], echoConfig(ns, "a")). // client
				With(&instances[1], echoConfig(ns, "b")). // target
				With(&instances[2], echoConfig(ns, "c")). // receives mirrored requests
				BuildOrFail(t)

			g.ApplyConfigOrFail(t, ns, fmt.Sprintf(sidecar, ns.Name(), instances[1].Config().Domain, fakeExternalURL))
			g.ApplyConfigOrFail(t, ns, fmt.Sprintf(serviceEntry, fakeExternalURL, instances[2].Address()))
			if err := outboundtrafficpolicy.WaitUntilNotCallable(instances[0], instances[2]); err != nil {
				t.Fatalf("failed to apply sidecar, %v", err)
			}

			c := testCaseMirror{
				name:       "mirror-external",
				absent:     true,
				percentage: 100.0,
				threshold:  0.0,
			}
			vsc := VirtualServiceMirrorConfig{
				"mirror-external",
				ns.Name(),
				true,
				100,
				fakeExternalURL,
			}

			deployment := tmpl.EvaluateOrFail(t,
				file.AsStringOrFail(t, "testdata/traffic-mirroring-template.yaml"), vsc)
			g.ApplyConfigOrFail(t, ns, deployment)

			workloads, err := instances[0].Workloads()
			if err != nil {
				t.Fatalf("Failed to get workloads. Error: %v", err)
			}

			for _, w := range workloads {
				if err = w.Sidecar().WaitForConfig(func(cfg *envoyAdmin.ConfigDump) (bool, error) {
					validator := structpath.ForProto(cfg)
					if err = checkIfMirrorWasApplied(instances[1], fakeExternalURL, c, validator); err != nil {
						return false, err
					}
					return true, nil
				}); err != nil {
					t.Fatalf("Failed to apply configuration. Error: %v", err)
				}
			}

			for _, proto := range mirrorProtocols {
				t.Run(string(proto), func(t *testing.T) {
					testID := util.RandomString(16)
					sendTrafficMirror(t, instances, proto, testID)
					verifyTrafficMirror(t, instances, c, testID)
				})
			}
		})
}
<|endoftext|>