<commit_before>package main

import "math"

// SurvivalDifficulty controls how the game difficulty scales with the player's score.
type SurvivalDifficulty interface {
	// NumBugInit returns the number of bugs to initializer the game with.
	NumBugInit() int

	// BugRateInit returns a constant amount of time between bug spawns during
	// initialization.
	BugRateInit() float64

	// BugDistribution returns the distribution of bugs and colors
	BugDistribution(lvl int) BugDistribution

	// NextLevel returns the total score required to achive the next level.
	NextLevel(lvl int) int64

	// BugRate returns the expected number of seconds between individual bug
	// spawns for the current level.  An exponential distribution will
	// determine the actual duration between each spawn.
	BugRate(lvl int) float64

	// ItemRate returns the expected number of seconds between individual item
	// spawns for the current level and the expected number of seconds for a
	// spawned item to despawn.  An exponential distribution will determine the
	// actual duration between each spawn.  Another exponential distribution
	// determins the duration each spawn exists.  Multiple items may exist at
	// the same time.
	ItemRate(lvl int) (spawn, despawn float64)
}

type simpleSurvivalDifficulty struct{}

func (s *simpleSurvivalDifficulty) NextLevel(lvl int) int64 {
	if lvl <= 0 {
		return 0
	}
	if lvl < 63 {
		return 1 << uint(lvl)
	}
	return -1
}

func (s *simpleSurvivalDifficulty) NumBugInit() int {
	return 12
}

func (s *simpleSurvivalDifficulty) BugRateInit() float64 {
	return 0.3
}

func (s *simpleSurvivalDifficulty) BugRate(lvl int) float64 {
	const initialRate = 7 // about every 5 seconds
	const baseReduction = 0.99
	return initialRate * math.Pow(baseReduction, float64(lvl))
}

func (s *simpleSurvivalDifficulty) ItemRate(lvl int) (spawn, despawn float64) {
	const initialSpawnRate = 10  // about every 10 seconds
	const initialDespawnRate = 5 // about 5 seconds
	const baseSpawnReduction = 0.90
	const baseDespawnReduction = 0.96
	spawn = initialSpawnRate * math.Pow(baseSpawnReduction, float64(lvl))
	despawn = initialDespawnRate * math.Pow(baseDespawnReduction, float64(lvl))
	return spawn, despawn
}

func (s *simpleSurvivalDifficulty) BugDistribution(lvl int) BugDistribution {
	if lvl < 3 {
		return &simpleDistribution{
			&bugTypeDistn{
				BugSmall:      500,
				BugLarge:      400,
				BugGnat:       200,
				BugMagic:      0,
				BugBomb:       0,
				BugLightning:  0,
				BugRock:       0,
				BugMultiChain: 0,
			},
			&bugColorCondDistn{
				BugSmall: {ColorBug + 0: 1},
				BugLarge: {ColorBug + 2: 1},
			},
		}
	}
	if lvl < 5 {
		return &simpleDistribution{
			&bugTypeDistn{
				BugSmall:      390,
				BugLarge:      385,
				BugGnat:       195,
				BugMagic:      0,
				BugBomb:       30,
				BugLightning:  0,
				BugRock:       0,
				BugMultiChain: 0,
			},
			&bugColorCondDistn{
				BugSmall: {ColorBug + 0: 1, ColorBug + 1: 1},
				BugLarge: {ColorBug + 2: 1, ColorBug + 3: 0},
			},
		}
	}
	if lvl == 6 {
		return &simpleDistribution{
			&bugTypeDistn{
				BugSmall:      380,
				BugLarge:      375,
				BugGnat:       192,
				BugMagic:      0,
				BugBomb:       15,
				BugLightning:  15,
				BugRock:       10,
				BugMultiChain: 10,
			},
			&bugColorCondDistn{
				BugSmall: {ColorBug + 0: 1, ColorBug + 1: 1},
				BugLarge: {ColorBug + 2: 1, ColorBug + 3: 1},
			},
		}
	}
	if lvl == 7 {
		return &simpleDistribution{
			&bugTypeDistn{
				BugSmall:      373,
				BugLarge:      363,
				BugGnat:       190,
				BugMagic:      10,
				BugBomb:       15,
				BugLightning:  15,
				BugRock:       15,
				BugMultiChain: 10,
			},
			&bugColorCondDistn{
				BugSmall: {ColorBug + 0: 1, ColorBug + 1: 1},
				BugLarge: {ColorBug + 2: 1, ColorBug + 3: 1},
			},
		}
	}
	if lvl == 8 {
		return &simpleDistribution{
			&bugTypeDistn{
				BugSmall:      378,
				BugLarge:      358,
				BugGnat:       180,
				BugMagic:      15,
				BugBomb:       15,
				BugLightning:  15,
				BugRock:       15,
				BugMultiChain: 15,
			},
			&bugColorCondDistn{
				BugSmall: {ColorBug + 0: 1, ColorBug + 1: 1},
				BugLarge: {ColorBug + 2: 1, ColorBug + 3: 1},
			},
		}
	}
	if lvl == 9 {
		return &simpleDistribution{
			&bugTypeDistn{
				BugSmall:      380,
				BugLarge:      350,
				BugGnat:       170,
				BugMagic:      20,
				BugBomb:       20,
				BugLightning:  20,
				BugRock:       20,
				BugMultiChain: 20,
			},
			&bugColorCondDistn{
				BugSmall: {ColorBug + 0: 1, ColorBug + 1: 1},
				BugLarge: {ColorBug + 2: 1, ColorBug + 3: 1},
			},
		}
	}
	if lvl == 10 {
		return &simpleDistribution{
			&bugTypeDistn{
				BugSmall:      383,
				BugLarge:      343,
				BugGnat:       160,
				BugMagic:      25,
				BugBomb:       25,
				BugLightning:  25,
				BugRock:       25,
				BugMultiChain: 25,
			},
			&bugColorCondDistn{
				BugSmall: {ColorBug + 0: 1, ColorBug + 1: 1},
				BugLarge: {ColorBug + 2: 1, ColorBug + 3: 1},
			},
		}
	}
	return &simpleDistribution{
		&bugTypeDistn{
			BugSmall:      390,
			BugLarge:      340,
			BugGnat:       150,
			BugMagic:      30,
			BugBomb:       30,
			BugLightning:  30,
			BugRock:       30,
			BugMultiChain: 30,
		},
		&bugColorCondDistn{
			BugSmall: {ColorBug + 0: 1, ColorBug + 1: 1},
			BugLarge: {ColorBug + 2: 1, ColorBug + 3: 1},
		},
	}
}
<commit_msg>adjust the level formula to avoid blow through the first levels.<commit_after>package main

import "math"

const defaultSurvivalLevelOne = 30
const defaultSurvivalLevelBase = 1.5

// SurvivalDifficulty controls how the game difficulty scales with the player's score.
type SurvivalDifficulty interface {
	// NumBugInit returns the number of bugs to initializer the game with.
	NumBugInit() int

	// BugRateInit returns a constant amount of time between bug spawns during
	// initialization.
	BugRateInit() float64

	// BugDistribution returns the distribution of bugs and colors
	BugDistribution(lvl int) BugDistribution

	// NextLevel returns the total score required to achive the next level.
	NextLevel(lvl int) int64

	// BugRate returns the expected number of seconds between individual bug
	// spawns for the current level.  An exponential distribution will
	// determine the actual duration between each spawn.
	BugRate(lvl int) float64

	// ItemRate returns the expected number of seconds between individual item
	// spawns for the current level and the expected number of seconds for a
	// spawned item to despawn.  An exponential distribution will determine the
	// actual duration between each spawn.  Another exponential distribution
	// determins the duration each spawn exists.  Multiple items may exist at
	// the same time.
	ItemRate(lvl int) (spawn, despawn float64)
}

type simpleSurvivalDifficulty struct{}

func (s *simpleSurvivalDifficulty) NextLevel(lvl int) int64 {
	return int64(float64(defaultSurvivalLevelOne) * math.Pow(defaultSurvivalLevelBase, float64(lvl)))
}

func (s *simpleSurvivalDifficulty) NumBugInit() int {
	return 12
}

func (s *simpleSurvivalDifficulty) BugRateInit() float64 {
	return 0.3
}

func (s *simpleSurvivalDifficulty) BugRate(lvl int) float64 {
	const initialRate = 7 // about every 5 seconds
	const baseReduction = 0.99
	return initialRate * math.Pow(baseReduction, float64(lvl))
}

func (s *simpleSurvivalDifficulty) ItemRate(lvl int) (spawn, despawn float64) {
	const initialSpawnRate = 10  // about every 10 seconds
	const initialDespawnRate = 5 // about 5 seconds
	const baseSpawnReduction = 0.90
	const baseDespawnReduction = 0.96
	spawn = initialSpawnRate * math.Pow(baseSpawnReduction, float64(lvl))
	despawn = initialDespawnRate * math.Pow(baseDespawnReduction, float64(lvl))
	return spawn, despawn
}

func (s *simpleSurvivalDifficulty) BugDistribution(lvl int) BugDistribution {
	if lvl < 3 {
		return &simpleDistribution{
			&bugTypeDistn{
				BugSmall:      500,
				BugLarge:      400,
				BugGnat:       200,
				BugMagic:      0,
				BugBomb:       0,
				BugLightning:  0,
				BugRock:       0,
				BugMultiChain: 0,
			},
			&bugColorCondDistn{
				BugSmall: {ColorBug + 0: 1},
				BugLarge: {ColorBug + 2: 1},
			},
		}
	}
	if lvl < 5 {
		return &simpleDistribution{
			&bugTypeDistn{
				BugSmall:      390,
				BugLarge:      385,
				BugGnat:       195,
				BugMagic:      0,
				BugBomb:       30,
				BugLightning:  0,
				BugRock:       0,
				BugMultiChain: 0,
			},
			&bugColorCondDistn{
				BugSmall: {ColorBug + 0: 1, ColorBug + 1: 1},
				BugLarge: {ColorBug + 2: 1, ColorBug + 3: 0},
			},
		}
	}
	if lvl == 6 {
		return &simpleDistribution{
			&bugTypeDistn{
				BugSmall:      380,
				BugLarge:      375,
				BugGnat:       192,
				BugMagic:      0,
				BugBomb:       15,
				BugLightning:  15,
				BugRock:       10,
				BugMultiChain: 10,
			},
			&bugColorCondDistn{
				BugSmall: {ColorBug + 0: 1, ColorBug + 1: 1},
				BugLarge: {ColorBug + 2: 1, ColorBug + 3: 1},
			},
		}
	}
	if lvl == 7 {
		return &simpleDistribution{
			&bugTypeDistn{
				BugSmall:      373,
				BugLarge:      363,
				BugGnat:       190,
				BugMagic:      10,
				BugBomb:       15,
				BugLightning:  15,
				BugRock:       15,
				BugMultiChain: 10,
			},
			&bugColorCondDistn{
				BugSmall: {ColorBug + 0: 1, ColorBug + 1: 1},
				BugLarge: {ColorBug + 2: 1, ColorBug + 3: 1},
			},
		}
	}
	if lvl == 8 {
		return &simpleDistribution{
			&bugTypeDistn{
				BugSmall:      378,
				BugLarge:      358,
				BugGnat:       180,
				BugMagic:      15,
				BugBomb:       15,
				BugLightning:  15,
				BugRock:       15,
				BugMultiChain: 15,
			},
			&bugColorCondDistn{
				BugSmall: {ColorBug + 0: 1, ColorBug + 1: 1},
				BugLarge: {ColorBug + 2: 1, ColorBug + 3: 1},
			},
		}
	}
	if lvl == 9 {
		return &simpleDistribution{
			&bugTypeDistn{
				BugSmall:      380,
				BugLarge:      350,
				BugGnat:       170,
				BugMagic:      20,
				BugBomb:       20,
				BugLightning:  20,
				BugRock:       20,
				BugMultiChain: 20,
			},
			&bugColorCondDistn{
				BugSmall: {ColorBug + 0: 1, ColorBug + 1: 1},
				BugLarge: {ColorBug + 2: 1, ColorBug + 3: 1},
			},
		}
	}
	if lvl == 10 {
		return &simpleDistribution{
			&bugTypeDistn{
				BugSmall:      383,
				BugLarge:      343,
				BugGnat:       160,
				BugMagic:      25,
				BugBomb:       25,
				BugLightning:  25,
				BugRock:       25,
				BugMultiChain: 25,
			},
			&bugColorCondDistn{
				BugSmall: {ColorBug + 0: 1, ColorBug + 1: 1},
				BugLarge: {ColorBug + 2: 1, ColorBug + 3: 1},
			},
		}
	}
	return &simpleDistribution{
		&bugTypeDistn{
			BugSmall:      390,
			BugLarge:      340,
			BugGnat:       150,
			BugMagic:      30,
			BugBomb:       30,
			BugLightning:  30,
			BugRock:       30,
			BugMultiChain: 30,
		},
		&bugColorCondDistn{
			BugSmall: {ColorBug + 0: 1, ColorBug + 1: 1},
			BugLarge: {ColorBug + 2: 1, ColorBug + 3: 1},
		},
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/sirupsen/logrus"

	"k8s.io/apimachinery/pkg/util/sets"

	"k8s.io/test-infra/prow/config"
	"k8s.io/test-infra/prow/kube"
	"k8s.io/test-infra/prow/logrusutil"
	"k8s.io/test-infra/prow/pjutil"
)

var (
	submitQueueURL = flag.String("submit-queue-endpoint", "http://submit-queue.k8s.io/github-e2e-queue", "Submit Queue status URL")
	remoteURL      = flag.String("remote-url", "https://github.com/kubernetes/kubernetes", "Remote Git URL")
	orgName        = flag.String("org", "kubernetes", "Org name")
	repoName       = flag.String("repo", "kubernetes", "Repo name")
	configPath     = flag.String("config-path", "/etc/config/config.yaml", "Path to config.yaml.")
	jobConfigPath  = flag.String("job-config-path", "", "Path to prow job configs.")
	maxBatchSize   = flag.Int("batch-size", 5, "Maximum batch size")
	alwaysRun      = flag.String("always-run", "", "Job names that should be treated as always_run: true in Splice")
)

// Call a binary and return its output and success status.
func call(binary string, args ...string) (string, error) {
	cmdout := "+ " + binary + " "
	for _, arg := range args {
		cmdout += arg + " "
	}
	logrus.Info(cmdout)

	cmd := exec.Command(binary, args...)
	output, err := cmd.CombinedOutput()
	return string(output), err
}

// getQueuedPRs reads the list of queued PRs from the Submit Queue.
func getQueuedPRs(url string) ([]int, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	queue := struct {
		E2EQueue []struct {
			Number  int
			BaseRef string
		}
	}{}
	err = json.Unmarshal(body, &queue)
	if err != nil {
		return nil, err
	}

	ret := []int{}
	for _, e := range queue.E2EQueue {
		if e.BaseRef == "" || e.BaseRef == "master" {
			ret = append(ret, e.Number)
		}
	}
	return ret, nil
}

// Splicer manages a git repo in specific directory.
type splicer struct {
	dir string // The repository location.
}

// makeSplicer returns a splicer in a new temporary directory,
// with an initial .git dir created.
func makeSplicer() (*splicer, error) {
	dir, err := ioutil.TempDir("", "splice_")
	if err != nil {
		return nil, err
	}
	s := &splicer{dir}
	err = s.gitCalls([][]string{
		{"init"},
		{"config", "--local", "user.name", "K8S Prow Splice"},
		{"config", "--local", "user.email", "splice@localhost"},
		{"config", "--local", "commit.gpgsign", "false"},
	})
	if err != nil {
		s.cleanup()
		return nil, err
	}
	logrus.Infof("Splicer created in %s.", dir)
	return s, nil
}

// cleanup recurisvely deletes the repository
func (s *splicer) cleanup() {
	os.RemoveAll(s.dir)
}

// gitCall is a helper to call `git -C $path $args`.
func (s *splicer) gitCall(args ...string) error {
	fullArgs := append([]string{"-C", s.dir}, args...)
	output, err := call("git", fullArgs...)
	if len(output) > 0 {
		logrus.Info(output)
	}
	return err
}

// gitCalls is a helper to chain repeated gitCall invocations,
// returning the first failure, or nil if they all succeeded.
func (s *splicer) gitCalls(argsList [][]string) error {
	for _, args := range argsList {
		err := s.gitCall(args...)
		if err != nil {
			return err
		}
	}
	return nil
}

// findMergeable fetches given PRs from upstream, merges them locally,
// and finally returns a list of PRs that can be merged without conflicts.
func (s *splicer) findMergeable(remote string, prs []int) ([]int, error) {
	args := []string{"fetch", "-f", remote, "master:master"}
	for _, pr := range prs {
		args = append(args, fmt.Sprintf("pull/%d/head:pr/%d", pr, pr))
	}

	err := s.gitCalls([][]string{
		{"reset", "--hard"},
		{"checkout", "--orphan", "blank"},
		{"reset", "--hard"},
		{"clean", "-fdx"},
		args,
		{"checkout", "-B", "batch", "master"},
	})
	if err != nil {
		return nil, err
	}

	out := []int{}
	for _, pr := range prs {
		err := s.gitCall("merge", "--no-ff", "--no-stat",
			"-m", fmt.Sprintf("merge #%d", pr),
			fmt.Sprintf("pr/%d", pr))
		if err != nil {
			// merge conflict: cleanup and move on
			err = s.gitCall("merge", "--abort")
			if err != nil {
				return nil, err
			}
			continue
		}
		out = append(out, pr)
	}
	return out, nil
}

// gitRef returns the SHA for the given git object-- a branch, generally.
func (s *splicer) gitRef(ref string) string {
	output, err := call("git", "-C", s.dir, "rev-parse", ref)
	if err != nil {
		return ""
	}
	return strings.TrimSpace(output)
}

// Produce a kube.Refs for the given pull requests. This involves computing the
// git ref for master and the PRs.
func (s *splicer) makeBuildRefs(org, repo string, prs []int) kube.Refs {
	refs := kube.Refs{
		Org:     org,
		Repo:    repo,
		BaseRef: "master",
		BaseSHA: s.gitRef("master"),
	}
	for _, pr := range prs {
		branch := fmt.Sprintf("pr/%d", pr)
		refs.Pulls = append(refs.Pulls, kube.Pull{Number: pr, SHA: s.gitRef(branch)})
	}
	return refs
}

// Filters to the list of jobs which already passed this commit
func completedJobs(currentJobs []kube.ProwJob, refs kube.Refs) []kube.ProwJob {
	var skippable []kube.ProwJob
	rs := refs.String()

	for _, job := range currentJobs {
		if job.Spec.Type != kube.BatchJob {
			continue
		}
		if !job.Complete() {
			continue
		}
		if job.Status.State != kube.SuccessState {
			continue
		}
		if job.Spec.Refs.String() != rs {
			continue
		}
		skippable = append(skippable, job)
	}
	return skippable
}

// Filters to the list of required presubmits that report
func requiredPresubmits(presubmits []config.Presubmit, alwaysRunOverride sets.String) []config.Presubmit {
	var out []config.Presubmit
	for _, job := range presubmits {
		if !job.AlwaysRun && !alwaysRunOverride.Has(job.Name) { // Ignore manual jobs as these do not block
			continue
		}
		if job.SkipReport { // Ignore silent jobs as these do not block
			continue
		}
		if !job.RunsAgainstBranch("master") { // Ignore jobs that don't run on master
			continue
		}
		out = append(out, job)
	}
	return out
}

// Filters to the list of required presubmit which have not already passed this commit
func neededPresubmits(presubmits []config.Presubmit, currentJobs []kube.ProwJob, refs kube.Refs, alwaysRunOverride sets.String) []config.Presubmit {
	skippable := make(map[string]bool)
	for _, job := range completedJobs(currentJobs, refs) {
		skippable[job.Spec.Context] = true
	}

	var needed []config.Presubmit
	for _, job := range requiredPresubmits(presubmits, alwaysRunOverride) {
		if skippable[job.Context] {
			continue
		}
		needed = append(needed, job)
	}
	return needed
}

func main() {
	flag.Parse()
	logrus.SetFormatter(
		logrusutil.NewDefaultFieldsFormatter(nil, logrus.Fields{"component": "splice"}),
	)

	splicer, err := makeSplicer()
	if err != nil {
		logrus.WithError(err).Fatal("Could not make splicer.")
	}
	defer splicer.cleanup()

	configAgent := &config.Agent{}
	if err := configAgent.Start(*configPath, *jobConfigPath); err != nil {
		logrus.WithError(err).Fatal("Error starting config agent.")
	}

	kc, err := kube.NewClientInCluster(configAgent.Config().ProwJobNamespace)
	if err != nil {
		logrus.WithError(err).Fatal("Error getting kube client.")
	}

	// get overridden always_run jobs
	alwaysRunOverride := sets.NewString(strings.Split(*alwaysRun, ",")...)

	cooldown := 0
	// Loop endlessly, sleeping a minute between iterations
	for range time.Tick(1 * time.Minute) {
		start := time.Now()
		// List batch jobs, only start a new one if none are active.
		currentJobs, err := kc.ListProwJobs(kube.EmptySelector)
		if err != nil {
			logrus.WithError(err).Error("Error listing prow jobs.")
			continue
		}

		running := []string{}
		for _, job := range currentJobs {
			if job.Spec.Type != kube.BatchJob {
				continue
			}
			if !job.Complete() {
				running = append(running, job.Spec.Job)
			}
		}
		if len(running) > 0 {
			logrus.Infof("Waiting on %d jobs: %v", len(running), running)
			continue
		}

		// Start a new batch if the cooldown is 0, otherwise wait. This gives
		// the SQ some time to merge before we start a new batch.
		if cooldown > 0 {
			cooldown--
			continue
		}

		queue, err := getQueuedPRs(*submitQueueURL)
		if err != nil {
			logrus.WithError(err).Warning("Error getting queued PRs. Is the submit queue down?")
			continue
		}
		// No need to check for mergeable PRs if none is in the queue.
		if len(queue) == 0 {
			continue
		}
		logrus.Infof("PRs in queue: %v", queue)
		batchPRs, err := splicer.findMergeable(*remoteURL, queue)
		if err != nil {
			logrus.WithError(err).Error("Error computing mergeable PRs.")
			continue
		}
		// No need to start batches for single PRs
		if len(batchPRs) <= 1 {
			continue
		}
		// Trim down to the desired batch size.
		if len(batchPRs) > *maxBatchSize {
			batchPRs = batchPRs[:*maxBatchSize]
		}
		logrus.Infof("Starting a batch for the following PRs: %v", batchPRs)
		refs := splicer.makeBuildRefs(*orgName, *repoName, batchPRs)
		presubmits := configAgent.Config().Presubmits[fmt.Sprintf("%s/%s", *orgName, *repoName)]
		for _, job := range neededPresubmits(presubmits, currentJobs, refs, alwaysRunOverride) {
			if _, err := kc.CreateProwJob(pjutil.NewProwJob(pjutil.BatchSpec(job, refs), job.Labels)); err != nil {
				logrus.WithError(err).WithField("job", job.Name).Error("Error starting batch job.")
			}
		}
		cooldown = 5
		logrus.Infof("Sync time: %v", time.Since(start))
	}
}
<commit_msg>Prow: Use option struct instead of global flags in splice binary<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/sirupsen/logrus"

	"k8s.io/apimachinery/pkg/util/sets"

	"k8s.io/test-infra/prow/config"
	"k8s.io/test-infra/prow/kube"
	"k8s.io/test-infra/prow/logrusutil"
	"k8s.io/test-infra/prow/pjutil"
)

type options struct {
	submitQueueURL string
	remoteURL      string
	orgName        string
	repoName       string
	configPath     string
	jobConfigPath  string
	maxBatchSize   int
	alwaysRun      string
}

func gatherOptions() options {
	o := options{}
	flag.StringVar(&o.submitQueueURL, "submit-queue-endpoint", "http://submit-queue.k8s.io/github-e2e-queue", "Submit Queue status URL")
	flag.StringVar(&o.remoteURL, "remote-url", "https://github.com/kubernetes/kubernetes", "Remote Git URL")
	flag.StringVar(&o.orgName, "org", "kubernetes", "Org name")
	flag.StringVar(&o.repoName, "repo", "kubernetes", "Repo name")
	flag.StringVar(&o.configPath, "config-path", "/etc/config/config.yaml", "Path to config.yaml.")
	flag.StringVar(&o.jobConfigPath, "job-config-path", "", "Path to prow job configs.")
	flag.IntVar(&o.maxBatchSize, "batch-size", 5, "Maximum batch size")
	flag.StringVar(&o.alwaysRun, "always-run", "", "Job names that should be treated as always_run: true in Splice")
	flag.Parse()
	return o
}

func (o *options) Validate() error {
	if o.maxBatchSize < 1 {
		return errors.New("batch size cannot be less that one")
	}

	return nil
}

// Call a binary and return its output and success status.
func call(binary string, args ...string) (string, error) {
	cmdout := "+ " + binary + " "
	for _, arg := range args {
		cmdout += arg + " "
	}
	logrus.Info(cmdout)

	cmd := exec.Command(binary, args...)
	output, err := cmd.CombinedOutput()
	return string(output), err
}

// getQueuedPRs reads the list of queued PRs from the Submit Queue.
func getQueuedPRs(url string) ([]int, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	queue := struct {
		E2EQueue []struct {
			Number  int
			BaseRef string
		}
	}{}
	err = json.Unmarshal(body, &queue)
	if err != nil {
		return nil, err
	}

	ret := []int{}
	for _, e := range queue.E2EQueue {
		if e.BaseRef == "" || e.BaseRef == "master" {
			ret = append(ret, e.Number)
		}
	}
	return ret, nil
}

// Splicer manages a git repo in specific directory.
type splicer struct {
	dir string // The repository location.
}

// makeSplicer returns a splicer in a new temporary directory,
// with an initial .git dir created.
func makeSplicer() (*splicer, error) {
	dir, err := ioutil.TempDir("", "splice_")
	if err != nil {
		return nil, err
	}
	s := &splicer{dir}
	err = s.gitCalls([][]string{
		{"init"},
		{"config", "--local", "user.name", "K8S Prow Splice"},
		{"config", "--local", "user.email", "splice@localhost"},
		{"config", "--local", "commit.gpgsign", "false"},
	})
	if err != nil {
		s.cleanup()
		return nil, err
	}
	logrus.Infof("Splicer created in %s.", dir)
	return s, nil
}

// cleanup recurisvely deletes the repository
func (s *splicer) cleanup() {
	os.RemoveAll(s.dir)
}

// gitCall is a helper to call `git -C $path $args`.
func (s *splicer) gitCall(args ...string) error {
	fullArgs := append([]string{"-C", s.dir}, args...)
	output, err := call("git", fullArgs...)
	if len(output) > 0 {
		logrus.Info(output)
	}
	return err
}

// gitCalls is a helper to chain repeated gitCall invocations,
// returning the first failure, or nil if they all succeeded.
func (s *splicer) gitCalls(argsList [][]string) error {
	for _, args := range argsList {
		err := s.gitCall(args...)
		if err != nil {
			return err
		}
	}
	return nil
}

// findMergeable fetches given PRs from upstream, merges them locally,
// and finally returns a list of PRs that can be merged without conflicts.
func (s *splicer) findMergeable(remote string, prs []int) ([]int, error) {
	args := []string{"fetch", "-f", remote, "master:master"}
	for _, pr := range prs {
		args = append(args, fmt.Sprintf("pull/%d/head:pr/%d", pr, pr))
	}

	err := s.gitCalls([][]string{
		{"reset", "--hard"},
		{"checkout", "--orphan", "blank"},
		{"reset", "--hard"},
		{"clean", "-fdx"},
		args,
		{"checkout", "-B", "batch", "master"},
	})
	if err != nil {
		return nil, err
	}

	out := []int{}
	for _, pr := range prs {
		err := s.gitCall("merge", "--no-ff", "--no-stat",
			"-m", fmt.Sprintf("merge #%d", pr),
			fmt.Sprintf("pr/%d", pr))
		if err != nil {
			// merge conflict: cleanup and move on
			err = s.gitCall("merge", "--abort")
			if err != nil {
				return nil, err
			}
			continue
		}
		out = append(out, pr)
	}
	return out, nil
}

// gitRef returns the SHA for the given git object-- a branch, generally.
func (s *splicer) gitRef(ref string) string {
	output, err := call("git", "-C", s.dir, "rev-parse", ref)
	if err != nil {
		return ""
	}
	return strings.TrimSpace(output)
}

// Produce a kube.Refs for the given pull requests. This involves computing the
// git ref for master and the PRs.
func (s *splicer) makeBuildRefs(org, repo string, prs []int) kube.Refs {
	refs := kube.Refs{
		Org:     org,
		Repo:    repo,
		BaseRef: "master",
		BaseSHA: s.gitRef("master"),
	}
	for _, pr := range prs {
		branch := fmt.Sprintf("pr/%d", pr)
		refs.Pulls = append(refs.Pulls, kube.Pull{Number: pr, SHA: s.gitRef(branch)})
	}
	return refs
}

// Filters to the list of jobs which already passed this commit
func completedJobs(currentJobs []kube.ProwJob, refs kube.Refs) []kube.ProwJob {
	var skippable []kube.ProwJob
	rs := refs.String()

	for _, job := range currentJobs {
		if job.Spec.Type != kube.BatchJob {
			continue
		}
		if !job.Complete() {
			continue
		}
		if job.Status.State != kube.SuccessState {
			continue
		}
		if job.Spec.Refs.String() != rs {
			continue
		}
		skippable = append(skippable, job)
	}
	return skippable
}

// Filters to the list of required presubmits that report
func requiredPresubmits(presubmits []config.Presubmit, alwaysRunOverride sets.String) []config.Presubmit {
	var out []config.Presubmit
	for _, job := range presubmits {
		if !job.AlwaysRun && !alwaysRunOverride.Has(job.Name) { // Ignore manual jobs as these do not block
			continue
		}
		if job.SkipReport { // Ignore silent jobs as these do not block
			continue
		}
		if !job.RunsAgainstBranch("master") { // Ignore jobs that don't run on master
			continue
		}
		out = append(out, job)
	}
	return out
}

// Filters to the list of required presubmit which have not already passed this commit
func neededPresubmits(presubmits []config.Presubmit, currentJobs []kube.ProwJob, refs kube.Refs, alwaysRunOverride sets.String) []config.Presubmit {
	skippable := make(map[string]bool)
	for _, job := range completedJobs(currentJobs, refs) {
		skippable[job.Spec.Context] = true
	}

	var needed []config.Presubmit
	for _, job := range requiredPresubmits(presubmits, alwaysRunOverride) {
		if skippable[job.Context] {
			continue
		}
		needed = append(needed, job)
	}
	return needed
}

func main() {
	o := gatherOptions()
	if err := o.Validate(); err != nil {
		logrus.Fatalf("Invalid options: %v", err)
	}

	logrus.SetFormatter(
		logrusutil.NewDefaultFieldsFormatter(nil, logrus.Fields{"component": "splice"}),
	)

	splicer, err := makeSplicer()
	if err != nil {
		logrus.WithError(err).Fatal("Could not make splicer.")
	}
	defer splicer.cleanup()

	configAgent := &config.Agent{}
	if err := configAgent.Start(o.configPath, o.jobConfigPath); err != nil {
		logrus.WithError(err).Fatal("Error starting config agent.")
	}

	kc, err := kube.NewClientInCluster(configAgent.Config().ProwJobNamespace)
	if err != nil {
		logrus.WithError(err).Fatal("Error getting kube client.")
	}

	// get overridden always_run jobs
	alwaysRunOverride := sets.NewString(strings.Split(o.alwaysRun, ",")...)

	cooldown := 0
	// Loop endlessly, sleeping a minute between iterations
	for range time.Tick(1 * time.Minute) {
		start := time.Now()
		// List batch jobs, only start a new one if none are active.
		currentJobs, err := kc.ListProwJobs(kube.EmptySelector)
		if err != nil {
			logrus.WithError(err).Error("Error listing prow jobs.")
			continue
		}

		running := []string{}
		for _, job := range currentJobs {
			if job.Spec.Type != kube.BatchJob {
				continue
			}
			if !job.Complete() {
				running = append(running, job.Spec.Job)
			}
		}
		if len(running) > 0 {
			logrus.Infof("Waiting on %d jobs: %v", len(running), running)
			continue
		}

		// Start a new batch if the cooldown is 0, otherwise wait. This gives
		// the SQ some time to merge before we start a new batch.
		if cooldown > 0 {
			cooldown--
			continue
		}

		queue, err := getQueuedPRs(o.submitQueueURL)
		if err != nil {
			logrus.WithError(err).Warning("Error getting queued PRs. Is the submit queue down?")
			continue
		}
		// No need to check for mergeable PRs if none is in the queue.
		if len(queue) == 0 {
			continue
		}
		logrus.Infof("PRs in queue: %v", queue)
		batchPRs, err := splicer.findMergeable(o.remoteURL, queue)
		if err != nil {
			logrus.WithError(err).Error("Error computing mergeable PRs.")
			continue
		}
		// No need to start batches for single PRs
		if len(batchPRs) <= 1 {
			continue
		}
		// Trim down to the desired batch size.
		if len(batchPRs) > o.maxBatchSize {
			batchPRs = batchPRs[:o.maxBatchSize]
		}
		logrus.Infof("Starting a batch for the following PRs: %v", batchPRs)
		refs := splicer.makeBuildRefs(o.orgName, o.repoName, batchPRs)
		presubmits := configAgent.Config().Presubmits[fmt.Sprintf("%s/%s", o.orgName, o.repoName)]
		for _, job := range neededPresubmits(presubmits, currentJobs, refs, alwaysRunOverride) {
			if _, err := kc.CreateProwJob(pjutil.NewProwJob(pjutil.BatchSpec(job, refs), job.Labels)); err != nil {
				logrus.WithError(err).WithField("job", job.Name).Error("Error starting batch job.")
			}
		}
		cooldown = 5
		logrus.Infof("Sync time: %v", time.Since(start))
	}
}
<|endoftext|>


<commit_before>package proxyhandler

import (
	"io"
	"encoding/xml"
	"strings"
	"encoding/json"
	"log"
)

// A Decoder reads and decodes XML objects from an input stream.
type Decoder struct {
	reader io.Reader
	schema map[string]xsdElement
}

// Node is a data element on a tree
type Node struct {
	Children map[string]Nodes
	Data     string
	Complex  bool
	List     bool
}

// Nodes is a list of nodes
type Nodes []*Node

// addChild appends a node to the list of children
func (n *Node) addChild(s string, c *Node) {
	if n.Children == nil {
		n.Children = map[string]Nodes{}
	}
	n.Children[s] = append(n.Children[s], c)
}

type Element struct {
	parent   *Element
	node     *Node
	label    string
}

// Creates a New Decoder.
func NewDecoder(r io.Reader, schema map[string]xsdElement) *Decoder {
	return &Decoder{reader: r, schema: schema}
}

// Decodes XML
// returnElementName indicates the element to return, otherwise the root element is returned
func (dec *Decoder) Decode(root *Node, label string) (*Node, error) {

	xmlDec := xml.NewDecoder(dec.reader)

	element := &Element{parent:nil, node:root}

	var section *Node = nil

	for {
		t, err := xmlDec.Token()
		if t == nil || err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		}
		switch se := t.(type) {
		case xml.StartElement:
			schema := dec.schema[se.Name.Local]
			element = &Element{
				parent: element,
				node:   &Node{Complex:schema.isComplex(), List:schema.isList()},
				label:  se.Name.Local,
			}
		case xml.CharData:
			element.node.Data = strings.TrimSpace(string(xml.CharData(se)))
		case xml.EndElement:
			if element.parent != nil {
				element.parent.node.addChild(element.label, element.node)
			}
			if element.label == label {
				section = element.node
			}
			element = element.parent
		}
	}

	log.Printf("Return element %v\n", root)

	return section, nil
}

// encodes to json
func (n *Node) encode() ([]byte, error) {
	object := n.serialize()
	bytes, err := json.Marshal(object)
	if err != nil {
		return nil, err
	}
	return bytes, nil
}

// serializes a node to plain anonymous objects
func (n *Node) serialize() (interface{}) {
	if n.Complex {
		object := make(map[string]interface{})
		for label, children := range n.Children {
			if len(children) > 1 || (len(children) == 1 && children[0].List) {
				s := make([]interface{}, 0)
				for _, c := range children {
					s = append(s, c.serialize())
				}
				object[label] = s
			} else {
				object[label] = children[0].serialize()
			}
		}
		return object
	} else {
		return n.Data
	}
}
<commit_msg>docs updated<commit_after>package proxyhandler

import (
	"io"
	"encoding/xml"
	"strings"
	"encoding/json"
	"log"
)

// A Decoder reads and decodes XML objects from an input stream.
type Decoder struct {
	reader io.Reader
	schema map[string]xsdElement
}

// Node is a data element on a tree
type Node struct {
	Children map[string]Nodes
	Data     string
	Complex  bool
	List     bool
}

// Nodes is a list of nodes
type Nodes []*Node

// addChild appends a node to the list of children
func (n *Node) addChild(s string, c *Node) {
	if n.Children == nil {
		n.Children = map[string]Nodes{}
	}
	n.Children[s] = append(n.Children[s], c)
}

type Element struct {
	parent   *Element
	node     *Node
	label    string
}

// Creates a New Decoder.
func NewDecoder(r io.Reader, schema map[string]xsdElement) *Decoder {
	return &Decoder{reader: r, schema: schema}
}

// Decodes XML
// label indicates the element to return, otherwise the root element is returned
func (dec *Decoder) Decode(root *Node, label string) (*Node, error) {

	xmlDec := xml.NewDecoder(dec.reader)

	element := &Element{parent:nil, node:root}

	var section *Node = root

	for {
		t, err := xmlDec.Token()
		if t == nil || err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		}
		switch se := t.(type) {
		case xml.StartElement:
			schema := dec.schema[se.Name.Local]
			element = &Element{
				parent: element,
				node:   &Node{Complex:schema.isComplex(), List:schema.isList()},
				label:  se.Name.Local,
			}
		case xml.CharData:
			element.node.Data = strings.TrimSpace(string(xml.CharData(se)))
		case xml.EndElement:
			if element.parent != nil {
				element.parent.node.addChild(element.label, element.node)
			}
			if element.label == label {
				section = element.node
			}
			element = element.parent
		}
	}

	log.Printf("Return element %v\n", root)

	return section, nil
}

// encodes to json
func (n *Node) encode() ([]byte, error) {
	object := n.serialize()
	bytes, err := json.Marshal(object)
	if err != nil {
		return nil, err
	}
	return bytes, nil
}

// serializes a node to plain anonymous objects
func (n *Node) serialize() (interface{}) {
	if n.Complex {
		object := make(map[string]interface{})
		for label, children := range n.Children {
			if len(children) > 1 || (len(children) == 1 && children[0].List) {
				s := make([]interface{}, 0)
				for _, c := range children {
					s = append(s, c.serialize())
				}
				object[label] = s
			} else {
				object[label] = children[0].serialize()
			}
		}
		return object
	} else {
		return n.Data
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/url"
	"runtime/debug"
	"strings"
	"time"

	"code.google.com/p/go-uuid/uuid"
)

var (
	listen           = flag.String("l", ":8888", "port to accept requests")
	targetProduction = flag.String("a", "http://localhost:8080", "where production traffic goes. http://localhost:8080/production")
	altTarget        = flag.String("b", "http://localhost:8081", "where testing traffic goes. response are skipped. http://localhost:8081/test")
	retryCount       = flag.Int("rc", 3, "how many times to retry on alternative destination server errors")
	retryTimeoutMs   = flag.Int("rt", 250, "timeout in milliseconds between retries on alternative destination server errors")

	// Hop-by-hop headers. These are removed when sent to the backend.
	// http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html
	hopHeaders = []string{
		"Connection",
		"Keep-Alive",
		"Proxy-Authenticate",
		"Proxy-Authorization",
		"Te", // canonicalized version of "TE"
		"Trailers",
		"Transfer-Encoding",
		"Upgrade",
	}
)

type Hosts struct {
	Target      url.URL
	Alternative url.URL
}

var hosts Hosts
var proxy *httputil.ReverseProxy

type TimeoutTransport struct {
	http.Transport
}

func (t *TimeoutTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	return t.Transport.RoundTrip(req)
}

func clientCall(id string, req *http.Request) {
	defer func() {
		if r := recover(); r != nil {
			logMessage(id, "ERROR", fmt.Sprintf("Recovered in clientCall: <%v> <%s>", r, removeEndsOfLines(string(debug.Stack()))))
		}
	}()

	// once request is send, the body is read and is empty for second try, need to recreate body reader each time request is made
	req2, bodyBytes := duplicateRequest(req)

	for retry := 0; retry < *retryCount; retry++ {
		req2.Body = ioutil.NopCloser(bytes.NewReader(bodyBytes))

		resp, err := http.DefaultTransport.RoundTrip(req2)
		if err != nil {
			logMessage(id, "ERROR", fmt.Sprintf("Invoking client failed: <%v>. Request: <%s>.", err, prettyPrint(req2)))
			return
		}

		r, e := httputil.DumpResponse(resp, true)
		if e != nil {
			logMessage(id, "ERROR", fmt.Sprintf("Could not create response dump: <%v>", e))
		} else {
			logMessage(id, "INFO", fmt.Sprintf("Response: <%s>", removeEndsOfLines(string(r))))
		}

		io.Copy(ioutil.Discard, resp.Body)
		resp.Body.Close()

		if resp.StatusCode < 500 || resp.StatusCode >= 600 {
			return
		}

		if retry+1 != *retryCount {
			logMessage(id, "WARN", fmt.Sprintf("Received 5xx response. Retrying request %v/%v", retry+2, *retryCount))
			time.Sleep(time.Duration(*retryTimeoutMs) * time.Millisecond)
		}
	}

	logMessage(id, "ERROR", "Request failed")
}

func teeDirector(req *http.Request) {
	id := uuid.NewUUID().String()

	r, e := httputil.DumpRequest(req, true)
	if e != nil {
		logMessage(id, "ERROR", fmt.Sprintf("Could not create request dump: <%v>", e))
		r = []byte{}
	}

	logMessage(id, "INFO", fmt.Sprintf("Request: <%s>", removeEndsOfLines(string(r))))

	go clientCall(id, req)

	targetQuery := hosts.Target.RawQuery
	req.URL.Scheme = hosts.Target.Scheme
	req.URL.Host = hosts.Target.Host
	req.URL.Path = singleJoiningSlash(hosts.Target.Path, req.URL.Path)
	if targetQuery == "" || req.URL.RawQuery == "" {
		req.URL.RawQuery = targetQuery + req.URL.RawQuery
	} else {
		req.URL.RawQuery = targetQuery + "&" + req.URL.RawQuery
	}
}

// return copied request with empty body and request body bytes, this is because each time request is sent body is read and emptied
// we want to send same request multiple times, so returning body bytes to use for setting up body reader on each new request
func duplicateRequest(request *http.Request) (*http.Request, []byte) {
	b1 := new(bytes.Buffer)
	b2 := new(bytes.Buffer)
	w := io.MultiWriter(b1, b2)
	io.Copy(w, request.Body)
	request.Body = ioutil.NopCloser(bytes.NewReader(b2.Bytes()))

	request2 := &http.Request{
		Method: request.Method,
		URL: &url.URL{
			Scheme:   hosts.Alternative.Scheme,
			Host:     hosts.Alternative.Host,
			Path:     singleJoiningSlash(hosts.Alternative.Path, request.URL.Path),
			RawQuery: request.URL.RawQuery,
		},
		Proto:         request.Proto,
		ProtoMajor:    request.ProtoMajor,
		ProtoMinor:    request.ProtoMinor,
		Header:        request.Header,
		ContentLength: request.ContentLength,
		Close:         false,
	}

	// Remove hop-by-hop headers to the backend.  Especially
	// important is "Connection" because we want a persistent
	// connection, regardless of what the client sent to us.  This
	// is modifying the same underlying map from req (shallow
	// copied above) so we only copy it if necessary.
	copiedHeaders := false
	for _, h := range hopHeaders {
		if request2.Header.Get(h) != "" {
			if !copiedHeaders {
				request2.Header = make(http.Header)
				copyHeader(request2.Header, request.Header)
				copiedHeaders = true
			}
			request2.Header.Del(h)
		}
	}

	return request2, b1.Bytes()
}

func copyHeader(dst, src http.Header) {
	for k, vv := range src {
		for _, v := range vv {
			dst.Add(k, v)
		}
	}
}

func handler(w http.ResponseWriter, r *http.Request) {
	proxy.ServeHTTP(w, r)
}

// want to keep log messages on a single line, one line is one log entry
func removeEndsOfLines(s string) string {
	return strings.Replace(strings.Replace(s, "\n", "\\n", -1), "\r", "\\r", -1)
}

func prettyPrint(obj interface{}) string {
	return removeEndsOfLines(fmt.Sprintf("%+v", obj))
}

func logMessage(id, messageType, message string) {
	fmt.Printf("[%s][%s][%s][%s]\n", time.Now().Format(time.RFC3339Nano), id, messageType, message)
}

func singleJoiningSlash(a, b string) string {
	aslash := strings.HasSuffix(a, "/")
	bslash := strings.HasPrefix(b, "/")
	switch {
	case aslash && bslash:
		return a + b[1:]
	case !aslash && !bslash:
		return a + "/" + b
	}
	return a + b
}

func main() {
	flag.Parse()

	target, _ := url.Parse(*targetProduction)
	alt, _ := url.Parse(*altTarget)

	hosts = Hosts{
		Target:      *target,
		Alternative: *alt,
	}

	u, _ := url.Parse(*targetProduction)
	proxy = httputil.NewSingleHostReverseProxy(u)
	proxy.Transport = &TimeoutTransport{}
	proxy.Director = teeDirector

	http.HandleFunc("/", handler)
	http.ListenAndServe(*listen, nil)
}
<commit_msg>Don't retry 500 as that means request reached the service<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/url"
	"runtime/debug"
	"strings"
	"time"

	"code.google.com/p/go-uuid/uuid"
)

var (
	listen           = flag.String("l", ":8888", "port to accept requests")
	targetProduction = flag.String("a", "http://localhost:8080", "where production traffic goes. http://localhost:8080/production")
	altTarget        = flag.String("b", "http://localhost:8081", "where testing traffic goes. response are skipped. http://localhost:8081/test")
	retryCount       = flag.Int("rc", 3, "how many times to retry on alternative destination server errors")
	retryTimeoutMs   = flag.Int("rt", 250, "timeout in milliseconds between retries on alternative destination server errors")

	// Hop-by-hop headers. These are removed when sent to the backend.
	// http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html
	hopHeaders = []string{
		"Connection",
		"Keep-Alive",
		"Proxy-Authenticate",
		"Proxy-Authorization",
		"Te", // canonicalized version of "TE"
		"Trailers",
		"Transfer-Encoding",
		"Upgrade",
	}
)

type Hosts struct {
	Target      url.URL
	Alternative url.URL
}

var hosts Hosts
var proxy *httputil.ReverseProxy

type TimeoutTransport struct {
	http.Transport
}

func (t *TimeoutTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	return t.Transport.RoundTrip(req)
}

func clientCall(id string, req *http.Request) {
	defer func() {
		if r := recover(); r != nil {
			logMessage(id, "ERROR", fmt.Sprintf("Recovered in clientCall: <%v> <%s>", r, removeEndsOfLines(string(debug.Stack()))))
		}
	}()

	// once request is send, the body is read and is empty for second try, need to recreate body reader each time request is made
	req2, bodyBytes := duplicateRequest(req)

	for retry := 0; retry < *retryCount; retry++ {
		req2.Body = ioutil.NopCloser(bytes.NewReader(bodyBytes))

		resp, err := http.DefaultTransport.RoundTrip(req2)
		if err != nil {
			logMessage(id, "ERROR", fmt.Sprintf("Invoking client failed: <%v>. Request: <%s>.", err, prettyPrint(req2)))
			return
		}

		r, e := httputil.DumpResponse(resp, true)
		if e != nil {
			logMessage(id, "ERROR", fmt.Sprintf("Could not create response dump: <%v>", e))
		} else {
			logMessage(id, "INFO", fmt.Sprintf("Response: <%s>", removeEndsOfLines(string(r))))
		}

		io.Copy(ioutil.Discard, resp.Body)
		resp.Body.Close()

		// Want to retry server errors like gateway time-out, bad gateway, service unavailable etc.
		// We specifically don't want to retry 500 as that means request reached the server
		if resp.StatusCode < 501 || resp.StatusCode >= 600 {
			return
		}

		if retry+1 != *retryCount {
			logMessage(id, "WARN", fmt.Sprintf("Received 5xx response. Retrying request %v/%v", retry+2, *retryCount))
			time.Sleep(time.Duration(*retryTimeoutMs) * time.Millisecond)
		}
	}

	logMessage(id, "ERROR", "Request failed")
}

func teeDirector(req *http.Request) {
	id := uuid.NewUUID().String()

	r, e := httputil.DumpRequest(req, true)
	if e != nil {
		logMessage(id, "ERROR", fmt.Sprintf("Could not create request dump: <%v>", e))
		r = []byte{}
	}

	logMessage(id, "INFO", fmt.Sprintf("Request: <%s>", removeEndsOfLines(string(r))))

	go clientCall(id, req)

	targetQuery := hosts.Target.RawQuery
	req.URL.Scheme = hosts.Target.Scheme
	req.URL.Host = hosts.Target.Host
	req.URL.Path = singleJoiningSlash(hosts.Target.Path, req.URL.Path)
	if targetQuery == "" || req.URL.RawQuery == "" {
		req.URL.RawQuery = targetQuery + req.URL.RawQuery
	} else {
		req.URL.RawQuery = targetQuery + "&" + req.URL.RawQuery
	}
}

// return copied request with empty body and request body bytes, this is because each time request is sent body is read and emptied
// we want to send same request multiple times, so returning body bytes to use for setting up body reader on each new request
func duplicateRequest(request *http.Request) (*http.Request, []byte) {
	b1 := new(bytes.Buffer)
	b2 := new(bytes.Buffer)
	w := io.MultiWriter(b1, b2)
	io.Copy(w, request.Body)
	request.Body = ioutil.NopCloser(bytes.NewReader(b2.Bytes()))

	request2 := &http.Request{
		Method: request.Method,
		URL: &url.URL{
			Scheme:   hosts.Alternative.Scheme,
			Host:     hosts.Alternative.Host,
			Path:     singleJoiningSlash(hosts.Alternative.Path, request.URL.Path),
			RawQuery: request.URL.RawQuery,
		},
		Proto:         request.Proto,
		ProtoMajor:    request.ProtoMajor,
		ProtoMinor:    request.ProtoMinor,
		Header:        request.Header,
		ContentLength: request.ContentLength,
		Close:         false,
	}

	// Remove hop-by-hop headers to the backend.  Especially
	// important is "Connection" because we want a persistent
	// connection, regardless of what the client sent to us.  This
	// is modifying the same underlying map from req (shallow
	// copied above) so we only copy it if necessary.
	copiedHeaders := false
	for _, h := range hopHeaders {
		if request2.Header.Get(h) != "" {
			if !copiedHeaders {
				request2.Header = make(http.Header)
				copyHeader(request2.Header, request.Header)
				copiedHeaders = true
			}
			request2.Header.Del(h)
		}
	}

	return request2, b1.Bytes()
}

func copyHeader(dst, src http.Header) {
	for k, vv := range src {
		for _, v := range vv {
			dst.Add(k, v)
		}
	}
}

func handler(w http.ResponseWriter, r *http.Request) {
	proxy.ServeHTTP(w, r)
}

// want to keep log messages on a single line, one line is one log entry
func removeEndsOfLines(s string) string {
	return strings.Replace(strings.Replace(s, "\n", "\\n", -1), "\r", "\\r", -1)
}

func prettyPrint(obj interface{}) string {
	return removeEndsOfLines(fmt.Sprintf("%+v", obj))
}

func logMessage(id, messageType, message string) {
	fmt.Printf("[%s][%s][%s][%s]\n", time.Now().Format(time.RFC3339Nano), id, messageType, message)
}

func singleJoiningSlash(a, b string) string {
	aslash := strings.HasSuffix(a, "/")
	bslash := strings.HasPrefix(b, "/")
	switch {
	case aslash && bslash:
		return a + b[1:]
	case !aslash && !bslash:
		return a + "/" + b
	}
	return a + b
}

func main() {
	flag.Parse()

	target, _ := url.Parse(*targetProduction)
	alt, _ := url.Parse(*altTarget)

	hosts = Hosts{
		Target:      *target,
		Alternative: *alt,
	}

	u, _ := url.Parse(*targetProduction)
	proxy = httputil.NewSingleHostReverseProxy(u)
	proxy.Transport = &TimeoutTransport{}
	proxy.Director = teeDirector

	http.HandleFunc("/", handler)
	http.ListenAndServe(*listen, nil)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httputil"
	"runtime"
	"time"
)

// Console flags
var (
	listen                = flag.String("l", ":8888", "port to accept requests")
	targetProduction      = flag.String("a", "localhost:8080", "where production traffic goes. http://localhost:8080/production")
	altTarget             = flag.String("b", "localhost:8081", "where testing traffic goes. response are skipped. http://localhost:8081/test")
	debug                 = flag.Bool("debug", false, "more logging, showing ignored output")
	productionTimeout     = flag.Int("a.timeout", 3, "timeout in seconds for production traffic")
	alternateTimeout      = flag.Int("b.timeout", 1, "timeout in seconds for alternate site traffic")
	productionHostRewrite = flag.Bool("a.rewrite", false, "rewrite the host header when proxying production traffic")
	alternateHostRewrite  = flag.Bool("b.rewrite", false, "rewrite the host header when proxying alternate site traffic")
)

// handler contains the address of the main Target and the one for the Alternative target
type handler struct {
	Target      string
	Alternative string
}

// ServeHTTP duplicates the incoming request (req) and does the request to the Target and the Alternate target discading the Alternate response
func (h handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	req1, req2 := DuplicateRequest(req)
	go func() {
		defer func() {
			if r := recover(); r != nil && *debug {
				fmt.Println("Recovered in f", r)
			}
		}()
		// Open new TCP connection to the server
		clientTcpConn, err := net.DialTimeout("tcp", h.Alternative, time.Duration(time.Duration(*alternateTimeout)*time.Second))
		if err != nil {
			if *debug {
				fmt.Printf("Failed to connect to %s\n", h.Alternative)
			}
			return
		}
		clientHttpConn := httputil.NewClientConn(clientTcpConn, nil) // Start a new HTTP connection on it
		defer clientHttpConn.Close()                                 // Close the connection to the server
		if *alternateHostRewrite {
			req1.Host = h.Alternative
		}
		err = clientHttpConn.Write(req1) // Pass on the request
		if err != nil {
			if *debug {
				fmt.Printf("Failed to send to %s: %v\n", h.Alternative, err)
			}
			return
		}
		_, err = clientHttpConn.Read(req1) // Read back the reply
		if err != nil {
			if *debug {
				fmt.Printf("Failed to receive from %s: %v\n", h.Alternative, err)
			}
			return
		}
	}()
	defer func() {
		if r := recover(); r != nil && *debug {
			fmt.Println("Recovered in f", r)
		}
	}()

	// Open new TCP connection to the server
	clientTcpConn, err := net.DialTimeout("tcp", h.Target, time.Duration(time.Duration(*productionTimeout)*time.Second))
	if err != nil {
		fmt.Printf("Failed to connect to %s\n", h.Target)
		return
	}
	clientHttpConn := httputil.NewClientConn(clientTcpConn, nil) // Start a new HTTP connection on it
	defer clientHttpConn.Close()                                 // Close the connection to the server
	if *productionHostRewrite {
		req2.Host = h.Target
	}
	err = clientHttpConn.Write(req2) // Pass on the request
	if err != nil {
		fmt.Printf("Failed to send to %s: %v\n", h.Target, err)
		return
	}
	resp, err := clientHttpConn.Read(req2) // Read back the reply
	if err != nil {
		fmt.Printf("Failed to receive from %s: %v\n", h.Target, err)
		return
	}
	for k, v := range resp.Header {
		w.Header()[k] = v
	}
	w.WriteHeader(resp.StatusCode)
	body, _ := ioutil.ReadAll(resp.Body)
	w.Write(body)
}

func main() {
	flag.Parse()

	runtime.GOMAXPROCS(runtime.NumCPU())

	local, err := net.Listen("tcp", *listen)
	if err != nil {
		fmt.Printf("Failed to listen to %s\n", *listen)
		return
	}
	h := handler{
		Target:      *targetProduction,
		Alternative: *altTarget,
	}
	http.Serve(local, h)
}

type nopCloser struct {
	io.Reader
}

func (nopCloser) Close() error { return nil }

func DuplicateRequest(request *http.Request) (request1 *http.Request, request2 *http.Request) {
	b1 := new(bytes.Buffer)
	b2 := new(bytes.Buffer)
	w := io.MultiWriter(b1, b2)
	io.Copy(w, request.Body)
	defer request.Body.Close()
	request1 = &http.Request{
		Method:        request.Method,
		URL:           request.URL,
		Proto:         "HTTP/1.1",
		ProtoMajor:    1,
		ProtoMinor:    1,
		Header:        request.Header,
		Body:          nopCloser{b1},
		Host:          request.Host,
		ContentLength: request.ContentLength,
	}
	request2 = &http.Request{
		Method:        request.Method,
		URL:           request.URL,
		Proto:         "HTTP/1.1",
		ProtoMajor:    1,
		ProtoMinor:    1,
		Header:        request.Header,
		Body:          nopCloser{b2},
		Host:          request.Host,
		ContentLength: request.ContentLength,
	}
	return
}
<commit_msg>forward a percentage of requests to testing site<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net"
	"net/http"
	"net/http/httputil"
	"runtime"
	"time"
)

// Console flags
var (
	listen                = flag.String("l", ":8888", "port to accept requests")
	targetProduction      = flag.String("a", "localhost:8080", "where production traffic goes. http://localhost:8080/production")
	altTarget             = flag.String("b", "localhost:8081", "where testing traffic goes. response are skipped. http://localhost:8081/test")
	debug                 = flag.Bool("debug", false, "more logging, showing ignored output")
	productionTimeout     = flag.Int("a.timeout", 3, "timeout in seconds for production traffic")
	alternateTimeout      = flag.Int("b.timeout", 1, "timeout in seconds for alternate site traffic")
	productionHostRewrite = flag.Bool("a.rewrite", false, "rewrite the host header when proxying production traffic")
	alternateHostRewrite  = flag.Bool("b.rewrite", false, "rewrite the host header when proxying alternate site traffic")
	percent               = flag.Float64("p", 100.0, "float64 percentage of traffic to send to testing")
)

// handler contains the address of the main Target and the one for the Alternative target
type handler struct {
	Target      string
	Alternative string
	Randomizer  rand.Rand
}

// ServeHTTP duplicates the incoming request (req) and does the request to the Target and the Alternate target discading the Alternate response
func (h handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	var req1, req2 *http.Request
	if *percent == 100.0 || h.Randomizer.Float64()*100 < *percent {
		req1, req2 = DuplicateRequest(req)
		go func() {
			defer func() {
				if r := recover(); r != nil && *debug {
					fmt.Println("Recovered in f", r)
				}
			}()
			// Open new TCP connection to the server
			clientTcpConn, err := net.DialTimeout("tcp", h.Alternative, time.Duration(time.Duration(*alternateTimeout)*time.Second))
			if err != nil {
				if *debug {
					fmt.Printf("Failed to connect to %s\n", h.Alternative)
				}
				return
			}
			clientHttpConn := httputil.NewClientConn(clientTcpConn, nil) // Start a new HTTP connection on it
			defer clientHttpConn.Close()                                 // Close the connection to the server
			if *alternateHostRewrite {
				req1.Host = h.Alternative
			}
			err = clientHttpConn.Write(req1) // Pass on the request
			if err != nil {
				if *debug {
					fmt.Printf("Failed to send to %s: %v\n", h.Alternative, err)
				}
				return
			}
			_, err = clientHttpConn.Read(req1) // Read back the reply
			if err != nil {
				if *debug {
					fmt.Printf("Failed to receive from %s: %v\n", h.Alternative, err)
				}
				return
			}
		}()
	} else {
		req2 = req
	}
	defer func() {
		if r := recover(); r != nil && *debug {
			fmt.Println("Recovered in f", r)
		}
	}()

	// Open new TCP connection to the server
	clientTcpConn, err := net.DialTimeout("tcp", h.Target, time.Duration(time.Duration(*productionTimeout)*time.Second))
	if err != nil {
		fmt.Printf("Failed to connect to %s\n", h.Target)
		return
	}
	clientHttpConn := httputil.NewClientConn(clientTcpConn, nil) // Start a new HTTP connection on it
	defer clientHttpConn.Close()                                 // Close the connection to the server
	if *productionHostRewrite {
		req2.Host = h.Target
	}
	err = clientHttpConn.Write(req2) // Pass on the request
	if err != nil {
		fmt.Printf("Failed to send to %s: %v\n", h.Target, err)
		return
	}
	resp, err := clientHttpConn.Read(req2) // Read back the reply
	if err != nil {
		fmt.Printf("Failed to receive from %s: %v\n", h.Target, err)
		return
	}
	for k, v := range resp.Header {
		w.Header()[k] = v
	}
	w.WriteHeader(resp.StatusCode)
	body, _ := ioutil.ReadAll(resp.Body)
	w.Write(body)
}

func main() {
	flag.Parse()

	runtime.GOMAXPROCS(runtime.NumCPU())

	local, err := net.Listen("tcp", *listen)
	if err != nil {
		fmt.Printf("Failed to listen to %s\n", *listen)
		return
	}
	h := handler{
		Target:      *targetProduction,
		Alternative: *altTarget,
		Randomizer:  *rand.New(rand.NewSource(time.Now().UnixNano())),
	}
	http.Serve(local, h)
}

type nopCloser struct {
	io.Reader
}

func (nopCloser) Close() error { return nil }

func DuplicateRequest(request *http.Request) (request1 *http.Request, request2 *http.Request) {
	b1 := new(bytes.Buffer)
	b2 := new(bytes.Buffer)
	w := io.MultiWriter(b1, b2)
	io.Copy(w, request.Body)
	defer request.Body.Close()
	request1 = &http.Request{
		Method:        request.Method,
		URL:           request.URL,
		Proto:         "HTTP/1.1",
		ProtoMajor:    1,
		ProtoMinor:    1,
		Header:        request.Header,
		Body:          nopCloser{b1},
		Host:          request.Host,
		ContentLength: request.ContentLength,
	}
	request2 = &http.Request{
		Method:        request.Method,
		URL:           request.URL,
		Proto:         "HTTP/1.1",
		ProtoMajor:    1,
		ProtoMinor:    1,
		Header:        request.Header,
		Body:          nopCloser{b2},
		Host:          request.Host,
		ContentLength: request.ContentLength,
	}
	return
}
<|endoftext|>


<commit_before>package bootstrap

import (
	_ "github.com/go-sql-driver/mysql"
	_ "github.com/lib/pq"
	_ "github.com/viant/asc"
	_ "github.com/viant/bgc"
	_ "github.com/viant/endly/static" //load external resource like .csv .json files to mem storage
	_ "github.com/viant/toolbox/storage/aws"
	_ "github.com/viant/toolbox/storage/gs"

	"encoding/json"
	"flag"
	"fmt"
	"github.com/viant/endly"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/url"
	"gopkg.in/yaml.v2"
	"log"
	"os"
	"path"
	"strings"
	"time"
)

func init() {

	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
	flag.String("r", "run.json", "<path/url to workflow run request in JSON format> ")
	flag.String("w", "manager", "<workflow name>  if both -r and -w valid options are specified, -w is ignored")
	flag.String("t", "*", "<task/s to run>, t='?' to list all task for current workflow")
	flag.String("l", "logs", "<log directory>")
	flag.Bool("d", false, "enable logging")
	flag.Bool("p", false, "print neatly workflow as JSON")
	flag.String("f", "json", "<workflow or request format>, json or yaml")

	flag.Bool("h", false, "print help")
	flag.Bool("v", false, "print version")

	flag.String("s", "", "<serviceID> print service details, -s='*' prints all service IDs")
	flag.String("a", "", "<action> prints action request representation")

}

func Bootstrap() {
	flag.Usage = printHelp
	flag.Parse()
	flagset := make(map[string]string)
	flag.Visit(func(f *flag.Flag) {
		flagset[f.Name] = f.Value.String()
	})

	_, shouldQuit := flagset["v"]
	flagset["v"] = flag.Lookup("v").Value.String()

	if toolbox.AsBoolean(flagset["v"]) {
		printVersion()
		if shouldQuit {
			return
		}
	}

	if _, ok := flagset["h"]; ok {
		printHelp()
		return
	}

	if _, ok := flagset["a"]; ok {
		printServiceActionRequest()
		return
	}
	if _, ok := flagset["s"]; ok {
		printServiceActions()
		return
	}

	request, option, err := getRunRequestWithOptons(flagset)
	if request == nil {
		flagset["r"] = flag.Lookup("r").Value.String()
		flagset["w"] = flag.Lookup("w").Value.String()
		request, option, err = getRunRequestWithOptons(flagset)
		if err != nil && strings.Contains(err.Error(), "failed to locate workflow: manager") {
			printHelp()
			return
		}
	}

	if err != nil {
		log.Fatal(err)
	}
	if value, ok := flagset["p"]; ok && toolbox.AsBoolean(value) {
		printWorkflow(request.WorkflowURL)
		return
	}

	if flagset["t"] == "?" {
		printWorkflowTasks(request.WorkflowURL)
		return
	}

	runner := endly.NewCliRunner()
	err = runner.Run(request, option)
	if err != nil {
		log.Fatal(err)
	}
	time.Sleep(time.Second)
}

func printWorkflowTasks(URL string) {
	workflow, err := getWorkflow(URL);
	if err != nil {
		log.Fatal(err)
	}
	fmt.Fprintf(os.Stderr,"Workflow '%v' (%v) tasks:\n", workflow.Name, workflow.Source.URL)
	for _, task := range workflow.Tasks {
		fmt.Fprintf(os.Stderr, "\t%v: %v\n", task.Name, task.Description)
	}
}

func printServiceActionRequest() {
	manager := endly.NewManager()
	context := manager.NewContext(toolbox.NewContext())

	var serviceID = flag.Lookup("s").Value.String()
	service, err := context.Service(serviceID)
	if err != nil {
		log.Fatal(err)
	}
	var action = flag.Lookup("a").Value.String()
	request, err := service.NewRequest(action)
	if err != nil {
		log.Fatal(err)
	}
	toolbox.InitStruct(request)
	fmt.Printf("Request: %T\n", request)
	printInFormat(request, fmt.Sprintf("failed to print %v.%v request (%T)", serviceID, action, request)+", %v")
	response, _ := service.NewResponse(action)
	fmt.Printf("\nResponse: %T\n", response)
	toolbox.InitStruct(response)
	printInFormat(response, fmt.Sprintf("failed to print %v.%v response (%T)", serviceID, action, request)+", %v")

}

func printServiceActions() {
	manager := endly.NewManager()
	context := manager.NewContext(toolbox.NewContext())

	var serviceID = flag.Lookup("s").Value.String()

	if serviceID == "*" {
		fmt.Printf("endly services:\n")
		for k, v := range endly.Services(manager) {
			fmt.Printf("%v %T\n", k, v)
		}
		return;
	}

	service, err := context.Service(serviceID)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("'%v' service actions: \n", serviceID)
	for _, action := range service.Actions() {
		fmt.Printf("\t%v\n", action)
	}
}

func getWorkflow(URL string) (*endly.Workflow, error) {
	dao := endly.NewWorkflowDao()
	manager := endly.NewManager()
	context := manager.NewContext(toolbox.NewContext())
	return dao.Load(context, url.NewResource(URL))
}

func printWorkflow(URL string) {
	workflow, err := getWorkflow(URL);
	if err != nil {
		log.Fatal(err)
	}
	printInFormat(workflow, "failed to print workflow: "+URL+", %v")

}

func printInFormat(source interface{}, errorTemplate string) {
	format := flag.Lookup("f").Value.String()
	var buf []byte
	var err error
	switch format {
	case "yaml":
		buf, err = yaml.Marshal(source)
	default:
		buf, err = json.MarshalIndent(source, "", "\t")
	}
	if err != nil {
		log.Fatalf(errorTemplate, err)
	}
	fmt.Printf("%s\n", buf)
}

func printHelp() {
	_, name := path.Split(os.Args[0])
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", name)
	fmt.Fprintf(os.Stderr, "endly [options] [params...]\n")
	fmt.Fprintf(os.Stderr, "\tparams should be key value pair to be supplied as actual workflow parameters\n")
	fmt.Fprintf(os.Stderr, "\tif -r options is used, original request params may be overriden\n\n")

	fmt.Fprintf(os.Stderr, "where options include:\n")
	flag.PrintDefaults()
}

func printVersion() {
	fmt.Fprintf(os.Stdout, "%v %v\n", endly.AppName, endly.GetVersion())
}

func getWorkflowURL(candidate string) (string, string, error) {
	var _, name = path.Split(candidate)
	if path.Ext(candidate) == "" {
		candidate = candidate + ".csv"
	} else {
		name = string(name[:len(name)-4]) //remove extension
	}
	resource := url.NewResource(candidate)
	if _, err := resource.Download(); err != nil {
		resource = url.NewResource(fmt.Sprintf("mem://%v/workflow/%v", endly.EndlyNamespace, candidate))
		if _, memError := resource.Download(); memError != nil {
			return "", "", err
		}
	}
	return name, resource.URL, nil
}

func getRunRequestURL(candidate string) (*url.Resource, error) {
	if path.Ext(candidate) == "" {
		candidate = candidate + ".json"
	}
	resource := url.NewResource(candidate)
	if _, err := resource.Download(); err != nil {
		resource = url.NewResource(fmt.Sprintf("mem://%v/req/%v", endly.EndlyNamespace, candidate))
		if _, memError := resource.Download(); memError != nil {
			return nil, err
		}
	}
	return resource, nil

}

func getRunRequestWithOptons(flagset map[string]string) (*endly.WorkflowRunRequest, *endly.RunnerReportingOptions, error) {
	var request *endly.WorkflowRunRequest
	var options = &endly.RunnerReportingOptions{}
	if value, ok := flagset["w"]; ok {
		name, URL, err := getWorkflowURL(value)
		if err != nil {
			return nil, nil, fmt.Errorf("failed to locate workflow: %v %v", value, err)
		}
		request = &endly.WorkflowRunRequest{
			WorkflowURL: URL,
			Name:        name,
		}
		options = endly.DefaultRunnerReportingOption()
	}
	if value, ok := flagset["r"]; ok {
		resource, err := getRunRequestURL(value)
		if err == nil {
			request = &endly.WorkflowRunRequest{}
			err = resource.JSONDecode(request)
		}
		if request.WorkflowURL == "" {
			parent, _ := toolbox.URLSplit(resource.URL)
			parent = strings.Replace(parent, "req", "workflow", 1)
			request.WorkflowURL = toolbox.URLPathJoin(parent, request.Name+".csv")
		}
		if err != nil {
			return nil, nil, fmt.Errorf("failed to locate workflow run request: %v %v", value, err)
		}
		resource.JSONDecode(options)
		if options.Filter == nil {
			options.Filter = endly.DefaultRunnerReportingOption().Filter
		}
	}

	var params = endly.Pairs(getArguments()...)

	if request != nil {
		if len(request.Params) == 0 {
			request.Params = params
		}
		for k, v := range params {
			request.Params[k] = v
		}
		if value, ok := flagset["d"]; ok {
			request.EnableLogging = toolbox.AsBoolean(value)
			request.LoggingDirectory = flag.Lookup("l").Value.String()
		}
		if value, ok := flagset["t"]; ok {
			request.Tasks = value
		}
	}
	return request, options, nil
}

func normalizeArgument(value string) interface{} {
	value = strings.Trim(value, " \"'")
	if strings.HasPrefix(value, "#") || strings.HasPrefix(value, "@") {
		resource := url.NewResource(string(value[1:]))
		text, err := resource.DownloadText();
		if err == nil {
			value = text
		}
	}
	if structure, err := toolbox.JSONToInterface(value); err == nil {
		return structure
	}
	return value
}

func getArguments() []interface{} {
	var arguments = make([]interface{}, 0)
	if len(os.Args) > 1 {
		for i := 1; i < len(os.Args); i++ {
			if strings.HasPrefix(os.Args[i], "-") {
				if !strings.Contains(os.Args[i], "=") {
					i++
				}
				continue
			}
			arguments = append(arguments, normalizeArgument(os.Args[i]))
		}
	}
	return arguments
}
<commit_msg>corrected type<commit_after>package bootstrap

import (
	_ "github.com/go-sql-driver/mysql"
	_ "github.com/lib/pq"
	_ "github.com/viant/asc"
	_ "github.com/viant/bgc"
	_ "github.com/viant/endly/static" //load external resource like .csv .json files to mem storage
	_ "github.com/viant/toolbox/storage/aws"
	_ "github.com/viant/toolbox/storage/gs"

	"encoding/json"
	"flag"
	"fmt"
	"github.com/viant/endly"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/url"
	"gopkg.in/yaml.v2"
	"log"
	"os"
	"path"
	"strings"
	"time"
)

func init() {

	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
	flag.String("r", "run.json", "<path/url to workflow run request in JSON format> ")
	flag.String("w", "manager", "<workflow name>  if both -r and -w valid options are specified, -w is ignored")
	flag.String("t", "*", "<task/s to run>, t='?' to list all tasks for selected workflow")
	flag.String("l", "logs", "<log directory>")
	flag.Bool("d", false, "enable logging")
	flag.Bool("p", false, "print neatly workflow as JSON")
	flag.String("f", "json", "<workflow or request format>, json or yaml")

	flag.Bool("h", false, "print help")
	flag.Bool("v", false, "print version")

	flag.String("s", "", "<serviceID> print service details, -s='*' prints all service IDs")
	flag.String("a", "", "<action> prints action request representation")

}

func Bootstrap() {
	flag.Usage = printHelp
	flag.Parse()
	flagset := make(map[string]string)
	flag.Visit(func(f *flag.Flag) {
		flagset[f.Name] = f.Value.String()
	})

	_, shouldQuit := flagset["v"]
	flagset["v"] = flag.Lookup("v").Value.String()

	if toolbox.AsBoolean(flagset["v"]) {
		printVersion()
		if shouldQuit {
			return
		}
	}

	if _, ok := flagset["h"]; ok {
		printHelp()
		return
	}

	if _, ok := flagset["a"]; ok {
		printServiceActionRequest()
		return
	}
	if _, ok := flagset["s"]; ok {
		printServiceActions()
		return
	}

	request, option, err := getRunRequestWithOptons(flagset)
	if request == nil {
		flagset["r"] = flag.Lookup("r").Value.String()
		flagset["w"] = flag.Lookup("w").Value.String()
		request, option, err = getRunRequestWithOptons(flagset)
		if err != nil && strings.Contains(err.Error(), "failed to locate workflow: manager") {
			printHelp()
			return
		}
	}

	if err != nil {
		log.Fatal(err)
	}
	if value, ok := flagset["p"]; ok && toolbox.AsBoolean(value) {
		printWorkflow(request.WorkflowURL)
		return
	}

	if flagset["t"] == "?" {
		printWorkflowTasks(request.WorkflowURL)
		return
	}

	runner := endly.NewCliRunner()
	err = runner.Run(request, option)
	if err != nil {
		log.Fatal(err)
	}
	time.Sleep(time.Second)
}

func printWorkflowTasks(URL string) {
	workflow, err := getWorkflow(URL);
	if err != nil {
		log.Fatal(err)
	}
	fmt.Fprintf(os.Stderr,"Workflow '%v' (%v) tasks:\n", workflow.Name, workflow.Source.URL)
	for _, task := range workflow.Tasks {
		fmt.Fprintf(os.Stderr, "\t%v: %v\n", task.Name, task.Description)
	}
}

func printServiceActionRequest() {
	manager := endly.NewManager()
	context := manager.NewContext(toolbox.NewContext())

	var serviceID = flag.Lookup("s").Value.String()
	service, err := context.Service(serviceID)
	if err != nil {
		log.Fatal(err)
	}
	var action = flag.Lookup("a").Value.String()
	request, err := service.NewRequest(action)
	if err != nil {
		log.Fatal(err)
	}
	toolbox.InitStruct(request)
	fmt.Printf("Request: %T\n", request)
	printInFormat(request, fmt.Sprintf("failed to print %v.%v request (%T)", serviceID, action, request)+", %v")
	response, _ := service.NewResponse(action)
	fmt.Printf("\nResponse: %T\n", response)
	toolbox.InitStruct(response)
	printInFormat(response, fmt.Sprintf("failed to print %v.%v response (%T)", serviceID, action, request)+", %v")

}

func printServiceActions() {
	manager := endly.NewManager()
	context := manager.NewContext(toolbox.NewContext())

	var serviceID = flag.Lookup("s").Value.String()

	if serviceID == "*" {
		fmt.Printf("endly services:\n")
		for k, v := range endly.Services(manager) {
			fmt.Printf("%v %T\n", k, v)
		}
		return;
	}

	service, err := context.Service(serviceID)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("'%v' service actions: \n", serviceID)
	for _, action := range service.Actions() {
		fmt.Printf("\t%v\n", action)
	}
}

func getWorkflow(URL string) (*endly.Workflow, error) {
	dao := endly.NewWorkflowDao()
	manager := endly.NewManager()
	context := manager.NewContext(toolbox.NewContext())
	return dao.Load(context, url.NewResource(URL))
}

func printWorkflow(URL string) {
	workflow, err := getWorkflow(URL);
	if err != nil {
		log.Fatal(err)
	}
	printInFormat(workflow, "failed to print workflow: "+URL+", %v")

}

func printInFormat(source interface{}, errorTemplate string) {
	format := flag.Lookup("f").Value.String()
	var buf []byte
	var err error
	switch format {
	case "yaml":
		buf, err = yaml.Marshal(source)
	default:
		buf, err = json.MarshalIndent(source, "", "\t")
	}
	if err != nil {
		log.Fatalf(errorTemplate, err)
	}
	fmt.Printf("%s\n", buf)
}

func printHelp() {
	_, name := path.Split(os.Args[0])
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", name)
	fmt.Fprintf(os.Stderr, "endly [options] [params...]\n")
	fmt.Fprintf(os.Stderr, "\tparams should be key value pair to be supplied as actual workflow parameters\n")
	fmt.Fprintf(os.Stderr, "\tif -r options is used, original request params may be overriden\n\n")

	fmt.Fprintf(os.Stderr, "where options include:\n")
	flag.PrintDefaults()
}

func printVersion() {
	fmt.Fprintf(os.Stdout, "%v %v\n", endly.AppName, endly.GetVersion())
}

func getWorkflowURL(candidate string) (string, string, error) {
	var _, name = path.Split(candidate)
	if path.Ext(candidate) == "" {
		candidate = candidate + ".csv"
	} else {
		name = string(name[:len(name)-4]) //remove extension
	}
	resource := url.NewResource(candidate)
	if _, err := resource.Download(); err != nil {
		resource = url.NewResource(fmt.Sprintf("mem://%v/workflow/%v", endly.EndlyNamespace, candidate))
		if _, memError := resource.Download(); memError != nil {
			return "", "", err
		}
	}
	return name, resource.URL, nil
}

func getRunRequestURL(candidate string) (*url.Resource, error) {
	if path.Ext(candidate) == "" {
		candidate = candidate + ".json"
	}
	resource := url.NewResource(candidate)
	if _, err := resource.Download(); err != nil {
		resource = url.NewResource(fmt.Sprintf("mem://%v/req/%v", endly.EndlyNamespace, candidate))
		if _, memError := resource.Download(); memError != nil {
			return nil, err
		}
	}
	return resource, nil

}

func getRunRequestWithOptons(flagset map[string]string) (*endly.WorkflowRunRequest, *endly.RunnerReportingOptions, error) {
	var request *endly.WorkflowRunRequest
	var options = &endly.RunnerReportingOptions{}
	if value, ok := flagset["w"]; ok {
		name, URL, err := getWorkflowURL(value)
		if err != nil {
			return nil, nil, fmt.Errorf("failed to locate workflow: %v %v", value, err)
		}
		request = &endly.WorkflowRunRequest{
			WorkflowURL: URL,
			Name:        name,
		}
		options = endly.DefaultRunnerReportingOption()
	}
	if value, ok := flagset["r"]; ok {
		resource, err := getRunRequestURL(value)
		if err == nil {
			request = &endly.WorkflowRunRequest{}
			err = resource.JSONDecode(request)
		}
		if request.WorkflowURL == "" {
			parent, _ := toolbox.URLSplit(resource.URL)
			parent = strings.Replace(parent, "req", "workflow", 1)
			request.WorkflowURL = toolbox.URLPathJoin(parent, request.Name+".csv")
		}
		if err != nil {
			return nil, nil, fmt.Errorf("failed to locate workflow run request: %v %v", value, err)
		}
		resource.JSONDecode(options)
		if options.Filter == nil {
			options.Filter = endly.DefaultRunnerReportingOption().Filter
		}
	}

	var params = endly.Pairs(getArguments()...)

	if request != nil {
		if len(request.Params) == 0 {
			request.Params = params
		}
		for k, v := range params {
			request.Params[k] = v
		}
		if value, ok := flagset["d"]; ok {
			request.EnableLogging = toolbox.AsBoolean(value)
			request.LoggingDirectory = flag.Lookup("l").Value.String()
		}
		if value, ok := flagset["t"]; ok {
			request.Tasks = value
		}
	}
	return request, options, nil
}

func normalizeArgument(value string) interface{} {
	value = strings.Trim(value, " \"'")
	if strings.HasPrefix(value, "#") || strings.HasPrefix(value, "@") {
		resource := url.NewResource(string(value[1:]))
		text, err := resource.DownloadText();
		if err == nil {
			value = text
		}
	}
	if structure, err := toolbox.JSONToInterface(value); err == nil {
		return structure
	}
	return value
}

func getArguments() []interface{} {
	var arguments = make([]interface{}, 0)
	if len(os.Args) > 1 {
		for i := 1; i < len(os.Args); i++ {
			if strings.HasPrefix(os.Args[i], "-") {
				if !strings.Contains(os.Args[i], "=") {
					i++
				}
				continue
			}
			arguments = append(arguments, normalizeArgument(os.Args[i]))
		}
	}
	return arguments
}
<|endoftext|>


<commit_before>package transloadit

import (
	"testing"
)

func TestNewClient(t *testing.T) {

	client := setup(t)

	bored, err := client.getBoredInstance()
	if err != nil {
		t.Fatal(err)
	}

	if bored == "" {
		t.Fatal("no bored instance provided")
	}
}
<commit_msg>Renamed GetBoredInstance<commit_after>package transloadit

import (
	"testing"
)

func TestGetBoredInstance(t *testing.T) {

	client := setup(t)

	bored, err := client.getBoredInstance()
	if err != nil {
		t.Fatal(err)
	}

	if bored == "" {
		t.Fatal("no bored instance provided")
	}
}

<|endoftext|>


<commit_before>package main

import (
	"flag"
	_ "github.com/BurntSushi/toml"
	"log"
	"net/http"
)

func client(api *TelegramBotApi) {
	for {
		updates, _ := api.GetUpdates(0, 100, 0, []string{})
		log.Println("updates:", updates)
		log.Println("")
		break
	}
}

func server(api *TelegramBotApi) {
	mux := http.NewServeMux()
	mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
	})

	srv := http.Server{
		Addr:    ":8080",
		Handler: mux,
	}

	log.Fatal(srv.ListenAndServe())
}

func main() {
	configPath := flag.String("config", "", "Path to toml config file.")
	token := flag.String("token", "", "A unique authentication token.")

	flag.Parse()

	log.Println("load config from " + *configPath)
	log.Println("use token " + *token)

	api := New(*token)

	if me, err := api.GetMe(); err != nil {
        log.Fatal("exit: ", err)
	} else {
		log.Println("Telegram Bot API: /getMe:")
		log.Println("    Id:", me.Id)
		log.Println("    First Name:", me.FirstName)
		log.Println("    Last Name:", me.LastName)
		log.Println("    Username:", me.UserName)
	}

	client(api)
}
<commit_msg>Make frame for request processing.<commit_after>package main

import (
	"flag"
	_ "github.com/BurntSushi/toml"
	"log"
	"net/http"
)

var storage *Storage

type TelePyth struct {
	api     *TelegramBotApi
	storage *Storage

	polling bool
	timeout int
}

func (t *TelePyth) HandleTelegramUpdate(update *Update) {
	log.Println("updates:", update.Message)
	log.Println("updates:", update.Message.From)
	log.Println("")

	token, err := storage.InsertUser(&update.Message.From)

	if err != nil {
		//  TODO: log error and ask try again
		return
	}

	if update.Message.Text == "/show" {
		user, err := storage.SelectUserBy(token)
		log.Println("user: ", user, err)
		return
	}

	err = (&SendMessage{
		ChatId: update.Message.From.Id,
		Text:   "Your access token is " + token,
	}).To(t.api)

	if err != nil {
		log.Println("error: ", err)
	}
}

func (t *TelePyth) HandleHttpRequest() {
}

func (t *TelePyth) HandleWebhookRequest(w http.ResponseWriter, req *http.Request) {
	log.Println("HandleWebhookRequest(): not implemented!")
}

func (t *TelePyth) HandleNotifyRequest(w http.ResponseWriter, req *http.Request) {
	log.Println("HandleNotifyRequest(): not implemented!")
}

func (t *TelePyth) PollUpdates() {
	log.Println("timeout: ", t.timeout)
	for {
		updates, err := t.api.GetUpdates(0, 100, t.timeout, nil)

		if err != nil {
			//  TODO: more logging
			log.Println(err)
		} else {
			log.Println(updates)
		}
	}
}

func (t *TelePyth) Serve() error {
	// run go-routing for long polling
	if t.polling {
		log.Println("poling:", t.polling)
		go t.PollUpdates()
	}

	// run http server
	mux := http.NewServeMux()
	mux.HandleFunc("/api/webhook/"+t.api.GetToken(), t.HandleWebhookRequest)
	mux.HandleFunc("/api/notify/", t.HandleNotifyRequest)

	srv := http.Server{
		Addr:    ":8080",
		Handler: mux,
	}

	return srv.ListenAndServe()
}

func main() {
	configPath := flag.String("config", "", "Path to toml config file.")
	token := flag.String("token", "", "A unique authentication token.")
	dbPath := flag.String("database", "var/bolt.data", "Create or open a database at the given path.")

	flag.Parse()

	log.Println("load config from " + *configPath)
	log.Println("open database at " + *dbPath)

	if db, err := NewStorage(*dbPath); err != nil {
		log.Fatal(err)
	} else {
		storage = db
		defer storage.Close()
	}

	log.Println("use token " + *token)
	api := New(*token)

	if me, err := api.GetMe(); err != nil {
		log.Fatal("exit: ", err)
	} else {
		log.Println("Telegram Bot API: /getMe:")
		log.Println("    Id:", me.Id)
		log.Println("    First Name:", me.FirstName)
		log.Println("    Last Name:", me.LastName)
		log.Println("    Username:", me.UserName)
	}

	log.Fatal((&TelePyth{
		api:     api,
		storage: storage,
		polling: true,
		timeout: 30,
	}).Serve())
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"text/template"

	"github.com/armon/consul-api"
)

type Template struct {
	Input string
}

// GoString returns the detailed format of this object
func (t *Template) GoString() string {
	return fmt.Sprintf("*%#v", *t)
}

// Dependencies returns the dependencies that this template has.
func (t *Template) Dependencies() ([]Dependency, error) {
	var deps []Dependency

	contents, err := ioutil.ReadFile(t.Input)
	if err != nil {
		return nil, err
	}

	tmpl, err := template.New("out").Funcs(template.FuncMap{
		"service":   t.dependencyAcc(&deps, DependencyTypeService),
		"key":       t.dependencyAcc(&deps, DependencyTypeKey),
		"keyPrefix": t.dependencyAcc(&deps, DependencyTypeKeyPrefix),
	}).Parse(string(contents))

	if err != nil {
		return nil, err
	}

	err = tmpl.Execute(ioutil.Discard, nil)
	if err != nil {
		return nil, err
	}

	return deps, nil
}

// Execute takes the given template context and processes the template.
//
// If the TemplateContext is nil, an error will be returned.
//
// If the TemplateContext does not have all required Dependencies, an error will
// be returned.
func (t *Template) Execute(wr io.Writer, c *TemplateContext) error {
	if wr == nil {
		return errors.New("wr must be given")
	}

	if c == nil {
		return errors.New("templateContext must be given")
	}

	// Make sure the context contains everything we need
	if err := t.validateDependencies(c); err != nil {
		return err
	}

	// Render the template
	contents, err := ioutil.ReadFile(t.Input)
	if err != nil {
		return err
	}

	tmpl, err := template.New("out").Funcs(template.FuncMap{
		"service":   c.Evaluator(DependencyTypeService),
		"key":       c.Evaluator(DependencyTypeKey),
		"keyPrefix": c.Evaluator(DependencyTypeKeyPrefix),
	}).Parse(string(contents))

	if err != nil {
		return err
	}

	err = tmpl.Execute(wr, c)
	if err != nil {
		return err
	}

	return nil
}

// Helper function that is used by the dependency collecting.
func (t *Template) dependencyAcc(d *[]Dependency, dt DependencyType) interface{} {
	return func(s string) (interface{}, error) {
		switch dt {
		case DependencyTypeService:
			sd, err := ParseServiceDependency(s)
			if err != nil {
				return nil, err
			}
			*d = append(*d, sd)

			return []*Service{}, nil
		case DependencyTypeKey:
			kd, err := ParseKeyDependency(s)
			if err != nil {
				return nil, err
			}
			*d = append(*d, kd)

			return "", nil
		case DependencyTypeKeyPrefix:
			kpd, err := ParseKeyPrefixDependency(s)
			if err != nil {
				return nil, err
			}
			*d = append(*d, kpd)

			return []*KeyPair{}, nil
		default:
			return nil, fmt.Errorf("unknown DependencyType %#v", dt)
		}
	}
}

// Validates that all required dependencies in t are defined in c.
func (t *Template) validateDependencies(c *TemplateContext) error {
	deps, err := t.Dependencies()
	if err != nil {
		return err
	}

	for _, dep := range deps {
		switch dep.(type) {
		case *ServiceDependency:
			sd, ok := dep.(*ServiceDependency)
			if !ok {
				return fmt.Errorf("could not convert to ServiceDependency")
			}
			if _, ok := c.Services[sd.Key()]; !ok {
				return fmt.Errorf("templateContext missing service `%s'", sd.Key())
			}
		case *KeyDependency:
			kd, ok := dep.(*KeyDependency)
			if !ok {
				return fmt.Errorf("could not convert to KeyDependency")
			}
			if _, ok := c.Keys[kd.Key()]; !ok {
				return fmt.Errorf("templateContext missing key `%s'", kd.Key())
			}
		case *KeyPrefixDependency:
			kpd, ok := dep.(*KeyPrefixDependency)
			if !ok {
				return fmt.Errorf("could not convert to KeyPrefixDependency")
			}
			if _, ok := c.KeyPrefixes[kpd.Key()]; !ok {
				return fmt.Errorf("templateContext missing keyPrefix `%s'", kpd.Key())
			}
		default:
			return fmt.Errorf("unknown dependency type %#v", dep)
		}
	}

	return nil
}

/// ------------------------- ///

// TemplateContext is what Template uses to determine the values that are
// available for template parsing.
type TemplateContext struct {
	Services    map[string][]*Service
	Keys        map[string]string
	KeyPrefixes map[string][]*KeyPair
}

// GoString returns the detailed format of this object
func (c *TemplateContext) GoString() string {
	return fmt.Sprintf("*%#v", *c)
}

// Evaluator takes a DependencyType and returns a function which returns the
// value in the TemplateContext that corresponds to the requested item.
func (c *TemplateContext) Evaluator(dt DependencyType) interface{} {
	return func(s string) (interface{}, error) {
		switch dt {
		case DependencyTypeService:
			return c.Services[s], nil
		case DependencyTypeKey:
			return c.Keys[s], nil
		case DependencyTypeKeyPrefix:
			return c.KeyPrefixes[s], nil
		default:
			return nil, fmt.Errorf("unexpected DependencyType %#v", dt)
		}
	}
}

/// ------------------------- ///

type Service struct {
	Node    string
	Address string
	ID      string
	Name    string
	Tags    []string
	Port    uint
}

// GoString returns the detailed format of this object
func (s *Service) GoString() string {
	return fmt.Sprintf("*%#v", *s)
}

/// ------------------------- ///

type KeyPair struct {
	Key   string
	Value string
}

// GoString returns the detailed format of this object
func (kp *KeyPair) GoString() string {
	return fmt.Sprintf("*%#v", *kp)
}

// NewFromConsul creates a new KeyPair object by parsing the values in the
// consulapi.KVPair. Not all values are transferred.
func (kp KeyPair) NewFromConsul(c *consulapi.KVPair) {
	// TODO: lol
	panic("not done!")
}

// DependencyType is an enum type that says the kind of the dependency.
type DependencyType byte

const (
	DependencyTypeInvalid DependencyType = iota
	DependencyTypeService
	DependencyTypeKey
	DependencyTypeKeyPrefix
)
<commit_msg>Use a stronger return value binding from template context functions<commit_after>package main

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"text/template"

	"github.com/armon/consul-api"
)

type Template struct {
	Input string
}

// GoString returns the detailed format of this object
func (t *Template) GoString() string {
	return fmt.Sprintf("*%#v", *t)
}

// Dependencies returns the dependencies that this template has.
func (t *Template) Dependencies() ([]Dependency, error) {
	var deps []Dependency

	contents, err := ioutil.ReadFile(t.Input)
	if err != nil {
		return nil, err
	}

	tmpl, err := template.New("out").Funcs(template.FuncMap{
		"service":   t.dependencyAcc(&deps, DependencyTypeService),
		"key":       t.dependencyAcc(&deps, DependencyTypeKey),
		"keyPrefix": t.dependencyAcc(&deps, DependencyTypeKeyPrefix),
	}).Parse(string(contents))

	if err != nil {
		return nil, err
	}

	err = tmpl.Execute(ioutil.Discard, nil)
	if err != nil {
		return nil, err
	}

	return deps, nil
}

// Execute takes the given template context and processes the template.
//
// If the TemplateContext is nil, an error will be returned.
//
// If the TemplateContext does not have all required Dependencies, an error will
// be returned.
func (t *Template) Execute(wr io.Writer, c *TemplateContext) error {
	if wr == nil {
		return errors.New("wr must be given")
	}

	if c == nil {
		return errors.New("templateContext must be given")
	}

	// Make sure the context contains everything we need
	if err := t.validateDependencies(c); err != nil {
		return err
	}

	// Render the template
	contents, err := ioutil.ReadFile(t.Input)
	if err != nil {
		return err
	}

	tmpl, err := template.New("out").Funcs(template.FuncMap{
		"service":   c.Evaluator(DependencyTypeService),
		"key":       c.Evaluator(DependencyTypeKey),
		"keyPrefix": c.Evaluator(DependencyTypeKeyPrefix),
	}).Parse(string(contents))

	if err != nil {
		return err
	}

	err = tmpl.Execute(wr, c)
	if err != nil {
		return err
	}

	return nil
}

// Helper function that is used by the dependency collecting.
func (t *Template) dependencyAcc(d *[]Dependency, dt DependencyType) func(string) (interface{}, error) {
	return func(s string) (interface{}, error) {
		switch dt {
		case DependencyTypeService:
			sd, err := ParseServiceDependency(s)
			if err != nil {
				return nil, err
			}
			*d = append(*d, sd)

			return []*Service{}, nil
		case DependencyTypeKey:
			kd, err := ParseKeyDependency(s)
			if err != nil {
				return nil, err
			}
			*d = append(*d, kd)

			return "", nil
		case DependencyTypeKeyPrefix:
			kpd, err := ParseKeyPrefixDependency(s)
			if err != nil {
				return nil, err
			}
			*d = append(*d, kpd)

			return []*KeyPair{}, nil
		default:
			return nil, fmt.Errorf("unknown DependencyType %#v", dt)
		}
	}
}

// Validates that all required dependencies in t are defined in c.
func (t *Template) validateDependencies(c *TemplateContext) error {
	deps, err := t.Dependencies()
	if err != nil {
		return err
	}

	for _, dep := range deps {
		switch dep.(type) {
		case *ServiceDependency:
			sd, ok := dep.(*ServiceDependency)
			if !ok {
				return fmt.Errorf("could not convert to ServiceDependency")
			}
			if _, ok := c.Services[sd.Key()]; !ok {
				return fmt.Errorf("templateContext missing service `%s'", sd.Key())
			}
		case *KeyDependency:
			kd, ok := dep.(*KeyDependency)
			if !ok {
				return fmt.Errorf("could not convert to KeyDependency")
			}
			if _, ok := c.Keys[kd.Key()]; !ok {
				return fmt.Errorf("templateContext missing key `%s'", kd.Key())
			}
		case *KeyPrefixDependency:
			kpd, ok := dep.(*KeyPrefixDependency)
			if !ok {
				return fmt.Errorf("could not convert to KeyPrefixDependency")
			}
			if _, ok := c.KeyPrefixes[kpd.Key()]; !ok {
				return fmt.Errorf("templateContext missing keyPrefix `%s'", kpd.Key())
			}
		default:
			return fmt.Errorf("unknown dependency type %#v", dep)
		}
	}

	return nil
}

/// ------------------------- ///

// TemplateContext is what Template uses to determine the values that are
// available for template parsing.
type TemplateContext struct {
	Services    map[string][]*Service
	Keys        map[string]string
	KeyPrefixes map[string][]*KeyPair
}

// GoString returns the detailed format of this object
func (c *TemplateContext) GoString() string {
	return fmt.Sprintf("*%#v", *c)
}

// Evaluator takes a DependencyType and returns a function which returns the
// value in the TemplateContext that corresponds to the requested item.
func (c *TemplateContext) Evaluator(dt DependencyType) func(string) (interface{}, error) {
	return func(s string) (interface{}, error) {
		switch dt {
		case DependencyTypeService:
			return c.Services[s], nil
		case DependencyTypeKey:
			return c.Keys[s], nil
		case DependencyTypeKeyPrefix:
			return c.KeyPrefixes[s], nil
		default:
			return nil, fmt.Errorf("unexpected DependencyType %#v", dt)
		}
	}
}

/// ------------------------- ///

type Service struct {
	Node    string
	Address string
	ID      string
	Name    string
	Tags    []string
	Port    uint
}

// GoString returns the detailed format of this object
func (s *Service) GoString() string {
	return fmt.Sprintf("*%#v", *s)
}

/// ------------------------- ///

type KeyPair struct {
	Key   string
	Value string
}

// GoString returns the detailed format of this object
func (kp *KeyPair) GoString() string {
	return fmt.Sprintf("*%#v", *kp)
}

// NewFromConsul creates a new KeyPair object by parsing the values in the
// consulapi.KVPair. Not all values are transferred.
func (kp KeyPair) NewFromConsul(c *consulapi.KVPair) {
	// TODO: lol
	panic("not done!")
}

// DependencyType is an enum type that says the kind of the dependency.
type DependencyType byte

const (
	DependencyTypeInvalid DependencyType = iota
	DependencyTypeService
	DependencyTypeKey
	DependencyTypeKeyPrefix
)
<|endoftext|>


<commit_before>package wuxia

import (
	"errors"
	"fmt"
	"html/template"

	"github.com/robertkrimen/otto"
)

//Template extends the *template.Template struct by supporting template
//functions defined in the javascript programming language. This depends heavily
//on the otto VM , and the functions are extracted from the otto Virtual
//Machine.
type Template struct {
	vm      *otto.Otto
	jsFuncs []string
	funcs   template.FuncMap
	*template.Template
}

func (t *Template) funcMap() template.FuncMap {
	rst := make(template.FuncMap)
	for _, name := range t.jsFuncs {
		rst[name] = t.jsTplFunc(name)
	}
	for k, v := range t.funcs {
		rst[k] = v
	}
	return rst
}

// returns a function that can be executed within Go template. The defined
// javascript function should accept one argument and return a string. Any
// exceptions raised by the javascript function will we returned when the
// templates are executed( Effectively halting rendering process).
//
// The functions are the one registered on the Tpl global pbject exposed in the
// Javascript runtine.
//
// You can register a function by attaching it to  Tpl.funcs
//   // Example
//   Tpl.funcs.world=function(hello){return hello+",world"}
// The xample function adds ",world" string to the passed argument. The above
// function can then be used like any other Go template functions like this
//   {{"hello"|world}}
//
// The type of the argument is not enforced so the template function
// implementations should be careful on what type of objects they are operating
// on and also great care should be taken on the conext object passed to these
// functions within the templates.
func (t *Template) jsTplFunc(name string) func(interface{}) (string, error) {
	return func(arg interface{}) (string, error) {
		call := fmt.Sprintf("Tpl.funcs.%s", name)
		rst, err := t.vm.Call(call, nil, arg)
		if err != nil {
			return "", err
		}
		if !rst.IsString() {
			return "", errors.New("non string retrun value from " + name + " template func")
		}
		s, _ := rst.ToString()
		return s, nil
	}
}

func (t *Template) New() *Template {
	if t.jsFuncs == nil || len(t.jsFuncs) == 0 {
		if t.vm != nil {
			rst, err := t.vm.Call("Tpl.getTplFuncs", nil)
			if err == nil {
				v, _ := rst.Export()
				if va, ok := v.([]string); ok {
					t.jsFuncs = va
				}
			}
		}
	}
	tpl := template.New("base").Funcs(t.funcMap())
	return &Template{
		jsFuncs:  t.jsFuncs,
		vm:       t.vm,
		funcs:    t.funcs,
		Template: tpl,
	}
}
<commit_msg>Update comment<commit_after>package wuxia

import (
	"errors"
	"fmt"
	"html/template"

	"github.com/robertkrimen/otto"
)

//Template extends the *template.Template struct by supporting template
//functions defined in the javascript programming language. This depends heavily
//on the otto VM , and the functions are extracted from the otto Virtual
//Machine.
type Template struct {
	vm      *otto.Otto
	jsFuncs []string
	funcs   template.FuncMap
	*template.Template
}

func (t *Template) funcMap() template.FuncMap {
	rst := make(template.FuncMap)
	for _, name := range t.jsFuncs {
		rst[name] = t.jsTplFunc(name)
	}
	for k, v := range t.funcs {
		rst[k] = v
	}
	return rst
}

// returns a JS function that can be executed within Go template. The defined
// javascript function should accept one argument and return a string. Any
// exceptions raised by the javascript function will we returned when the
// templates are executed( Effectively halting rendering process).
//
// The functions are the one registered on the Tpl global pbject exposed in the
// Javascript runtine.
//
// You can register a function by attaching it to  Tpl.funcs
//   // Example
//   Tpl.funcs.world=function(hello){return hello+",world"}
// The xample function adds ",world" string to the passed argument. The above
// function can then be used like any other Go template functions like this
//   {{"hello"|world}}
//
// The type of the argument is not enforced so the template function
// implementations should be careful on what type of objects they are operating
// on and also great care should be taken on the conext object passed to these
// functions within the templates.
func (t *Template) jsTplFunc(name string) func(interface{}) (string, error) {
	return func(arg interface{}) (string, error) {
		call := fmt.Sprintf("Tpl.funcs.%s", name)
		rst, err := t.vm.Call(call, nil, arg)
		if err != nil {
			return "", err
		}
		if !rst.IsString() {
			return "", errors.New("non string retrun value from " + name + " template func")
		}
		s, _ := rst.ToString()
		return s, nil
	}
}

func (t *Template) New() *Template {
	if t.jsFuncs == nil || len(t.jsFuncs) == 0 {
		if t.vm != nil {
			rst, err := t.vm.Call("Tpl.getTplFuncs", nil)
			if err == nil {
				v, _ := rst.Export()
				if va, ok := v.([]string); ok {
					t.jsFuncs = va
				}
			}
		}
	}
	tpl := template.New("base").Funcs(t.funcMap())
	return &Template{
		jsFuncs:  t.jsFuncs,
		vm:       t.vm,
		funcs:    t.funcs,
		Template: tpl,
	}
}
<|endoftext|>