<commit_before>package ginkgo

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/signal"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"syscall"
	"time"

	"github.com/openshift/origin/test/extended/testdata"

	"github.com/onsi/ginkgo/config"
	"github.com/openshift/origin/pkg/monitor"
	monitorserialization "github.com/openshift/origin/pkg/monitor/serialization"
	"k8s.io/apimachinery/pkg/util/sets"
)

// Options is used to run a suite of tests by invoking each test
// as a call to a child worker (the run-tests command).
type Options struct {
	Parallelism int
	Count       int
	FailFast    bool
	Timeout     time.Duration
	JUnitDir    string
	TestFile    string
	OutFile     string

	// Regex allows a selection of a subset of tests
	Regex string
	// MatchFn if set is also used to filter the suite contents
	MatchFn func(name string) bool

	// SyntheticEventTests allows the caller to translate events or outside
	// context into a failure.
	SyntheticEventTests JUnitsForEvents

	IncludeSuccessOutput bool

	CommandEnv []string

	DryRun        bool
	PrintCommands bool
	Out, ErrOut   io.Writer

	StartTime time.Time
}

func (opt *Options) AsEnv() []string {
	var args []string
	args = append(args, fmt.Sprintf("TEST_SUITE_START_TIME=%d", opt.StartTime.Unix()))
	args = append(args, opt.CommandEnv...)
	return args
}

func (opt *Options) SelectSuite(suites []*TestSuite, args []string) (*TestSuite, error) {
	var suite *TestSuite

	if len(opt.TestFile) > 0 {
		var in []byte
		var err error
		if opt.TestFile == "-" {
			in, err = ioutil.ReadAll(os.Stdin)
			if err != nil {
				return nil, err
			}
		} else {
			in, err = ioutil.ReadFile(opt.TestFile)
		}
		if err != nil {
			return nil, err
		}
		suite, err = newSuiteFromFile("files", in)
		if err != nil {
			return nil, fmt.Errorf("could not read test suite from input: %v", err)
		}
	}

	if suite == nil && len(args) == 0 {
		fmt.Fprintf(opt.ErrOut, SuitesString(suites, "Select a test suite to run against the server:\n\n"))
		return nil, fmt.Errorf("specify a test suite to run, for example: %s run %s", filepath.Base(os.Args[0]), suites[0].Name)
	}
	if suite == nil && len(args) > 0 {
		for _, s := range suites {
			if s.Name == args[0] {
				suite = s
				break
			}
		}
	}
	if suite == nil {
		fmt.Fprintf(opt.ErrOut, SuitesString(suites, "Select a test suite to run against the server:\n\n"))
		return nil, fmt.Errorf("suite %q does not exist", args[0])
	}
	return suite, nil
}

func (opt *Options) Run(suite *TestSuite) error {
	if len(opt.Regex) > 0 {
		if err := filterWithRegex(suite, opt.Regex); err != nil {
			return err
		}
	}
	if opt.MatchFn != nil {
		original := suite.Matches
		suite.Matches = func(name string) bool {
			return original(name) && opt.MatchFn(name)
		}
	}

	syntheticEventTests := JUnitsForAllEvents{
		opt.SyntheticEventTests,
		suite.SyntheticEventTests,
	}

	tests, err := testsForSuite(config.GinkgoConfig)
	if err != nil {
		return err
	}

	// This ensures that tests in the identified paths do not run in parallel, because
	// the test suite reuses shared resources without considering whether another test
	// could be running at the same time. While these are technically [Serial], ginkgo
	// parallel mode provides this guarantee. Doing this for all suites would be too
	// slow.
	setTestExclusion(tests, func(suitePath string, t *testCase) bool {
		for _, name := range []string{
			"/k8s.io/kubernetes/test/e2e/apps/disruption.go",
		} {
			if strings.HasSuffix(suitePath, name) {
				return true
			}
		}
		return false
	})

	tests = suite.Filter(tests)
	if len(tests) == 0 {
		return fmt.Errorf("suite %q does not contain any tests", suite.Name)
	}

	count := opt.Count
	if count == 0 {
		count = suite.Count
	}

	start := time.Now()
	if opt.StartTime.IsZero() {
		opt.StartTime = start
	}

	if opt.PrintCommands {
		status := newTestStatus(opt.Out, true, len(tests), time.Minute, &monitor.Monitor{}, monitor.NewNoOpMonitor(), opt.AsEnv())
		newParallelTestQueue().Execute(context.Background(), tests, 1, status.OutputCommand)
		return nil
	}
	if opt.DryRun {
		for _, test := range sortedTests(tests) {
			fmt.Fprintf(opt.Out, "%q\n", test.name)
		}
		return nil
	}

	if len(opt.JUnitDir) > 0 {
		if _, err := os.Stat(opt.JUnitDir); err != nil {
			if !os.IsNotExist(err) {
				return fmt.Errorf("could not access --junit-dir: %v", err)
			}
			if err := os.MkdirAll(opt.JUnitDir, 0755); err != nil {
				return fmt.Errorf("could not create --junit-dir: %v", err)
			}
		}
	}

	parallelism := opt.Parallelism
	if parallelism == 0 {
		parallelism = suite.Parallelism
	}
	if parallelism == 0 {
		parallelism = 10
	}
	timeout := opt.Timeout
	if timeout == 0 {
		timeout = suite.TestTimeout
	}
	if timeout == 0 {
		timeout = 15 * time.Minute
	}

	ctx, cancelFn := context.WithCancel(context.Background())
	defer cancelFn()
	abortCh := make(chan os.Signal)
	go func() {
		<-abortCh
		fmt.Fprintf(opt.ErrOut, "Interrupted, terminating tests\n")
		cancelFn()
		sig := <-abortCh
		fmt.Fprintf(opt.ErrOut, "Interrupted twice, exiting (%s)\n", sig)
		switch sig {
		case syscall.SIGINT:
			os.Exit(130)
		default:
			os.Exit(0)
		}
	}()
	signal.Notify(abortCh, syscall.SIGINT, syscall.SIGTERM)

	m, err := monitor.Start(ctx)
	if err != nil {
		return err
	}
	// if we run a single test, always include success output
	includeSuccess := opt.IncludeSuccessOutput
	if len(tests) == 1 && count == 1 {
		includeSuccess = true
	}

	early, normal := splitTests(tests, func(t *testCase) bool {
		return strings.Contains(t.name, "[Early]")
	})

	late, normal := splitTests(normal, func(t *testCase) bool {
		return strings.Contains(t.name, "[Late]")
	})

	expectedTestCount := len(early) + len(late)
	if count != -1 {
		original := normal
		for i := 1; i < count; i++ {
			normal = append(normal, copyTests(original)...)
		}
	}
	expectedTestCount += len(normal)

	status := newTestStatus(opt.Out, includeSuccess, expectedTestCount, timeout, m, m, opt.AsEnv())
	testCtx := ctx
	if opt.FailFast {
		var cancelFn context.CancelFunc
		testCtx, cancelFn = context.WithCancel(testCtx)
		status.AfterTest(func(t *testCase) {
			if t.failed {
				cancelFn()
			}
		})
	}

	tests = nil

	// run our Early tests
	q := newParallelTestQueue()
	q.Execute(testCtx, early, parallelism, status.Run)
	tests = append(tests, early...)

	// repeat the normal suite until context cancel when in the forever loop
	for i := 0; (i < 1 || count == -1) && testCtx.Err() == nil; i++ {
		copied := copyTests(normal)
		q.Execute(testCtx, copied, parallelism, status.Run)
		tests = append(tests, copied...)
	}

	// run Late test suits after everything else
	q.Execute(testCtx, late, parallelism, status.Run)
	tests = append(tests, late...)

	// calculate the effective test set we ran, excluding any incompletes
	tests, _ = splitTests(tests, func(t *testCase) bool { return t.success || t.failed || t.skipped })

	duration := time.Now().Sub(start).Round(time.Second / 10)
	if duration > time.Minute {
		duration = duration.Round(time.Second)
	}

	pass, fail, skip, failing := summarizeTests(tests)

	// monitor the cluster while the tests are running and report any detected anomalies
	var syntheticTestResults []*JUnitTestCase
	var syntheticFailure bool
	events := m.EventIntervals(time.Time{}, time.Time{})
	if err = monitorserialization.EventsToFile(path.Join(os.Getenv("ARTIFACT_DIR"), "e2e-events.json"), events); err != nil {
		fmt.Fprintf(opt.Out, "Failed to write event file: %v\n", err)
	}
	if err = monitorserialization.EventsIntervalsToFile(path.Join(os.Getenv("ARTIFACT_DIR"), "e2e-intervals.json"), events); err != nil {
		fmt.Fprintf(opt.Out, "Failed to write event file: %v\n", err)
	}
	if eventIntervalsJSON, err := monitorserialization.EventsIntervalsToJSON(events); err == nil {
		e2eChartTemplate := testdata.MustAsset("e2echart/e2e-chart-template.html")
		e2eChartHTML := bytes.ReplaceAll(e2eChartTemplate, []byte("EVENT_INTERVAL_JSON_GOES_HERE"), eventIntervalsJSON)
		e2eChartHTMLPath := path.Join(os.Getenv("ARTIFACT_DIR"), "e2e-intervals.html")
		if err := ioutil.WriteFile(e2eChartHTMLPath, e2eChartHTML, 0644); err != nil {
			fmt.Fprintf(opt.Out, "Failed to write event html: %v\n", err)
		}
	} else {
		fmt.Fprintf(opt.Out, "Failed to write event html: %v\n", err)
	}

	if len(events) > 0 {
		eventsForTests := createEventsForTests(tests)

		var buf *bytes.Buffer
		syntheticTestResults, buf, _ = createSyntheticTestsFromMonitor(m, eventsForTests, duration)
		testCases := syntheticEventTests.JUnitsForEvents(events, duration)
		syntheticTestResults = append(syntheticTestResults, testCases...)

		if len(syntheticTestResults) > 0 {
			// mark any failures by name
			failing, flaky := sets.NewString(), sets.NewString()
			for _, test := range syntheticTestResults {
				if test.FailureOutput != nil {
					failing.Insert(test.Name)
				}
			}
			// if a test has both a pass and a failure, flag it
			// as a flake
			for _, test := range syntheticTestResults {
				if test.FailureOutput == nil {
					if failing.Has(test.Name) {
						flaky.Insert(test.Name)
					}
				}
			}
			failing = failing.Difference(flaky)
			if failing.Len() > 0 {
				fmt.Fprintf(buf, "Failing invariants:\n\n%s\n\n", strings.Join(failing.List(), "\n"))
				syntheticFailure = true
			}
			if flaky.Len() > 0 {
				fmt.Fprintf(buf, "Flaky invariants:\n\n%s\n\n", strings.Join(flaky.List(), "\n"))
			}
		}

		opt.Out.Write(buf.Bytes())
	}

	// attempt to retry failures to do flake detection
	if fail > 0 && fail <= suite.MaximumAllowedFlakes {
		var retries []*testCase
		for _, test := range failing {
			retry := test.Retry()
			retries = append(retries, retry)
			tests = append(tests, retry)
			if len(retries) > suite.MaximumAllowedFlakes {
				break
			}
		}

		q := newParallelTestQueue()
		status := newTestStatus(ioutil.Discard, opt.IncludeSuccessOutput, len(retries), timeout, m, m, opt.AsEnv())
		q.Execute(testCtx, retries, parallelism, status.Run)
		var flaky []string
		var repeatFailures []*testCase
		for _, test := range retries {
			if test.success {
				flaky = append(flaky, test.name)
			} else {
				repeatFailures = append(repeatFailures, test)
			}
		}
		if len(flaky) > 0 {
			failing = repeatFailures
			sort.Strings(flaky)
			fmt.Fprintf(opt.Out, "Flaky tests:\n\n%s\n\n", strings.Join(flaky, "\n"))
		}
	}

	// report the outcome of the test
	if len(failing) > 0 {
		names := sets.NewString(testNames(failing)...).List()
		fmt.Fprintf(opt.Out, "Failing tests:\n\n%s\n\n", strings.Join(names, "\n"))
	}

	if len(opt.JUnitDir) > 0 {
		if err := writeJUnitReport("junit_e2e", "openshift-tests", tests, opt.JUnitDir, duration, opt.ErrOut, syntheticTestResults...); err != nil {
			fmt.Fprintf(opt.Out, "error: Unable to write e2e JUnit results: %v", err)
		}
	}

	if fail > 0 {
		if len(failing) > 0 || suite.MaximumAllowedFlakes == 0 {
			return fmt.Errorf("%d fail, %d pass, %d skip (%s)", fail, pass, skip, duration)
		}
		fmt.Fprintf(opt.Out, "%d flakes detected, suite allows passing with only flakes\n\n", fail)
	}

	if syntheticFailure {
		return fmt.Errorf("failed because an invariant was violated, %d pass, %d skip (%s)\n", pass, skip, duration)
	}

	fmt.Fprintf(opt.Out, "%d pass, %d skip (%s)\n", pass, skip, duration)
	return ctx.Err()
}
<commit_msg>pkg/test/ginkgo/cmd_runsuite: Fix e2e-* clobber for upgrade-conformance<commit_after>package ginkgo

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/signal"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"syscall"
	"time"

	"github.com/openshift/origin/test/extended/testdata"

	"github.com/onsi/ginkgo/config"
	"github.com/openshift/origin/pkg/monitor"
	monitorserialization "github.com/openshift/origin/pkg/monitor/serialization"
	"k8s.io/apimachinery/pkg/util/sets"
)

// Options is used to run a suite of tests by invoking each test
// as a call to a child worker (the run-tests command).
type Options struct {
	Parallelism int
	Count       int
	FailFast    bool
	Timeout     time.Duration
	JUnitDir    string
	TestFile    string
	OutFile     string

	// Regex allows a selection of a subset of tests
	Regex string
	// MatchFn if set is also used to filter the suite contents
	MatchFn func(name string) bool

	// SyntheticEventTests allows the caller to translate events or outside
	// context into a failure.
	SyntheticEventTests JUnitsForEvents

	IncludeSuccessOutput bool

	CommandEnv []string

	DryRun        bool
	PrintCommands bool
	Out, ErrOut   io.Writer

	StartTime time.Time
}

func (opt *Options) AsEnv() []string {
	var args []string
	args = append(args, fmt.Sprintf("TEST_SUITE_START_TIME=%d", opt.StartTime.Unix()))
	args = append(args, opt.CommandEnv...)
	return args
}

func (opt *Options) SelectSuite(suites []*TestSuite, args []string) (*TestSuite, error) {
	var suite *TestSuite

	if len(opt.TestFile) > 0 {
		var in []byte
		var err error
		if opt.TestFile == "-" {
			in, err = ioutil.ReadAll(os.Stdin)
			if err != nil {
				return nil, err
			}
		} else {
			in, err = ioutil.ReadFile(opt.TestFile)
		}
		if err != nil {
			return nil, err
		}
		suite, err = newSuiteFromFile("files", in)
		if err != nil {
			return nil, fmt.Errorf("could not read test suite from input: %v", err)
		}
	}

	if suite == nil && len(args) == 0 {
		fmt.Fprintf(opt.ErrOut, SuitesString(suites, "Select a test suite to run against the server:\n\n"))
		return nil, fmt.Errorf("specify a test suite to run, for example: %s run %s", filepath.Base(os.Args[0]), suites[0].Name)
	}
	if suite == nil && len(args) > 0 {
		for _, s := range suites {
			if s.Name == args[0] {
				suite = s
				break
			}
		}
	}
	if suite == nil {
		fmt.Fprintf(opt.ErrOut, SuitesString(suites, "Select a test suite to run against the server:\n\n"))
		return nil, fmt.Errorf("suite %q does not exist", args[0])
	}
	return suite, nil
}

func (opt *Options) Run(suite *TestSuite) error {
	if len(opt.Regex) > 0 {
		if err := filterWithRegex(suite, opt.Regex); err != nil {
			return err
		}
	}
	if opt.MatchFn != nil {
		original := suite.Matches
		suite.Matches = func(name string) bool {
			return original(name) && opt.MatchFn(name)
		}
	}

	syntheticEventTests := JUnitsForAllEvents{
		opt.SyntheticEventTests,
		suite.SyntheticEventTests,
	}

	tests, err := testsForSuite(config.GinkgoConfig)
	if err != nil {
		return err
	}

	// This ensures that tests in the identified paths do not run in parallel, because
	// the test suite reuses shared resources without considering whether another test
	// could be running at the same time. While these are technically [Serial], ginkgo
	// parallel mode provides this guarantee. Doing this for all suites would be too
	// slow.
	setTestExclusion(tests, func(suitePath string, t *testCase) bool {
		for _, name := range []string{
			"/k8s.io/kubernetes/test/e2e/apps/disruption.go",
		} {
			if strings.HasSuffix(suitePath, name) {
				return true
			}
		}
		return false
	})

	tests = suite.Filter(tests)
	if len(tests) == 0 {
		return fmt.Errorf("suite %q does not contain any tests", suite.Name)
	}

	count := opt.Count
	if count == 0 {
		count = suite.Count
	}

	start := time.Now()
	if opt.StartTime.IsZero() {
		opt.StartTime = start
	}

	if opt.PrintCommands {
		status := newTestStatus(opt.Out, true, len(tests), time.Minute, &monitor.Monitor{}, monitor.NewNoOpMonitor(), opt.AsEnv())
		newParallelTestQueue().Execute(context.Background(), tests, 1, status.OutputCommand)
		return nil
	}
	if opt.DryRun {
		for _, test := range sortedTests(tests) {
			fmt.Fprintf(opt.Out, "%q\n", test.name)
		}
		return nil
	}

	if len(opt.JUnitDir) > 0 {
		if _, err := os.Stat(opt.JUnitDir); err != nil {
			if !os.IsNotExist(err) {
				return fmt.Errorf("could not access --junit-dir: %v", err)
			}
			if err := os.MkdirAll(opt.JUnitDir, 0755); err != nil {
				return fmt.Errorf("could not create --junit-dir: %v", err)
			}
		}
	}

	parallelism := opt.Parallelism
	if parallelism == 0 {
		parallelism = suite.Parallelism
	}
	if parallelism == 0 {
		parallelism = 10
	}
	timeout := opt.Timeout
	if timeout == 0 {
		timeout = suite.TestTimeout
	}
	if timeout == 0 {
		timeout = 15 * time.Minute
	}

	ctx, cancelFn := context.WithCancel(context.Background())
	defer cancelFn()
	abortCh := make(chan os.Signal)
	go func() {
		<-abortCh
		fmt.Fprintf(opt.ErrOut, "Interrupted, terminating tests\n")
		cancelFn()
		sig := <-abortCh
		fmt.Fprintf(opt.ErrOut, "Interrupted twice, exiting (%s)\n", sig)
		switch sig {
		case syscall.SIGINT:
			os.Exit(130)
		default:
			os.Exit(0)
		}
	}()
	signal.Notify(abortCh, syscall.SIGINT, syscall.SIGTERM)

	m, err := monitor.Start(ctx)
	if err != nil {
		return err
	}
	// if we run a single test, always include success output
	includeSuccess := opt.IncludeSuccessOutput
	if len(tests) == 1 && count == 1 {
		includeSuccess = true
	}

	early, normal := splitTests(tests, func(t *testCase) bool {
		return strings.Contains(t.name, "[Early]")
	})

	late, normal := splitTests(normal, func(t *testCase) bool {
		return strings.Contains(t.name, "[Late]")
	})

	expectedTestCount := len(early) + len(late)
	if count != -1 {
		original := normal
		for i := 1; i < count; i++ {
			normal = append(normal, copyTests(original)...)
		}
	}
	expectedTestCount += len(normal)

	status := newTestStatus(opt.Out, includeSuccess, expectedTestCount, timeout, m, m, opt.AsEnv())
	testCtx := ctx
	if opt.FailFast {
		var cancelFn context.CancelFunc
		testCtx, cancelFn = context.WithCancel(testCtx)
		status.AfterTest(func(t *testCase) {
			if t.failed {
				cancelFn()
			}
		})
	}

	tests = nil

	// run our Early tests
	q := newParallelTestQueue()
	q.Execute(testCtx, early, parallelism, status.Run)
	tests = append(tests, early...)

	// repeat the normal suite until context cancel when in the forever loop
	for i := 0; (i < 1 || count == -1) && testCtx.Err() == nil; i++ {
		copied := copyTests(normal)
		q.Execute(testCtx, copied, parallelism, status.Run)
		tests = append(tests, copied...)
	}

	// run Late test suits after everything else
	q.Execute(testCtx, late, parallelism, status.Run)
	tests = append(tests, late...)

	// calculate the effective test set we ran, excluding any incompletes
	tests, _ = splitTests(tests, func(t *testCase) bool { return t.success || t.failed || t.skipped })

	duration := time.Now().Sub(start).Round(time.Second / 10)
	if duration > time.Minute {
		duration = duration.Round(time.Second)
	}

	pass, fail, skip, failing := summarizeTests(tests)

	// monitor the cluster while the tests are running and report any detected anomalies
	var syntheticTestResults []*JUnitTestCase
	var syntheticFailure bool
	timeSuffix := fmt.Sprintf("_%s", start.UTC().Format("20060102-150405"))
	events := m.EventIntervals(time.Time{}, time.Time{})
	if err = monitorserialization.EventsToFile(path.Join(os.Getenv("ARTIFACT_DIR"), fmt.Sprintf("e2e-events%s.json", timeSuffix)), events); err != nil {
		fmt.Fprintf(opt.Out, "Failed to write event file: %v\n", err)
	}
	if err = monitorserialization.EventsIntervalsToFile(path.Join(os.Getenv("ARTIFACT_DIR"), fmt.Sprintf("e2e-intervals%s.json", timeSuffix)), events); err != nil {
		fmt.Fprintf(opt.Out, "Failed to write event file: %v\n", err)
	}
	if eventIntervalsJSON, err := monitorserialization.EventsIntervalsToJSON(events); err == nil {
		e2eChartTemplate := testdata.MustAsset("e2echart/e2e-chart-template.html")
		e2eChartHTML := bytes.ReplaceAll(e2eChartTemplate, []byte("EVENT_INTERVAL_JSON_GOES_HERE"), eventIntervalsJSON)
		e2eChartHTMLPath := path.Join(os.Getenv("ARTIFACT_DIR"), fmt.Sprintf("e2e-intervals%s.html", timeSuffix))
		if err := ioutil.WriteFile(e2eChartHTMLPath, e2eChartHTML, 0644); err != nil {
			fmt.Fprintf(opt.Out, "Failed to write event html: %v\n", err)
		}
	} else {
		fmt.Fprintf(opt.Out, "Failed to write event html: %v\n", err)
	}

	if len(events) > 0 {
		eventsForTests := createEventsForTests(tests)

		var buf *bytes.Buffer
		syntheticTestResults, buf, _ = createSyntheticTestsFromMonitor(m, eventsForTests, duration)
		testCases := syntheticEventTests.JUnitsForEvents(events, duration)
		syntheticTestResults = append(syntheticTestResults, testCases...)

		if len(syntheticTestResults) > 0 {
			// mark any failures by name
			failing, flaky := sets.NewString(), sets.NewString()
			for _, test := range syntheticTestResults {
				if test.FailureOutput != nil {
					failing.Insert(test.Name)
				}
			}
			// if a test has both a pass and a failure, flag it
			// as a flake
			for _, test := range syntheticTestResults {
				if test.FailureOutput == nil {
					if failing.Has(test.Name) {
						flaky.Insert(test.Name)
					}
				}
			}
			failing = failing.Difference(flaky)
			if failing.Len() > 0 {
				fmt.Fprintf(buf, "Failing invariants:\n\n%s\n\n", strings.Join(failing.List(), "\n"))
				syntheticFailure = true
			}
			if flaky.Len() > 0 {
				fmt.Fprintf(buf, "Flaky invariants:\n\n%s\n\n", strings.Join(flaky.List(), "\n"))
			}
		}

		opt.Out.Write(buf.Bytes())
	}

	// attempt to retry failures to do flake detection
	if fail > 0 && fail <= suite.MaximumAllowedFlakes {
		var retries []*testCase
		for _, test := range failing {
			retry := test.Retry()
			retries = append(retries, retry)
			tests = append(tests, retry)
			if len(retries) > suite.MaximumAllowedFlakes {
				break
			}
		}

		q := newParallelTestQueue()
		status := newTestStatus(ioutil.Discard, opt.IncludeSuccessOutput, len(retries), timeout, m, m, opt.AsEnv())
		q.Execute(testCtx, retries, parallelism, status.Run)
		var flaky []string
		var repeatFailures []*testCase
		for _, test := range retries {
			if test.success {
				flaky = append(flaky, test.name)
			} else {
				repeatFailures = append(repeatFailures, test)
			}
		}
		if len(flaky) > 0 {
			failing = repeatFailures
			sort.Strings(flaky)
			fmt.Fprintf(opt.Out, "Flaky tests:\n\n%s\n\n", strings.Join(flaky, "\n"))
		}
	}

	// report the outcome of the test
	if len(failing) > 0 {
		names := sets.NewString(testNames(failing)...).List()
		fmt.Fprintf(opt.Out, "Failing tests:\n\n%s\n\n", strings.Join(names, "\n"))
	}

	if len(opt.JUnitDir) > 0 {
		if err := writeJUnitReport("junit_e2e", "openshift-tests", tests, opt.JUnitDir, duration, opt.ErrOut, syntheticTestResults...); err != nil {
			fmt.Fprintf(opt.Out, "error: Unable to write e2e JUnit results: %v", err)
		}
	}

	if fail > 0 {
		if len(failing) > 0 || suite.MaximumAllowedFlakes == 0 {
			return fmt.Errorf("%d fail, %d pass, %d skip (%s)", fail, pass, skip, duration)
		}
		fmt.Fprintf(opt.Out, "%d flakes detected, suite allows passing with only flakes\n\n", fail)
	}

	if syntheticFailure {
		return fmt.Errorf("failed because an invariant was violated, %d pass, %d skip (%s)\n", pass, skip, duration)
	}

	fmt.Fprintf(opt.Out, "%d pass, %d skip (%s)\n", pass, skip, duration)
	return ctx.Err()
}
<|endoftext|>


<commit_before>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017 Red Hat, Inc.
 *
 */

package watch

import (
	"strings"
	"time"

	"github.com/jeevatkm/go-model"
	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/record"
	"k8s.io/client-go/util/workqueue"

	kubev1 "kubevirt.io/kubevirt/pkg/api/v1"
	"kubevirt.io/kubevirt/pkg/controller"
	"kubevirt.io/kubevirt/pkg/kubecli"
	"kubevirt.io/kubevirt/pkg/log"
	"kubevirt.io/kubevirt/pkg/virt-controller/services"
)

func NewVMController(restClient *rest.RESTClient, vmService services.VMService, queue workqueue.RateLimitingInterface, vmCache cache.Store, vmInformer cache.SharedIndexInformer, podInformer cache.SharedIndexInformer, recorder record.EventRecorder, clientset kubecli.KubevirtClient) *VMController {
	return &VMController{
		restClient:  restClient,
		vmService:   vmService,
		queue:       queue,
		store:       vmCache,
		vmInformer:  vmInformer,
		podInformer: podInformer,
		recorder:    recorder,
		clientset:   clientset,
	}
}

type VMController struct {
	restClient  *rest.RESTClient
	vmService   services.VMService
	clientset   kubecli.KubevirtClient
	queue       workqueue.RateLimitingInterface
	store       cache.Store
	vmInformer  cache.SharedIndexInformer
	podInformer cache.SharedIndexInformer
	recorder    record.EventRecorder
}

func (c *VMController) Run(threadiness int, stopCh chan struct{}) {
	defer controller.HandlePanic()
	defer c.queue.ShutDown()
	log.Log.Info("Starting controller.")

	// Wait for cache sync before we start the pod controller
	cache.WaitForCacheSync(stopCh, c.vmInformer.HasSynced, c.podInformer.HasSynced)

	// Start the actual work
	for i := 0; i < threadiness; i++ {
		go wait.Until(c.runWorker, time.Second, stopCh)
	}

	<-stopCh
	log.Log.Info("Stopping controller.")
}

func (c *VMController) runWorker() {
	for c.Execute() {
	}
}

func (c *VMController) Execute() bool {
	key, quit := c.queue.Get()
	if quit {
		return false
	}
	defer c.queue.Done(key)
	err := c.execute(key.(string))

	if err != nil {
		log.Log.Reason(err).Infof("reenqueuing VM %v", key)
		c.queue.AddRateLimited(key)
	} else {
		log.Log.V(4).Infof("processed VM %v", key)
		c.queue.Forget(key)
	}
	return true
}

func (c *VMController) execute(key string) error {

	// Fetch the latest Vm state from cache
	obj, exists, err := c.store.GetByKey(key)

	if err != nil {
		return err
	}

	// Retrieve the VM
	var vm *kubev1.VirtualMachine
	if !exists {
		namespace, name, err := cache.SplitMetaNamespaceKey(key)
		if err != nil {
			return err
		}
		vm = kubev1.NewVMReferenceFromNameWithNS(namespace, name)
	} else {
		vm = obj.(*kubev1.VirtualMachine)
	}
	logger := log.Log.Object(vm)

	if !exists {
		// Delete VM Pods
		err := c.vmService.DeleteVMPod(vm)
		if err != nil {
			logger.Reason(err).Error("Deleting VM target Pod failed.")
			return err
		}
		logger.Info("Deleting VM target Pod succeeded.")
		return nil
	}

	switch vm.Status.Phase {
	case kubev1.VmPhaseUnset, kubev1.Pending:
		// Schedule the VM

		// Deep copy the object, so that we can safely manipulate it
		vmCopy := kubev1.VirtualMachine{}
		model.Copy(&vmCopy, vm)
		logger = log.Log.Object(&vmCopy)

		// Check if there are already outdated VM Pods
		pods, err := c.vmService.GetRunningVMPods(&vmCopy)
		if err != nil {
			logger.Reason(err).Error("Fetching VM pods failed.")
			return err
		}

		// If there are already pods, delete them before continuing ...
		if len(pods.Items) > 0 {
			logger.Error("VM Pods do already exist, will clean up before continuing.")
			if err := c.vmService.DeleteVMPod(&vmCopy); err != nil {
				logger.Reason(err).Error("Deleting VM pods failed.")
				return err
			}

			log.Log.V(4).Infof("reenqueuing VM key %s because still waiting on previous pod to terminate", key)
			c.queue.AddAfter(key, 3*time.Second)
			return nil
		}

		// Defaulting and setting constants
		// TODO move defaulting to virt-api
		// TODO move constants to virt-handler and remove from the spec
		if vmCopy.Spec.Domain == nil {
			spec := kubev1.NewMinimalDomainSpec()
			vmCopy.Spec.Domain = spec
		}

		// TODO when we move this to virt-api, we have to block that they are set on POST or changed on PUT
		graphics := vmCopy.Spec.Domain.Devices.Graphics
		for i, _ := range graphics {
			if strings.ToLower(graphics[i].Type) == "spice" {
				graphics[i].Port = int32(-1)
				graphics[i].Listen = kubev1.Listen{
					Address: "0.0.0.0",
					Type:    "address",
				}

			}
		}

		// Create a Pod which will be the VM destination
		if err := c.vmService.StartVMPod(&vmCopy); err != nil {
			logger.Reason(err).Error("Defining a target pod for the VM failed.")
			return err
		}

		// Mark the VM as "initialized". After the created Pod above is scheduled by
		// kubernetes, virt-handler can take over.
		vmCopy.Status.Phase = kubev1.Scheduling
		if err := c.restClient.Put().Resource("virtualmachines").Body(&vmCopy).Name(vmCopy.ObjectMeta.Name).Namespace(vmCopy.ObjectMeta.Namespace).Do().Error(); err != nil {
			logger.Reason(err).Error("Updating the VM state to 'Scheduling' failed.")
			return err
		}
		logger.Info("Handing over the VM to the scheduler succeeded.")
	case kubev1.Scheduling:
		// Target Pod for the VM was already created, check if it is  running and update the VM to Scheduled

		// Deep copy the object, so that we can safely manipulate it
		vmCopy := kubev1.VirtualMachine{}
		model.Copy(&vmCopy, vm)
		logger = log.Log.Object(&vmCopy)

		pods, err := c.vmService.GetRunningVMPods(&vmCopy)
		if err != nil {
			logger.Reason(err).Error("Fetching VM pods failed.")
			return err
		}

		//TODO, we can improve the pod checks here, for now they are as good as before the refactoring
		// So far, no running Pod found, we will sooner or later get a started event.
		// If not, something is wrong and the VM, stay stuck in the Scheduling phase
		if len(pods.Items) == 0 {
			logger.V(3).Info("No VM target pod in running state found.")
			return nil
		}

		if len(pods.Items) > 1 {
			logger.V(3).Error("More than one VM target pods found.")

			wasDeleted := false
			for _, pod := range pods.Items {
				if pod.GetObjectMeta().GetDeletionTimestamp() != nil {
					wasDeleted = true
					break
				}
			}

			if wasDeleted {
				log.Log.V(4).Infof("reenqueuing VM key %s because still waiting on pod deletion cleanup", key)
				c.queue.AddAfter(key, 3*time.Second)
			}

			return nil
		}

		// Pod is not yet running
		if pods.Items[0].Status.Phase != k8sv1.PodRunning {
			return nil
		}

		if verifyReadiness(&pods.Items[0]) == false {
			logger.V(2).Info("Waiting on all virt-launcher containers to be marked ready")
			return nil
		}

		// VM got scheduled
		vmCopy.Status.Phase = kubev1.Scheduled

		// FIXME we store this in the metadata since field selectors are currently not working for TPRs
		if vmCopy.GetObjectMeta().GetLabels() == nil {
			vmCopy.ObjectMeta.Labels = map[string]string{}
		}
		vmCopy.ObjectMeta.Labels[kubev1.NodeNameLabel] = pods.Items[0].Spec.NodeName
		vmCopy.Status.NodeName = pods.Items[0].Spec.NodeName
		if _, err := c.vmService.PutVm(&vmCopy); err != nil {
			logger.Reason(err).Error("Updating the VM state to 'Scheduled' failed.")
			return err
		}
		logger.Infof("VM successfully scheduled to %s.", vmCopy.Status.NodeName)
	case kubev1.Failed, kubev1.Succeeded:
		err := c.vmService.DeleteVMPod(vm)
		if err != nil {
			logger.Reason(err).Error("Deleting VM target Pod failed.")
			return err
		}
		logger.Info("Deleted VM target Pod for VM in finalized state.")
	}
	return nil
}

func verifyReadiness(pod *k8sv1.Pod) bool {
	for _, containerStatus := range pod.Status.ContainerStatuses {
		if containerStatus.Ready == false {
			return false
		}
	}

	return true
}

func vmLabelHandler(vmQueue workqueue.RateLimitingInterface) func(obj interface{}) {
	return func(obj interface{}) {
		phase := obj.(*k8sv1.Pod).Status.Phase
		namespace := obj.(*k8sv1.Pod).ObjectMeta.Namespace
		appLabel, hasAppLabel := obj.(*k8sv1.Pod).ObjectMeta.Labels[kubev1.AppLabel]
		domainLabel, hasDomainLabel := obj.(*k8sv1.Pod).ObjectMeta.Labels[kubev1.DomainLabel]
		_, hasMigrationLabel := obj.(*k8sv1.Pod).ObjectMeta.Labels[kubev1.MigrationLabel]

		if phase != k8sv1.PodRunning {
			// Filter out non running pods from Queue
			return
		} else if hasMigrationLabel {
			// Filter out migration target pods
			return
		} else if hasDomainLabel == false || hasAppLabel == false {
			// missing required labels
			return
		} else if appLabel != "virt-launcher" {
			// ensure we're looking just for virt-launcher pods
			return
		}
		vmQueue.Add(namespace + "/" + domainLabel)
	}
}
<commit_msg>virt-controller needs to know about deleted pods<commit_after>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017 Red Hat, Inc.
 *
 */

package watch

import (
	"strings"
	"time"

	"github.com/jeevatkm/go-model"
	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/record"
	"k8s.io/client-go/util/workqueue"

	kubev1 "kubevirt.io/kubevirt/pkg/api/v1"
	"kubevirt.io/kubevirt/pkg/controller"
	"kubevirt.io/kubevirt/pkg/kubecli"
	"kubevirt.io/kubevirt/pkg/log"
	"kubevirt.io/kubevirt/pkg/virt-controller/services"
)

func NewVMController(restClient *rest.RESTClient, vmService services.VMService, queue workqueue.RateLimitingInterface, vmCache cache.Store, vmInformer cache.SharedIndexInformer, podInformer cache.SharedIndexInformer, recorder record.EventRecorder, clientset kubecli.KubevirtClient) *VMController {
	return &VMController{
		restClient:  restClient,
		vmService:   vmService,
		queue:       queue,
		store:       vmCache,
		vmInformer:  vmInformer,
		podInformer: podInformer,
		recorder:    recorder,
		clientset:   clientset,
	}
}

type VMController struct {
	restClient  *rest.RESTClient
	vmService   services.VMService
	clientset   kubecli.KubevirtClient
	queue       workqueue.RateLimitingInterface
	store       cache.Store
	vmInformer  cache.SharedIndexInformer
	podInformer cache.SharedIndexInformer
	recorder    record.EventRecorder
}

func (c *VMController) Run(threadiness int, stopCh chan struct{}) {
	defer controller.HandlePanic()
	defer c.queue.ShutDown()
	log.Log.Info("Starting controller.")

	// Wait for cache sync before we start the pod controller
	cache.WaitForCacheSync(stopCh, c.vmInformer.HasSynced, c.podInformer.HasSynced)

	// Start the actual work
	for i := 0; i < threadiness; i++ {
		go wait.Until(c.runWorker, time.Second, stopCh)
	}

	<-stopCh
	log.Log.Info("Stopping controller.")
}

func (c *VMController) runWorker() {
	for c.Execute() {
	}
}

func (c *VMController) Execute() bool {
	key, quit := c.queue.Get()
	if quit {
		return false
	}
	defer c.queue.Done(key)
	err := c.execute(key.(string))

	if err != nil {
		log.Log.Reason(err).Infof("reenqueuing VM %v", key)
		c.queue.AddRateLimited(key)
	} else {
		log.Log.V(4).Infof("processed VM %v", key)
		c.queue.Forget(key)
	}
	return true
}

func (c *VMController) execute(key string) error {

	// Fetch the latest Vm state from cache
	obj, exists, err := c.store.GetByKey(key)

	if err != nil {
		return err
	}

	// Retrieve the VM
	var vm *kubev1.VirtualMachine
	if !exists {
		namespace, name, err := cache.SplitMetaNamespaceKey(key)
		if err != nil {
			return err
		}
		vm = kubev1.NewVMReferenceFromNameWithNS(namespace, name)
	} else {
		vm = obj.(*kubev1.VirtualMachine)
	}
	logger := log.Log.Object(vm)

	if !exists {
		// Delete VM Pods
		err := c.vmService.DeleteVMPod(vm)
		if err != nil {
			logger.Reason(err).Error("Deleting VM target Pod failed.")
			return err
		}
		logger.Info("Deleting VM target Pod succeeded.")
		return nil
	}

	switch vm.Status.Phase {
	case kubev1.VmPhaseUnset, kubev1.Pending:
		// Schedule the VM

		// Deep copy the object, so that we can safely manipulate it
		vmCopy := kubev1.VirtualMachine{}
		model.Copy(&vmCopy, vm)
		logger = log.Log.Object(&vmCopy)

		// Check if there are already outdated VM Pods
		pods, err := c.vmService.GetRunningVMPods(&vmCopy)
		if err != nil {
			logger.Reason(err).Error("Fetching VM pods failed.")
			return err
		}

		// If there are already pods, delete them before continuing ...
		if len(pods.Items) > 0 {
			logger.Error("VM Pods do already exist, will clean up before continuing.")
			if err := c.vmService.DeleteVMPod(&vmCopy); err != nil {
				logger.Reason(err).Error("Deleting VM pods failed.")
				return err
			}
			// the pod informer will reenqueue the key as a result of it being deleted.
			return nil
		}

		// Defaulting and setting constants
		// TODO move defaulting to virt-api
		// TODO move constants to virt-handler and remove from the spec
		if vmCopy.Spec.Domain == nil {
			spec := kubev1.NewMinimalDomainSpec()
			vmCopy.Spec.Domain = spec
		}

		// TODO when we move this to virt-api, we have to block that they are set on POST or changed on PUT
		graphics := vmCopy.Spec.Domain.Devices.Graphics
		for i, _ := range graphics {
			if strings.ToLower(graphics[i].Type) == "spice" {
				graphics[i].Port = int32(-1)
				graphics[i].Listen = kubev1.Listen{
					Address: "0.0.0.0",
					Type:    "address",
				}

			}
		}

		// Create a Pod which will be the VM destination
		if err := c.vmService.StartVMPod(&vmCopy); err != nil {
			logger.Reason(err).Error("Defining a target pod for the VM failed.")
			return err
		}

		// Mark the VM as "initialized". After the created Pod above is scheduled by
		// kubernetes, virt-handler can take over.
		vmCopy.Status.Phase = kubev1.Scheduling
		if err := c.restClient.Put().Resource("virtualmachines").Body(&vmCopy).Name(vmCopy.ObjectMeta.Name).Namespace(vmCopy.ObjectMeta.Namespace).Do().Error(); err != nil {
			logger.Reason(err).Error("Updating the VM state to 'Scheduling' failed.")
			return err
		}
		logger.Info("Handing over the VM to the scheduler succeeded.")
	case kubev1.Scheduling:
		// Target Pod for the VM was already created, check if it is  running and update the VM to Scheduled

		// Deep copy the object, so that we can safely manipulate it
		vmCopy := kubev1.VirtualMachine{}
		model.Copy(&vmCopy, vm)
		logger = log.Log.Object(&vmCopy)

		pods, err := c.vmService.GetRunningVMPods(&vmCopy)
		if err != nil {
			logger.Reason(err).Error("Fetching VM pods failed.")
			return err
		}

		//TODO, we can improve the pod checks here, for now they are as good as before the refactoring
		// So far, no running Pod found, we will sooner or later get a started event.
		// If not, something is wrong and the VM, stay stuck in the Scheduling phase
		if len(pods.Items) == 0 {
			logger.V(3).Info("No VM target pod in running state found.")
			return nil
		}

		// If this occurs, the podinformer should reenqueue the key
		// if one of these pods terminates. This will let virt-controller continue
		// processing the VM.
		if len(pods.Items) > 1 {
			logger.V(3).Error("More than one VM target pods found.")
			return nil
		}

		// Pod is not yet running
		if pods.Items[0].Status.Phase != k8sv1.PodRunning {
			return nil
		}

		if verifyReadiness(&pods.Items[0]) == false {
			logger.V(2).Info("Waiting on all virt-launcher containers to be marked ready")
			return nil
		}

		// VM got scheduled
		vmCopy.Status.Phase = kubev1.Scheduled

		// FIXME we store this in the metadata since field selectors are currently not working for TPRs
		if vmCopy.GetObjectMeta().GetLabels() == nil {
			vmCopy.ObjectMeta.Labels = map[string]string{}
		}
		vmCopy.ObjectMeta.Labels[kubev1.NodeNameLabel] = pods.Items[0].Spec.NodeName
		vmCopy.Status.NodeName = pods.Items[0].Spec.NodeName
		if _, err := c.vmService.PutVm(&vmCopy); err != nil {
			logger.Reason(err).Error("Updating the VM state to 'Scheduled' failed.")
			return err
		}
		logger.Infof("VM successfully scheduled to %s.", vmCopy.Status.NodeName)
	case kubev1.Failed, kubev1.Succeeded:
		err := c.vmService.DeleteVMPod(vm)
		if err != nil {
			logger.Reason(err).Error("Deleting VM target Pod failed.")
			return err
		}
		logger.Info("Deleted VM target Pod for VM in finalized state.")
	}
	return nil
}

func verifyReadiness(pod *k8sv1.Pod) bool {
	for _, containerStatus := range pod.Status.ContainerStatuses {
		if containerStatus.Ready == false {
			return false
		}
	}

	return true
}

func vmLabelHandler(vmQueue workqueue.RateLimitingInterface) func(obj interface{}) {
	return func(obj interface{}) {
		phase := obj.(*k8sv1.Pod).Status.Phase
		namespace := obj.(*k8sv1.Pod).ObjectMeta.Namespace
		appLabel, hasAppLabel := obj.(*k8sv1.Pod).ObjectMeta.Labels[kubev1.AppLabel]
		domainLabel, hasDomainLabel := obj.(*k8sv1.Pod).ObjectMeta.Labels[kubev1.DomainLabel]
		_, hasMigrationLabel := obj.(*k8sv1.Pod).ObjectMeta.Labels[kubev1.MigrationLabel]

		deleted := false
		if obj.(*k8sv1.Pod).GetObjectMeta().GetDeletionTimestamp() != nil {
			deleted = true
		}

		if hasMigrationLabel {
			// Filter out migration target pods
			return
		} else if hasDomainLabel == false || hasAppLabel == false {
			// missing required labels
			return
		} else if appLabel != "virt-launcher" {
			// ensure we're looking just for virt-launcher pods
			return
		} else if phase != k8sv1.PodRunning && deleted == false {
			// Filter out non running pods from Queue that aren't deleted
			return
		}
		vmQueue.Add(namespace + "/" + domainLabel)
	}
}
<|endoftext|>


<commit_before>package dbus

import (
	"bytes"
	"encoding/binary"
	"io"
	"reflect"
	"unicode/utf8"
)

// An encoder encodes values to the D-Bus wire format.
type encoder struct {
	out   io.Writer
	order binary.ByteOrder
	pos   int
}

// NewEncoder returns a new encoder that writes to out in the given byte order.
func newEncoder(out io.Writer, order binary.ByteOrder) *encoder {
	return newEncoderAtOffset(out, 0, order)
}

// newEncoderAtOffset returns a new encoder that writes to out in the given
// byte order. Specify the offset to initialize pos for proper alignment
// computation.
func newEncoderAtOffset(out io.Writer, offset int, order binary.ByteOrder) *encoder {
	enc := new(encoder)
	enc.out = out
	enc.order = order
	enc.pos = offset
	return enc
}

// Aligns the next output to be on a multiple of n. Panics on write errors.
func (enc *encoder) align(n int) {
	pad := enc.padding(0, n)
	if pad > 0 {
		empty := make([]byte, pad)
		if _, err := enc.out.Write(empty); err != nil {
			panic(err)
		}
		enc.pos += pad
	}
}

// pad returns the number of bytes of padding, based on current position and additional offset.
// and alignment.
func (enc *encoder) padding(offset, algn int) int {
	abs := enc.pos + offset
	if abs%algn != 0 {
		newabs := (abs + algn - 1) & ^(algn - 1)
		return newabs - abs
	}
	return 0
}

// Calls binary.Write(enc.out, enc.order, v) and panics on write errors.
func (enc *encoder) binwrite(v interface{}) {
	if err := binary.Write(enc.out, enc.order, v); err != nil {
		panic(err)
	}
}

// Encode encodes the given values to the underlying reader. All written values
// are aligned properly as required by the D-Bus spec.
func (enc *encoder) Encode(vs ...interface{}) (err error) {
	defer func() {
		err, _ = recover().(error)
	}()
	for _, v := range vs {
		enc.encode(reflect.ValueOf(v), 0)
	}
	return nil
}

// encode encodes the given value to the writer and panics on error. depth holds
// the depth of the container nesting.
func (enc *encoder) encode(v reflect.Value, depth int) {
	enc.align(alignment(v.Type()))
	switch v.Kind() {
	case reflect.Uint8:
		var b [1]byte
		b[0] = byte(v.Uint())
		if _, err := enc.out.Write(b[:]); err != nil {
			panic(err)
		}
		enc.pos++
	case reflect.Bool:
		if v.Bool() {
			enc.encode(reflect.ValueOf(uint32(1)), depth)
		} else {
			enc.encode(reflect.ValueOf(uint32(0)), depth)
		}
	case reflect.Int16:
		enc.binwrite(int16(v.Int()))
		enc.pos += 2
	case reflect.Uint16:
		enc.binwrite(uint16(v.Uint()))
		enc.pos += 2
	case reflect.Int, reflect.Int32:
		enc.binwrite(int32(v.Int()))
		enc.pos += 4
	case reflect.Uint, reflect.Uint32:
		enc.binwrite(uint32(v.Uint()))
		enc.pos += 4
	case reflect.Int64:
		enc.binwrite(v.Int())
		enc.pos += 8
	case reflect.Uint64:
		enc.binwrite(v.Uint())
		enc.pos += 8
	case reflect.Float64:
		enc.binwrite(v.Float())
		enc.pos += 8
	case reflect.String:
		if !utf8.Valid([]byte(v.String())) {
			panic(FormatError("input has a not-utf8 char in string"))
		}
		enc.encode(reflect.ValueOf(uint32(len(v.String()))), depth)
		b := make([]byte, v.Len()+1)
		copy(b, v.String())
		b[len(b)-1] = 0
		n, err := enc.out.Write(b)
		if err != nil {
			panic(err)
		}
		enc.pos += n
	case reflect.Ptr:
		enc.encode(v.Elem(), depth)
	case reflect.Slice, reflect.Array:
		if depth >= 64 {
			panic(FormatError("input exceeds container depth limit"))
		}
		// Lookahead offset: 4 bytes for uint32 length (with alignment),
		// plus alignment for elements.
		n := enc.padding(0, 4) + 4
		offset := enc.pos + n + enc.padding(n, alignment(v.Type().Elem()))

		var buf bytes.Buffer
		bufenc := newEncoderAtOffset(&buf, offset, enc.order)

		for i := 0; i < v.Len(); i++ {
			bufenc.encode(v.Index(i), depth+1)
		}

		if buf.Len() > 1<<26 {
			panic(FormatError("input exceeds array size limitation"))
		}

		enc.encode(reflect.ValueOf(uint32(buf.Len())), depth)
		length := buf.Len()
		enc.align(alignment(v.Type().Elem()))
		if _, err := buf.WriteTo(enc.out); err != nil {
			panic(err)
		}
		enc.pos += length
	case reflect.Struct:
		if depth >= 64 && v.Type() != signatureType {
			panic(FormatError("input exceeds container depth limit"))
		}
		switch t := v.Type(); t {
		case signatureType:
			str := v.Field(0)
			enc.encode(reflect.ValueOf(byte(str.Len())), depth+1)
			b := make([]byte, str.Len()+1)
			copy(b, str.String())
			b[len(b)-1] = 0
			n, err := enc.out.Write(b)
			if err != nil {
				panic(err)
			}
			enc.pos += n
		case variantType:
			variant := v.Interface().(Variant)
			enc.encode(reflect.ValueOf(variant.sig), depth+1)
			enc.encode(reflect.ValueOf(variant.value), depth+1)
		default:
			for i := 0; i < v.Type().NumField(); i++ {
				field := t.Field(i)
				if field.PkgPath == "" && field.Tag.Get("dbus") != "-" {
					enc.encode(v.Field(i), depth+1)
				}
			}
		}
	case reflect.Map:
		// Maps are arrays of structures, so they actually increase the depth by
		// 2.
		if depth >= 63 {
			panic(FormatError("input exceeds container depth limit"))
		}
		if !isKeyType(v.Type().Key()) {
			panic(InvalidTypeError{v.Type()})
		}
		keys := v.MapKeys()
		// Lookahead offset: 4 bytes for uint32 length (with alignment),
		// plus 8-byte alignment
		n := enc.padding(0, 4) + 4
		offset := enc.pos + n + enc.padding(n, 8)

		var buf bytes.Buffer
		bufenc := newEncoderAtOffset(&buf, offset, enc.order)
		for _, k := range keys {
			bufenc.align(8)
			bufenc.encode(k, depth+2)
			bufenc.encode(v.MapIndex(k), depth+2)
		}
		enc.encode(reflect.ValueOf(uint32(buf.Len())), depth)
		length := buf.Len()
		enc.align(8)
		if _, err := buf.WriteTo(enc.out); err != nil {
			panic(err)
		}
		enc.pos += length
	case reflect.Interface:
		enc.encode(reflect.ValueOf(MakeVariant(v.Interface())), depth)
	default:
		panic(InvalidTypeError{v.Type()})
	}
}
<commit_msg>Refactor depth check in encoder<commit_after>package dbus

import (
	"bytes"
	"encoding/binary"
	"io"
	"reflect"
	"unicode/utf8"
)

// An encoder encodes values to the D-Bus wire format.
type encoder struct {
	out   io.Writer
	order binary.ByteOrder
	pos   int
}

// NewEncoder returns a new encoder that writes to out in the given byte order.
func newEncoder(out io.Writer, order binary.ByteOrder) *encoder {
	return newEncoderAtOffset(out, 0, order)
}

// newEncoderAtOffset returns a new encoder that writes to out in the given
// byte order. Specify the offset to initialize pos for proper alignment
// computation.
func newEncoderAtOffset(out io.Writer, offset int, order binary.ByteOrder) *encoder {
	enc := new(encoder)
	enc.out = out
	enc.order = order
	enc.pos = offset
	return enc
}

// Aligns the next output to be on a multiple of n. Panics on write errors.
func (enc *encoder) align(n int) {
	pad := enc.padding(0, n)
	if pad > 0 {
		empty := make([]byte, pad)
		if _, err := enc.out.Write(empty); err != nil {
			panic(err)
		}
		enc.pos += pad
	}
}

// pad returns the number of bytes of padding, based on current position and additional offset.
// and alignment.
func (enc *encoder) padding(offset, algn int) int {
	abs := enc.pos + offset
	if abs%algn != 0 {
		newabs := (abs + algn - 1) & ^(algn - 1)
		return newabs - abs
	}
	return 0
}

// Calls binary.Write(enc.out, enc.order, v) and panics on write errors.
func (enc *encoder) binwrite(v interface{}) {
	if err := binary.Write(enc.out, enc.order, v); err != nil {
		panic(err)
	}
}

// Encode encodes the given values to the underlying reader. All written values
// are aligned properly as required by the D-Bus spec.
func (enc *encoder) Encode(vs ...interface{}) (err error) {
	defer func() {
		err, _ = recover().(error)
	}()
	for _, v := range vs {
		enc.encode(reflect.ValueOf(v), 0)
	}
	return nil
}

// encode encodes the given value to the writer and panics on error. depth holds
// the depth of the container nesting.
func (enc *encoder) encode(v reflect.Value, depth int) {
	if depth > 64 {
		panic(FormatError("input exceeds depth limitation"))
	}
	enc.align(alignment(v.Type()))
	switch v.Kind() {
	case reflect.Uint8:
		var b [1]byte
		b[0] = byte(v.Uint())
		if _, err := enc.out.Write(b[:]); err != nil {
			panic(err)
		}
		enc.pos++
	case reflect.Bool:
		if v.Bool() {
			enc.encode(reflect.ValueOf(uint32(1)), depth)
		} else {
			enc.encode(reflect.ValueOf(uint32(0)), depth)
		}
	case reflect.Int16:
		enc.binwrite(int16(v.Int()))
		enc.pos += 2
	case reflect.Uint16:
		enc.binwrite(uint16(v.Uint()))
		enc.pos += 2
	case reflect.Int, reflect.Int32:
		enc.binwrite(int32(v.Int()))
		enc.pos += 4
	case reflect.Uint, reflect.Uint32:
		enc.binwrite(uint32(v.Uint()))
		enc.pos += 4
	case reflect.Int64:
		enc.binwrite(v.Int())
		enc.pos += 8
	case reflect.Uint64:
		enc.binwrite(v.Uint())
		enc.pos += 8
	case reflect.Float64:
		enc.binwrite(v.Float())
		enc.pos += 8
	case reflect.String:
		if !utf8.Valid([]byte(v.String())) {
			panic(FormatError("input has a not-utf8 char in string"))
		}
		enc.encode(reflect.ValueOf(uint32(len(v.String()))), depth)
		b := make([]byte, v.Len()+1)
		copy(b, v.String())
		b[len(b)-1] = 0
		n, err := enc.out.Write(b)
		if err != nil {
			panic(err)
		}
		enc.pos += n
	case reflect.Ptr:
		enc.encode(v.Elem(), depth)
	case reflect.Slice, reflect.Array:
		// Lookahead offset: 4 bytes for uint32 length (with alignment),
		// plus alignment for elements.
		n := enc.padding(0, 4) + 4
		offset := enc.pos + n + enc.padding(n, alignment(v.Type().Elem()))

		var buf bytes.Buffer
		bufenc := newEncoderAtOffset(&buf, offset, enc.order)

		for i := 0; i < v.Len(); i++ {
			bufenc.encode(v.Index(i), depth+1)
		}

		if buf.Len() > 1<<26 {
			panic(FormatError("input exceeds array size limitation"))
		}

		enc.encode(reflect.ValueOf(uint32(buf.Len())), depth)
		length := buf.Len()
		enc.align(alignment(v.Type().Elem()))
		if _, err := buf.WriteTo(enc.out); err != nil {
			panic(err)
		}
		enc.pos += length
	case reflect.Struct:
		switch t := v.Type(); t {
		case signatureType:
			str := v.Field(0)
			enc.encode(reflect.ValueOf(byte(str.Len())), depth)
			b := make([]byte, str.Len()+1)
			copy(b, str.String())
			b[len(b)-1] = 0
			n, err := enc.out.Write(b)
			if err != nil {
				panic(err)
			}
			enc.pos += n
		case variantType:
			variant := v.Interface().(Variant)
			enc.encode(reflect.ValueOf(variant.sig), depth+1)
			enc.encode(reflect.ValueOf(variant.value), depth+1)
		default:
			for i := 0; i < v.Type().NumField(); i++ {
				field := t.Field(i)
				if field.PkgPath == "" && field.Tag.Get("dbus") != "-" {
					enc.encode(v.Field(i), depth+1)
				}
			}
		}
	case reflect.Map:
		// Maps are arrays of structures, so they actually increase the depth by
		// 2.
		if !isKeyType(v.Type().Key()) {
			panic(InvalidTypeError{v.Type()})
		}
		keys := v.MapKeys()
		// Lookahead offset: 4 bytes for uint32 length (with alignment),
		// plus 8-byte alignment
		n := enc.padding(0, 4) + 4
		offset := enc.pos + n + enc.padding(n, 8)

		var buf bytes.Buffer
		bufenc := newEncoderAtOffset(&buf, offset, enc.order)
		for _, k := range keys {
			bufenc.align(8)
			bufenc.encode(k, depth+2)
			bufenc.encode(v.MapIndex(k), depth+2)
		}
		enc.encode(reflect.ValueOf(uint32(buf.Len())), depth)
		length := buf.Len()
		enc.align(8)
		if _, err := buf.WriteTo(enc.out); err != nil {
			panic(err)
		}
		enc.pos += length
	case reflect.Interface:
		enc.encode(reflect.ValueOf(MakeVariant(v.Interface())), depth)
	default:
		panic(InvalidTypeError{v.Type()})
	}
}
<|endoftext|>


<commit_before>package giraffe

import (
	"encoding/json"
	"fmt"
	"net/http"
)

const (
	// ContentBinary header value for binary data.
	ContentBinary = "application/octet-stream"
	// ContentJSON header value for JSON data.
	ContentJSON = "application/json"
	// ContentJSONP header value for JSONP data.
	ContentJSONP = "application/javascript"
	// ContentText header value for Text data.
	ContentText = "text/plain"
	// ContentType header constant.
	ContentType = "Content-Type"
	// ContentDefaultCharset default character encoding.
	ContentDefaultCharset = "UTF-8"
)

// ContentTypeWithCharset returns the contentype with the default charset
func ContentTypeWithCharset(contentType string) string {
	return fmt.Sprintf("%s; charset=%s", contentType, ContentDefaultCharset)
}

// Model represents a encoder data
type Model interface{}

// HTTPEncoder encodes into a different formats
type HTTPEncoder struct {
	writer http.ResponseWriter
}

// NewHTTPEncoder creates a new encoder for concrete writer
func NewHTTPEncoder(writer http.ResponseWriter) *HTTPEncoder {
	return &HTTPEncoder{writer: writer}
}

// EncodeJSON encodes a data as json
func (enc *HTTPEncoder) EncodeJSON(model Model) error {
	enc.setContentType(ContentJSON)

	err := json.NewEncoder(enc.writer).Encode(model)
	if err != nil {
		http.Error(enc.writer, fmt.Sprintf("Unable to encode '%v' as JSON data", model), http.StatusInternalServerError)
	}
	return err
}

// EncodeJSONP encodes a data as jsonp
func (enc *HTTPEncoder) EncodeJSONP(callback string, model Model) error {
	enc.setContentType(ContentJSONP)

	data, _ := json.Marshal(model)
	_, err := fmt.Fprintf(enc.writer, "%s(%s)", callback, string(data))
	if err != nil {
		http.Error(enc.writer, fmt.Sprintf("Unable to encode '%v' as JSON for javascript func %s", model, callback), http.StatusInternalServerError)
	}
	return err
}

// EncodeData encodes an array of bytes
func (enc *HTTPEncoder) EncodeData(data []byte) error {
	enc.setContentType(ContentBinary)

	_, err := enc.writer.Write(data)
	if err != nil {
		http.Error(enc.writer, "Unable to encode binary data", http.StatusInternalServerError)
	}
	return err
}

// EncodeText encodes a plain text
func (enc *HTTPEncoder) EncodeText(text string) error {
	enc.setContentType(ContentText)

	_, err := fmt.Fprint(enc.writer, text)
	if err != nil {
		http.Error(enc.writer, fmt.Sprintf("Unable to encode text '%s'", text), http.StatusInternalServerError)
	}
	return err
}

func (enc *HTTPEncoder) setContentType(contentType string) {
	if enc.writer.Header().Get(ContentType) != "" {
		return
	}
	enc.writer.Header().Set(ContentType, ContentTypeWithCharset(contentType))
}
<commit_msg>Organise the functions order<commit_after>package giraffe

import (
	"encoding/json"
	"fmt"
	"net/http"
)

const (
	// ContentBinary header value for binary data.
	ContentBinary = "application/octet-stream"
	// ContentJSON header value for JSON data.
	ContentJSON = "application/json"
	// ContentJSONP header value for JSONP data.
	ContentJSONP = "application/javascript"
	// ContentText header value for Text data.
	ContentText = "text/plain"
	// ContentType header constant.
	ContentType = "Content-Type"
	// ContentDefaultCharset default character encoding.
	ContentDefaultCharset = "UTF-8"
)

// Model represents a encoder data
type Model interface{}

// HTTPEncoder encodes into a different formats
type HTTPEncoder struct {
	writer http.ResponseWriter
}

// EncodeJSON encodes a data as json
func (enc *HTTPEncoder) EncodeJSON(model Model) error {
	enc.setContentType(ContentJSON)

	err := json.NewEncoder(enc.writer).Encode(model)
	if err != nil {
		http.Error(enc.writer, fmt.Sprintf("Unable to encode '%v' as JSON data", model), http.StatusInternalServerError)
	}
	return err
}

// EncodeJSONP encodes a data as jsonp
func (enc *HTTPEncoder) EncodeJSONP(callback string, model Model) error {
	enc.setContentType(ContentJSONP)

	data, _ := json.Marshal(model)
	_, err := fmt.Fprintf(enc.writer, "%s(%s)", callback, string(data))
	if err != nil {
		http.Error(enc.writer, fmt.Sprintf("Unable to encode '%v' as JSON for javascript func %s", model, callback), http.StatusInternalServerError)
	}
	return err
}

// EncodeData encodes an array of bytes
func (enc *HTTPEncoder) EncodeData(data []byte) error {
	enc.setContentType(ContentBinary)

	_, err := enc.writer.Write(data)
	if err != nil {
		http.Error(enc.writer, "Unable to encode binary data", http.StatusInternalServerError)
	}
	return err
}

// EncodeText encodes a plain text
func (enc *HTTPEncoder) EncodeText(text string) error {
	enc.setContentType(ContentText)

	_, err := fmt.Fprint(enc.writer, text)
	if err != nil {
		http.Error(enc.writer, fmt.Sprintf("Unable to encode text '%s'", text), http.StatusInternalServerError)
	}
	return err
}

func (enc *HTTPEncoder) setContentType(contentType string) {
	if enc.writer.Header().Get(ContentType) != "" {
		return
	}
	enc.writer.Header().Set(ContentType, ContentTypeWithCharset(contentType))
}

// NewHTTPEncoder creates a new encoder for concrete writer
func NewHTTPEncoder(writer http.ResponseWriter) *HTTPEncoder {
	return &HTTPEncoder{writer: writer}
}

// ContentTypeWithCharset returns the contentype with the default charset
func ContentTypeWithCharset(contentType string) string {
	return fmt.Sprintf("%s; charset=%s", contentType, ContentDefaultCharset)
}
<|endoftext|>


<commit_before>package env

import (
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"

	"github.com/redneckbeard/quimby"
)

var (
	root, staticPrefix, logFilePath, port string
	logger                                *log.Logger
	messages                              = make(chan []interface{})
	// Debug is set via the -debug flag for the serve command.
	Debug bool
	// Handler comes from calling Handler() on a gadget.App object. It's used by the serve command to run the server.
	Handler    http.HandlerFunc
	configured bool
)

func init() {
	quimby.Add(&Serve{})
}

// The Serve command makes it easy to run Gadget applications.
type Serve struct {
	*quimby.Flagger
}

func (s *Serve) Desc() string {
	return "Start a gadget server."
}

// SetFlags defines flags for the serve command.
func (s *Serve) SetFlags() {
	s.StringVar(&staticPrefix, "static", "/static/", "URL prefix for serving the 'static' directory")
	s.StringVar(&root, "root", "", "Directory that contains uncompiled application assets. Defaults to current working directory.")
	s.StringVar(&logFilePath, "log", "", "Path to log file")
	s.StringVar(&port, "port", "8090", "port on which the application will listen")
	s.BoolVar(&Debug, "debug", true, "Sets the env.Debug value within Gadget")
}

// Run sets up a logger and runs the Handler.
func (s *Serve) Run() {
	if root == "" {
		if wd, err := os.Getwd(); err != nil {
			panic(err)
		} else {
			root = wd
		}
	} else if !filepath.IsAbs(root) {
		panic("fileroot must be an absolute path")
	}
	var writer io.Writer
	if logFilePath != "" {
		if !filepath.IsAbs(logFilePath) {
			logFilePath = RelPath(logFilePath)
		}
		if f, err := os.OpenFile(logFilePath, os.O_RDWR|os.O_CREATE, 0666); err != nil {
			panic(err)
		} else {
			writer = f
		}
	} else {
		writer = os.Stdout
	}
	logger = log.New(writer, "", 0)
	go func() {
		for msg := range messages {
			logger.Println(msg...)
		}
	}()
	serveStatic()
	http.HandleFunc("/", Handler)
	Log("Running Gadget at 0.0.0.0:" + port + "...")
	err := http.ListenAndServe(":"+port, nil)
	if err != nil {
		panic(err)
	}
}

// RelPath creates an absolute path from path segments path relative to the project root.
func RelPath(path ...string) string {
	return filepath.Join(append([]string{root}, path...)...)
}

func serveStatic() {
	http.Handle(staticPrefix, http.StripPrefix(staticPrefix, http.FileServer(http.Dir(RelPath("static")))))
}

// Open wraps os.Open, but with the assumption that the path is relative to the project root.
func Open(path string) (*os.File, error) {
	return os.Open(RelPath(path))
}

// Log writes arguments v as a single line to the default logger.
func Log(v ...interface{}) {
	go func() { messages <- v }()
}
<commit_msg>Allow for overriding environment variables in debug mode with a .env file.<commit_after>package env

import (
	"bufio"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"github.com/redneckbeard/quimby"
)

var (
	root, staticPrefix, logFilePath, port string
	logger                                *log.Logger
	messages                              = make(chan []interface{})
	envVars                               map[string]string
	// Debug is set via the -debug flag for the serve command.
	Debug bool
	// Handler comes from calling Handler() on a gadget.App object. It's used by the serve command to run the server.
	Handler    http.HandlerFunc
	configured bool
)

func init() {
	quimby.Add(&Serve{})
}

// The Serve command makes it easy to run Gadget applications.
type Serve struct {
	*quimby.Flagger
}

func (s *Serve) Desc() string {
	return "Start a gadget server."
}

// SetFlags defines flags for the serve command.
func (s *Serve) SetFlags() {
	s.StringVar(&staticPrefix, "static", "/static/", "URL prefix for serving the 'static' directory")
	s.StringVar(&root, "root", "", "Directory that contains uncompiled application assets. Defaults to current working directory.")
	s.StringVar(&logFilePath, "log", "", "Path to log file")
	s.StringVar(&port, "port", "8090", "port on which the application will listen")
	s.BoolVar(&Debug, "debug", true, "Sets the env.Debug value within Gadget")
}

// Run sets up a logger and runs the Handler.
func (s *Serve) Run() {
	if root == "" {
		if wd, err := os.Getwd(); err != nil {
			panic(err)
		} else {
			root = wd
		}
	} else if !filepath.IsAbs(root) {
		panic("fileroot must be an absolute path")
	}
	var writer io.Writer
	if logFilePath != "" {
		if !filepath.IsAbs(logFilePath) {
			logFilePath = RelPath(logFilePath)
		}
		if f, err := os.OpenFile(logFilePath, os.O_RDWR|os.O_CREATE, 0666); err != nil {
			panic(err)
		} else {
			writer = f
		}
	} else {
		writer = os.Stdout
	}
	logger = log.New(writer, "", 0)
	go func() {
		for msg := range messages {
			logger.Println(msg...)
		}
	}()
	serveStatic()
	http.HandleFunc("/", Handler)
	Log("Running Gadget at 0.0.0.0:" + port + "...")
	err := http.ListenAndServe(":"+port, nil)
	if err != nil {
		panic(err)
	}
}

// RelPath creates an absolute path from path segments path relative to the project root.
func RelPath(path ...string) string {
	return filepath.Join(append([]string{root}, path...)...)
}

func serveStatic() {
	http.Handle(staticPrefix, http.StripPrefix(staticPrefix, http.FileServer(http.Dir(RelPath("static")))))
}

// Open wraps os.Open, but with the assumption that the path is relative to the project root.
func Open(path string) (*os.File, error) {
	return os.Open(RelPath(path))
}

// Log writes arguments v as a single line to the default logger.
func Log(v ...interface{}) {
	go func() { messages <- v }()
}

func Get(varname string) string {
	if Debug {
		if envVars == nil {
			envVars = make(map[string]string)
			f, err := Open(".env")
			defer f.Close()
			if err != nil {
				return ""
			}
			scanner := bufio.NewScanner(f)
			for scanner.Scan() {
				text := scanner.Text()
				pair := strings.SplitN(text, "=", 2)
				envVars[pair[0]] = pair[1]
			}
		}
		return envVars[varname]
	}
	return os.Getenv(varname)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
	"strings"
	"time"
	"unicode"
)

// EnvvarHelper is the interface for getting, setting and retrieving ESTAFETTE_ environment variables
type EnvvarHelper interface {
	toUpperSnake(string) string
	getCommandOutput(string, ...string) (string, error)
	setEstafetteGlobalEnvvars() error
	initGitRevision() error
	initGitBranch() error
	initBuildDatetime() error
	initBuildStatus() error
	collectEstafetteEnvvars(estafetteManifest) map[string]string
	unsetEstafetteEnvvars()
	getEstafetteEnv(string) string
	setEstafetteEnv(string, string) error
	unsetEstafetteEnv(string) error
	getEstafetteEnvvarName(string) string
	overrideEnvvars(...map[string]string) map[string]string
}

type envvarHelperImpl struct {
	prefix string
}

// NewEnvvarHelper returns a new EnvvarHelper
func NewEnvvarHelper(prefix string) EnvvarHelper {
	return &envvarHelperImpl{
		prefix: prefix,
	}
}

// https://gist.github.com/elwinar/14e1e897fdbe4d3432e1
func (h *envvarHelperImpl) toUpperSnake(in string) string {
	runes := []rune(in)
	length := len(runes)

	var out []rune
	for i := 0; i < length; i++ {
		if i > 0 && unicode.IsUpper(runes[i]) && ((i+1 < length && unicode.IsLower(runes[i+1])) || unicode.IsLower(runes[i-1])) {
			out = append(out, '_')
		}
		out = append(out, unicode.ToUpper(runes[i]))
	}

	return string(out)
}

func (h *envvarHelperImpl) getCommandOutput(name string, arg ...string) (string, error) {

	out, err := exec.Command(name, arg...).Output()
	if err != nil {
		return "", err
	}

	return strings.TrimSpace(string(out)), nil
}

func (h *envvarHelperImpl) setEstafetteGlobalEnvvars() (err error) {

	// initialize git revision envvar
	err = h.initGitRevision()
	if err != nil {
		return err
	}

	// initialize git branch envvar
	err = h.initGitBranch()
	if err != nil {
		return err
	}

	// initialize build datetime envvar
	err = h.initBuildDatetime()
	if err != nil {
		return err
	}

	// initialize build status envvar
	err = h.initBuildStatus()
	if err != nil {
		return err
	}

	return
}

func (h *envvarHelperImpl) initGitRevision() (err error) {
	if h.getEstafetteEnv("ESTAFETTE_GIT_REVISION") == "" {
		revision, err := h.getCommandOutput("git", "rev-parse", "HEAD")
		if err != nil {
			return err
		}
		return h.setEstafetteEnv("ESTAFETTE_GIT_REVISION", revision)
	}
	return
}

func (h *envvarHelperImpl) initGitBranch() (err error) {
	if h.getEstafetteEnv("ESTAFETTE_GIT_BRANCH") == "" {
		branch, err := h.getCommandOutput("git", "rev-parse", "--abbrev-ref", "HEAD")
		if err != nil {
			return err
		}
		return h.setEstafetteEnv("ESTAFETTE_GIT_BRANCH", branch)
	}
	return
}

func (h *envvarHelperImpl) initBuildDatetime() (err error) {
	if h.getEstafetteEnv("ESTAFETTE_BUILD_DATETIME") == "" {
		return h.setEstafetteEnv("ESTAFETTE_BUILD_DATETIME", time.Now().UTC().Format(time.RFC3339))
	}
	return
}

func (h *envvarHelperImpl) initBuildStatus() (err error) {
	if h.getEstafetteEnv("ESTAFETTE_BUILD_STATUS") == "" {
		return h.setEstafetteEnv("ESTAFETTE_BUILD_STATUS", "succeeded")
	}
	return
}

func (h *envvarHelperImpl) collectEstafetteEnvvars(m estafetteManifest) (envvars map[string]string) {

	// set labels as envvars
	if m.Labels != nil && len(m.Labels) > 0 {
		for key, value := range m.Labels {
			envvarName := "ESTAFETTE_LABEL_" + h.toUpperSnake(key)
			h.setEstafetteEnv(envvarName, value)
		}
	}

	// return all envvars starting with ESTAFETTE_
	envvars = map[string]string{}

	for _, e := range os.Environ() {
		kvPair := strings.SplitN(e, "=", 2)

		if len(kvPair) == 2 {
			envvarName := h.getEstafetteEnvvarName(kvPair[0])
			envvarValue := kvPair[1]

			if strings.HasPrefix(envvarName, h.prefix) {
				envvars[envvarName] = envvarValue
			}
		}
	}

	return
}

// only to be used from unit tests
func (h *envvarHelperImpl) unsetEstafetteEnvvars() {

	for _, e := range os.Environ() {
		kvPair := strings.SplitN(e, "=", 2)

		if len(kvPair) == 2 {
			envvarName := h.getEstafetteEnvvarName(kvPair[0])

			if strings.HasPrefix(envvarName, h.prefix) {
				h.unsetEstafetteEnv(envvarName)
			}
		}
	}
}

func (h *envvarHelperImpl) getEstafetteEnv(key string) string {

	key = h.getEstafetteEnvvarName(key)

	if strings.HasPrefix(key, h.prefix) {
		return os.Getenv(key)
	}

	return fmt.Sprintf("${%v}", key)
}

func (h *envvarHelperImpl) setEstafetteEnv(key, value string) error {

	key = h.getEstafetteEnvvarName(key)

	return os.Setenv(key, value)
}

func (h *envvarHelperImpl) unsetEstafetteEnv(key string) error {

	key = h.getEstafetteEnvvarName(key)

	return os.Unsetenv(key)
}

func (h *envvarHelperImpl) getEstafetteEnvvarName(key string) string {
	return strings.Replace(key, "ESTAFETTE_", h.prefix, -1)
}

func (h *envvarHelperImpl) overrideEnvvars(envvarMaps ...map[string]string) (envvars map[string]string) {

	envvars = make(map[string]string)
	for _, envvarMap := range envvarMaps {
		if envvarMap != nil && len(envvarMap) > 0 {
			for k, v := range envvarMap {
				envvars[k] = v
			}
		}
	}

	return
}
<commit_msg>don't use getEstafetteEnvvarName in collect and unset all estafette envvars<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"strings"
	"time"
	"unicode"
)

// EnvvarHelper is the interface for getting, setting and retrieving ESTAFETTE_ environment variables
type EnvvarHelper interface {
	toUpperSnake(string) string
	getCommandOutput(string, ...string) (string, error)
	setEstafetteGlobalEnvvars() error
	initGitRevision() error
	initGitBranch() error
	initBuildDatetime() error
	initBuildStatus() error
	collectEstafetteEnvvars(estafetteManifest) map[string]string
	unsetEstafetteEnvvars()
	getEstafetteEnv(string) string
	setEstafetteEnv(string, string) error
	unsetEstafetteEnv(string) error
	getEstafetteEnvvarName(string) string
	overrideEnvvars(...map[string]string) map[string]string
}

type envvarHelperImpl struct {
	prefix string
}

// NewEnvvarHelper returns a new EnvvarHelper
func NewEnvvarHelper(prefix string) EnvvarHelper {
	return &envvarHelperImpl{
		prefix: prefix,
	}
}

// https://gist.github.com/elwinar/14e1e897fdbe4d3432e1
func (h *envvarHelperImpl) toUpperSnake(in string) string {
	runes := []rune(in)
	length := len(runes)

	var out []rune
	for i := 0; i < length; i++ {
		if i > 0 && unicode.IsUpper(runes[i]) && ((i+1 < length && unicode.IsLower(runes[i+1])) || unicode.IsLower(runes[i-1])) {
			out = append(out, '_')
		}
		out = append(out, unicode.ToUpper(runes[i]))
	}

	return string(out)
}

func (h *envvarHelperImpl) getCommandOutput(name string, arg ...string) (string, error) {

	out, err := exec.Command(name, arg...).Output()
	if err != nil {
		return "", err
	}

	return strings.TrimSpace(string(out)), nil
}

func (h *envvarHelperImpl) setEstafetteGlobalEnvvars() (err error) {

	// initialize git revision envvar
	err = h.initGitRevision()
	if err != nil {
		return err
	}

	// initialize git branch envvar
	err = h.initGitBranch()
	if err != nil {
		return err
	}

	// initialize build datetime envvar
	err = h.initBuildDatetime()
	if err != nil {
		return err
	}

	// initialize build status envvar
	err = h.initBuildStatus()
	if err != nil {
		return err
	}

	return
}

func (h *envvarHelperImpl) initGitRevision() (err error) {
	if h.getEstafetteEnv("ESTAFETTE_GIT_REVISION") == "" {
		revision, err := h.getCommandOutput("git", "rev-parse", "HEAD")
		if err != nil {
			return err
		}
		return h.setEstafetteEnv("ESTAFETTE_GIT_REVISION", revision)
	}
	return
}

func (h *envvarHelperImpl) initGitBranch() (err error) {
	if h.getEstafetteEnv("ESTAFETTE_GIT_BRANCH") == "" {
		branch, err := h.getCommandOutput("git", "rev-parse", "--abbrev-ref", "HEAD")
		if err != nil {
			return err
		}
		return h.setEstafetteEnv("ESTAFETTE_GIT_BRANCH", branch)
	}
	return
}

func (h *envvarHelperImpl) initBuildDatetime() (err error) {
	if h.getEstafetteEnv("ESTAFETTE_BUILD_DATETIME") == "" {
		return h.setEstafetteEnv("ESTAFETTE_BUILD_DATETIME", time.Now().UTC().Format(time.RFC3339))
	}
	return
}

func (h *envvarHelperImpl) initBuildStatus() (err error) {
	if h.getEstafetteEnv("ESTAFETTE_BUILD_STATUS") == "" {
		return h.setEstafetteEnv("ESTAFETTE_BUILD_STATUS", "succeeded")
	}
	return
}

func (h *envvarHelperImpl) collectEstafetteEnvvars(m estafetteManifest) (envvars map[string]string) {

	// set labels as envvars
	if m.Labels != nil && len(m.Labels) > 0 {
		for key, value := range m.Labels {
			envvarName := "ESTAFETTE_LABEL_" + h.toUpperSnake(key)
			h.setEstafetteEnv(envvarName, value)
		}
	}

	// return all envvars starting with ESTAFETTE_
	envvars = map[string]string{}

	for _, e := range os.Environ() {
		kvPair := strings.SplitN(e, "=", 2)

		if len(kvPair) == 2 {
			envvarName := kvPair[0]
			envvarValue := kvPair[1]

			if strings.HasPrefix(envvarName, h.prefix) {
				envvars[envvarName] = envvarValue
			}
		}
	}

	return
}

// only to be used from unit tests
func (h *envvarHelperImpl) unsetEstafetteEnvvars() {

	for _, e := range os.Environ() {
		kvPair := strings.SplitN(e, "=", 2)

		if len(kvPair) == 2 {
			envvarName := kvPair[0]

			if strings.HasPrefix(envvarName, h.prefix) {
				h.unsetEstafetteEnv(envvarName)
			}
		}
	}
}

func (h *envvarHelperImpl) getEstafetteEnv(key string) string {

	key = h.getEstafetteEnvvarName(key)

	if strings.HasPrefix(key, h.prefix) {
		return os.Getenv(key)
	}

	return fmt.Sprintf("${%v}", key)
}

func (h *envvarHelperImpl) setEstafetteEnv(key, value string) error {

	key = h.getEstafetteEnvvarName(key)

	return os.Setenv(key, value)
}

func (h *envvarHelperImpl) unsetEstafetteEnv(key string) error {

	key = h.getEstafetteEnvvarName(key)

	return os.Unsetenv(key)
}

func (h *envvarHelperImpl) getEstafetteEnvvarName(key string) string {
	return strings.Replace(key, "ESTAFETTE_", h.prefix, -1)
}

func (h *envvarHelperImpl) overrideEnvvars(envvarMaps ...map[string]string) (envvars map[string]string) {

	envvars = make(map[string]string)
	for _, envvarMap := range envvarMaps {
		if envvarMap != nil && len(envvarMap) > 0 {
			for k, v := range envvarMap {
				envvars[k] = v
			}
		}
	}

	return
}
<|endoftext|>


<commit_before>package esa

import (
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
)

const (
	// defaultBaseURL esa API の host
	defaultBaseURL = "https://api.esa.io"
)

// Client esa API クライアント
type Client struct {
	client  *http.Client
	apiKey  string
	baseURL string
	Team    *TeamService
	Stats   *StatsService
	Post    *PostService
	Comment *CommentService
	Members *MembersService
}

// NewClient esa APIのClientを生成する
func NewClient(apikey string) *Client {
	c := &Client{}
	c.client = http.DefaultClient
	c.apiKey = apikey
	c.baseURL = defaultBaseURL
	c.Team = &TeamService{client: c}
	c.Stats = &StatsService{client: c}
	c.Post = &PostService{client: c}
	c.Comment = &CommentService{client: c}
	c.Members = &MembersService{client: c}

	return c
}

func (c *Client) createURL(esaURL string) string {
	return c.baseURL + esaURL + "?access_token=" + c.apiKey
}

func (c *Client) post(esaURL string, bodyType string, body io.Reader, v interface{}) (resp *http.Response, err error) {
	res, err := c.client.Post(c.createURL(esaURL), bodyType, body)
	if err != nil {
		return nil, err
	}

	defer res.Body.Close()

	if res.StatusCode != 201 {
		return nil, errors.New(http.StatusText(res.StatusCode))
	}

	if err := responseUnmarshal(res.Body, v); err != nil {
		return nil, err
	}

	return res, nil
}

func (c *Client) patch(esaURL string, bodyType string, body io.Reader, v interface{}) (resp *http.Response, err error) {
	path := c.createURL(esaURL)
	req, err := http.NewRequest("PATCH", path, body)
	if err != nil {
		return nil, err
	}
	req.Header.Add("Content-Type", bodyType)
	res, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}

	defer res.Body.Close()

	if res.StatusCode != 200 {
		return nil, errors.New(http.StatusText(res.StatusCode))
	}

	if err := responseUnmarshal(res.Body, v); err != nil {
		return nil, err
	}

	return res, nil
}

func (c *Client) delete(esaURL string) (resp *http.Response, err error) {
	path := c.createURL(esaURL)
	req, err := http.NewRequest("DELETE", path, nil)
	if err != nil {
		return nil, err
	}
	res, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}

	defer res.Body.Close()

	if res.StatusCode != 204 {
		return nil, errors.New(http.StatusText(res.StatusCode))
	}

	return res, nil
}

func (c *Client) get(esaURL string, query url.Values, v interface{}) (resp *http.Response, err error) {
	path := c.createURL(esaURL) + "?" + query.Encode()

	res, err := c.client.Get(path)
	if err != nil {
		return nil, err
	}

	defer res.Body.Close()

	if res.StatusCode != 200 {
		return nil, errors.New(http.StatusText(res.StatusCode))
	}

	if err := responseUnmarshal(res.Body, v); err != nil {
		return nil, err
	}

	return res, err
}

func responseUnmarshal(body io.ReadCloser, v interface{}) error {
	data, err := ioutil.ReadAll(body)
	if err != nil {
		return err
	}

	if err := json.Unmarshal(data, v); err != nil {
		return err
	}
	return nil
}
<commit_msg>:bug: fix http get method<commit_after>package esa

import (
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
)

const (
	// defaultBaseURL esa API の host
	defaultBaseURL = "https://api.esa.io"
)

// Client esa API クライアント
type Client struct {
	client  *http.Client
	apiKey  string
	baseURL string
	Team    *TeamService
	Stats   *StatsService
	Post    *PostService
	Comment *CommentService
	Members *MembersService
}

// NewClient esa APIのClientを生成する
func NewClient(apikey string) *Client {
	c := &Client{}
	c.client = http.DefaultClient
	c.apiKey = apikey
	c.baseURL = defaultBaseURL
	c.Team = &TeamService{client: c}
	c.Stats = &StatsService{client: c}
	c.Post = &PostService{client: c}
	c.Comment = &CommentService{client: c}
	c.Members = &MembersService{client: c}

	return c
}

func (c *Client) createURL(esaURL string) string {
	return c.baseURL + esaURL + "?access_token=" + c.apiKey
}

func (c *Client) post(esaURL string, bodyType string, body io.Reader, v interface{}) (resp *http.Response, err error) {
	res, err := c.client.Post(c.createURL(esaURL), bodyType, body)
	if err != nil {
		return nil, err
	}

	defer res.Body.Close()

	if res.StatusCode != 201 {
		return nil, errors.New(http.StatusText(res.StatusCode))
	}

	if err := responseUnmarshal(res.Body, v); err != nil {
		return nil, err
	}

	return res, nil
}

func (c *Client) patch(esaURL string, bodyType string, body io.Reader, v interface{}) (resp *http.Response, err error) {
	path := c.createURL(esaURL)
	req, err := http.NewRequest("PATCH", path, body)
	if err != nil {
		return nil, err
	}
	req.Header.Add("Content-Type", bodyType)
	res, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}

	defer res.Body.Close()

	if res.StatusCode != 200 {
		return nil, errors.New(http.StatusText(res.StatusCode))
	}

	if err := responseUnmarshal(res.Body, v); err != nil {
		return nil, err
	}

	return res, nil
}

func (c *Client) delete(esaURL string) (resp *http.Response, err error) {
	path := c.createURL(esaURL)
	req, err := http.NewRequest("DELETE", path, nil)
	if err != nil {
		return nil, err
	}
	res, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}

	defer res.Body.Close()

	if res.StatusCode != 204 {
		return nil, errors.New(http.StatusText(res.StatusCode))
	}

	return res, nil
}

func (c *Client) get(esaURL string, query url.Values, v interface{}) (resp *http.Response, err error) {
	path := c.createURL(esaURL)
	queries := query.Encode()
	if len(queries) != 0 {
		path += "?" + queries
	}

	res, err := c.client.Get(path)
	if err != nil {
		return nil, err
	}

	defer res.Body.Close()

	if res.StatusCode != 200 {
		return nil, errors.New(http.StatusText(res.StatusCode))
	}

	if err := responseUnmarshal(res.Body, v); err != nil {
		return nil, err
	}

	return res, err
}

func responseUnmarshal(body io.ReadCloser, v interface{}) error {
	data, err := ioutil.ReadAll(body)
	if err != nil {
		return err
	}

	if err := json.Unmarshal(data, v); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 The Neugram Authors. All rights reserved.
// See the LICENSE file for rights to use this source code.

package eval

import (
	"fmt"
	gotypes "go/types"
	"reflect"

	"neugram.io/eval/gowrap"
	"neugram.io/lang/tipe"
)

type GoPkg struct {
	Type  *tipe.Package
	GoPkg *gotypes.Package
	Wrap  *gowrap.Pkg
}

type GoValue struct {
	Type  tipe.Type
	Value interface{}
}

type GoFunc struct {
	Type *tipe.Func
	Func interface{}
}

func (f GoFunc) call(args []interface{}) (res []interface{}, err error) {
	var vres []reflect.Value
	v := reflect.ValueOf(f.Func)
	if f.Type.Variadic {
		nonVarLen := len(f.Type.Params.Elems) - 1
		var vargs []reflect.Value
		for i := 0; i < nonVarLen; i++ {
			vargs = append(vargs, reflect.ValueOf(args[i]))
		}
		if len(args) > nonVarLen {
			vargs = append(vargs, reflect.ValueOf(args[nonVarLen:]))
		} else {
			vargs = append(vargs, reflect.ValueOf([]interface{}{}))
		}
		vres = v.CallSlice(vargs)
	} else {
		return nil, fmt.Errorf("Call GoFunc TODO")
	}

	res = make([]interface{}, len(vres))
	for i, v := range vres {
		res[i] = &GoValue{
			Type:  f.Type.Results.Elems[i],
			Value: v.Interface(),
		}
	}
	return res, nil
}
<commit_msg>eval: call non-variadic go funcs<commit_after>// Copyright 2015 The Neugram Authors. All rights reserved.
// See the LICENSE file for rights to use this source code.

package eval

import (
	gotypes "go/types"
	"reflect"

	"neugram.io/eval/gowrap"
	"neugram.io/lang/tipe"
)

type GoPkg struct {
	Type  *tipe.Package
	GoPkg *gotypes.Package
	Wrap  *gowrap.Pkg
}

type GoValue struct {
	Type  tipe.Type
	Value interface{}
}

type GoFunc struct {
	Type *tipe.Func
	Func interface{}
}

func (f GoFunc) call(args []interface{}) (res []interface{}, err error) {
	var vargs []reflect.Value
	var vres []reflect.Value
	v := reflect.ValueOf(f.Func)
	if f.Type.Variadic {
		nonVarLen := len(f.Type.Params.Elems) - 1
		for i := 0; i < nonVarLen; i++ {
			vargs = append(vargs, reflect.ValueOf(args[i]))
		}
		if len(args) > nonVarLen {
			vargs = append(vargs, reflect.ValueOf(args[nonVarLen:]))
		} else {
			vargs = append(vargs, reflect.ValueOf([]interface{}{}))
		}
		vres = v.CallSlice(vargs)
	} else {
		var vargs []reflect.Value
		for _, arg := range args {
			vargs = append(vargs, reflect.ValueOf(arg))
		}
		vres = v.Call(vargs)
	}

	res = make([]interface{}, len(vres))
	for i, v := range vres {
		res[i] = &GoValue{
			Type:  f.Type.Results.Elems[i],
			Value: v.Interface(),
		}
	}
	return res, nil
}
<|endoftext|>


<commit_before>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package main

import (
	"os"
	"testing"

	"github.com/go-test/deep"
	"github.com/google/mtail/metrics"
	"github.com/google/mtail/mtail"
	"github.com/google/mtail/testdata"
	"github.com/google/mtail/watcher"
)

var exampleProgramTests = []struct {
	programfile string // Example program file.
	logfile     string // Sample log input.
	goldenfile  string // Expected metrics after processing.
}{
	// {
	// 	"examples/rsyncd.mtail",
	// 	"testdata/rsyncd.log",
	// 	"testdata/rsyncd.golden",
	// },
	// {
	// 	"examples/sftp.mtail",
	// 	"testdata/sftp_chroot.log",
	// 	"testdata/sftp_chroot.golden",
	// },
	// {
	// 	"examples/dhcpd.mtail",
	// 	"testdata/anonymised_dhcpd_log",
	// 	"testdata/anonymised_dhcpd_log.golden",
	// },
	{
		"examples/ntpd.mtail",
		"testdata/ntp4",
		"testdata/ntp4.golden",
	},
	// {
	// 	"examples/ntpd.mtail",
	// 	"testdata/xntp3_peerstats",
	// 	"testdata/xntp3_peerstats.golden",
	// },
	// {
	// 	"examples/otherwise.mtail",
	// 	"testdata/otherwise.log",
	// 	"testdata/otherwise.golden",
	// },
	// {
	// 	"examples/else.mtail",
	// 	"testdata/else.log",
	// 	"testdata/else.golden",
	// },
	{
		"examples/types.mtail",
		"testdata/types.log",
		"testdata/types.golden",
	},
}

func TestExamplePrograms(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping test in short mode")
	}
	for _, tc := range exampleProgramTests {
		w := watcher.NewFakeWatcher()
		store := metrics.NewStore()
		o := mtail.Options{Progs: tc.programfile, W: w, Store: store}
		o.DumpAstTypes = true
		o.DumpBytecode = true
		mtail, err := mtail.New(o)
		if err != nil {
			t.Fatalf("create mtail failed: %s", err)
		}

		if _, err := mtail.OneShot(tc.logfile, false); err != nil {
			t.Errorf("Oneshot failed for %s: %s", tc.logfile, err)
			continue
		}

		g, err := os.Open(tc.goldenfile)
		if err != nil {
			t.Fatalf("%s: could not open golden file: %s", tc.goldenfile, err)
		}
		defer g.Close()

		golden_store := metrics.NewStore()
		testdata.ReadTestData(g, tc.programfile, golden_store)

		mtail.Close()

		diff := deep.Equal(golden_store, store)

		if diff != nil {
			t.Errorf("%s: metrics don't match:\n%v", tc.programfile, diff)
			t.Errorf(" Golden metrics: %s", golden_store.Metrics)
			t.Errorf("Program metrics: %s", store.Metrics)
		}
	}
}
<commit_msg>Comment out new test.<commit_after>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package main

import (
	"os"
	"testing"

	"github.com/go-test/deep"
	"github.com/google/mtail/metrics"
	"github.com/google/mtail/mtail"
	"github.com/google/mtail/testdata"
	"github.com/google/mtail/watcher"
)

var exampleProgramTests = []struct {
	programfile string // Example program file.
	logfile     string // Sample log input.
	goldenfile  string // Expected metrics after processing.
}{
	// {
	// 	"examples/rsyncd.mtail",
	// 	"testdata/rsyncd.log",
	// 	"testdata/rsyncd.golden",
	// },
	// {
	// 	"examples/sftp.mtail",
	// 	"testdata/sftp_chroot.log",
	// 	"testdata/sftp_chroot.golden",
	// },
	// {
	// 	"examples/dhcpd.mtail",
	// 	"testdata/anonymised_dhcpd_log",
	// 	"testdata/anonymised_dhcpd_log.golden",
	// },
	{
		"examples/ntpd.mtail",
		"testdata/ntp4",
		"testdata/ntp4.golden",
	},
	// {
	// 	"examples/ntpd.mtail",
	// 	"testdata/xntp3_peerstats",
	// 	"testdata/xntp3_peerstats.golden",
	// },
	// {
	// 	"examples/otherwise.mtail",
	// 	"testdata/otherwise.log",
	// 	"testdata/otherwise.golden",
	// },
	// {
	// 	"examples/else.mtail",
	// 	"testdata/else.log",
	// 	"testdata/else.golden",
	// },
	// {
	// 	"examples/types.mtail",
	// 	"testdata/types.log",
	// 	"testdata/types.golden",
	// },
}

func TestExamplePrograms(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping test in short mode")
	}
	for _, tc := range exampleProgramTests {
		w := watcher.NewFakeWatcher()
		store := metrics.NewStore()
		o := mtail.Options{Progs: tc.programfile, W: w, Store: store}
		o.DumpAstTypes = true
		o.DumpBytecode = true
		mtail, err := mtail.New(o)
		if err != nil {
			t.Fatalf("create mtail failed: %s", err)
		}

		if _, err := mtail.OneShot(tc.logfile, false); err != nil {
			t.Errorf("Oneshot failed for %s: %s", tc.logfile, err)
			continue
		}

		g, err := os.Open(tc.goldenfile)
		if err != nil {
			t.Fatalf("%s: could not open golden file: %s", tc.goldenfile, err)
		}
		defer g.Close()

		golden_store := metrics.NewStore()
		testdata.ReadTestData(g, tc.programfile, golden_store)

		mtail.Close()

		diff := deep.Equal(golden_store, store)

		if diff != nil {
			t.Errorf("%s: metrics don't match:\n%v", tc.programfile, diff)
			t.Errorf(" Golden metrics: %s", golden_store.Metrics)
			t.Errorf("Program metrics: %s", store.Metrics)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
    "os"
    "fmt"
    "encoding/json"
    "./fritzbox"
)

func handleMessages(msgchan <-chan fritzbox.FbEvent){
  ev := <- msgchan

  jsonm,_ := json.Marshal(&ev)
  fmt.Printf("Some JSON: %s\n", jsonm)

  if ev.EventName == fritzbox.CALL {
    fmt.Printf("%s Event: %s->%s\n", ev.EventName, ev.Source, ev.Destination)
  } else if ev.EventName == fritzbox.RING {
    fmt.Printf("%s Event: %s->%s\n", ev.EventName, ev.Source, ev.Destination)
  } else {
      fmt.Printf("! %s\n", ev)
  }
}

func mainloop(host string) {
  c := new(fritzbox.CallmonHandler).Connect(host)

  defer c.Close()

  if c.Connected {
    recv := make(chan fritzbox.FbEvent)
    go handleMessages(recv)

    // Inject a test message
    f := c.Parse("06.08.14 14:52:26;CALL;1;10;50000001;012344567;SIP1;")
    recv <- f

    c.Loop(recv)
  }
}

func main() {
  arg := os.Args

  host := "fritz.box"
  if (len(arg) > 1 && arg[1] != "") {
    host = arg[1]
  }

  mainloop(host)
  fmt.Println("NEVER EVER GONNA GIVE YOU UP")
}
<commit_msg>Allow more than one event ;)<commit_after>package main

import (
    "os"
    "fmt"
    "encoding/json"
    "./fritzbox"
)

func handleMessages(msgchan <-chan fritzbox.FbEvent){
  for {
      ev := <- msgchan

      jsonm,_ := json.Marshal(&ev)
      fmt.Printf("Some JSON: %s\n", jsonm)

      if ev.EventName == fritzbox.CALL {
        fmt.Printf("%s Event: %s->%s\n", ev.EventName, ev.Source, ev.Destination)
      } else if ev.EventName == fritzbox.RING {
        fmt.Printf("%s Event: %s->%s\n", ev.EventName, ev.Source, ev.Destination)
      } else {
          fmt.Printf("! %s\n", ev)
      }
  }
}

func mainloop(host string) {
  c := new(fritzbox.CallmonHandler).Connect(host)

  defer c.Close()

  if c.Connected {
    recv := make(chan fritzbox.FbEvent)
    go handleMessages(recv)

    // Inject a test message
    f := c.Parse("06.08.14 14:52:26;CALL;1;10;50000001;012344567;SIP1;")
    recv <- f

    c.Loop(recv)
  }
}

func main() {
  arg := os.Args

  host := "fritz.box"
  if (len(arg) > 1 && arg[1] != "") {
    host = arg[1]
  }

  mainloop(host)
  fmt.Println("NEVER EVER GONNA GIVE YOU UP")
}
<|endoftext|>