<commit_before>package main

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	stdlog "log"
	"os"
	"os/signal"
	"runtime"
	"syscall"

	"github.com/alecthomas/kingpin"
	contracts "github.com/estafette/estafette-ci-contracts"
	crypt "github.com/estafette/estafette-ci-crypt"
	manifest "github.com/estafette/estafette-ci-manifest"
	"github.com/opentracing/opentracing-go"
	"github.com/rs/zerolog"
	"github.com/rs/zerolog/log"
	"github.com/uber/jaeger-client-go"
	jaegercfg "github.com/uber/jaeger-client-go/config"
)

var (
	app       string
	version   string
	branch    string
	revision  string
	buildDate string
	goVersion = runtime.Version()

	builderConfigFlag         = kingpin.Flag("builder-config", "The Estafette server passes in this json structure to parameterize the build, set trusted images and inject credentials.").Envar("BUILDER_CONFIG").String()
	secretDecryptionKey       = kingpin.Flag("secret-decryption-key", "The AES-256 key used to decrypt secrets that have been encrypted with it.").Envar("SECRET_DECRYPTION_KEY").String()
	secretDecryptionKeyBase64 = kingpin.Flag("secret-decryption-key-base64", "The base64 encoded AES-256 key used to decrypt secrets that have been encrypted with it.").Envar("SECRET_DECRYPTION_KEY_BASE64").String()
	runAsJob                  = kingpin.Flag("run-as-job", "To run the builder as a job and prevent build failures to fail the job.").Default("false").OverrideDefaultFromEnvar("RUN_AS_JOB").Bool()
)

func main() {

	// parse command line parameters
	kingpin.Parse()

	// define channel to catch SIGTERM and send out cancellation to stop further execution of stages and send the final state and logs to the ci server
	osSignals := make(chan os.Signal, 1)
	signal.Notify(osSignals, os.Interrupt, syscall.SIGTERM)
	cancellationChannel := make(chan struct{})
	go func(osSignals chan os.Signal, cancellationChannel chan struct{}) {
		// wait for sigterm
		<-osSignals
		// broadcast a cancellation
		close(cancellationChannel)
	}(osSignals, cancellationChannel)

	// support both base64 encoded decryption key and non-encoded
	secretDecryptionKeyBase64Encoded := *secretDecryptionKeyBase64 != ""
	decryptionKey := *secretDecryptionKey
	if secretDecryptionKeyBase64Encoded {
		decryptionKey = *secretDecryptionKeyBase64
	}

	secretHelper := crypt.NewSecretHelper(decryptionKey, secretDecryptionKeyBase64Encoded)

	// read builder config from envvar and unset envar; will replace parameterizing the job via separate envvars
	var builderConfig contracts.BuilderConfig
	builderConfigJSON := *builderConfigFlag
	if builderConfigJSON == "" {
		log.Fatal().Msg("BUILDER_CONFIG envvar is not set")
	}
	os.Unsetenv("BUILDER_CONFIG")

	// unmarshal builder config
	err := json.Unmarshal([]byte(builderConfigJSON), &builderConfig)
	if err != nil {
		log.Fatal().Err(err).Interface("builderConfigJSON", builderConfigJSON).Msg("Failed to unmarshal BUILDER_CONFIG")
	}

	// decrypt all credentials
	decryptedCredentials := []*contracts.CredentialConfig{}
	for _, c := range builderConfig.Credentials {

		// loop all additional properties and decrypt
		decryptedAdditionalProperties := map[string]interface{}{}
		for key, value := range c.AdditionalProperties {
			if s, isString := value.(string); isString {
				decryptedAdditionalProperties[key], err = secretHelper.DecryptAllEnvelopes(s)
				if err != nil {
					log.Fatal().Err(err).Msgf("Failed decrypting credential %v property %v", c.Name, key)
				}
			} else {
				decryptedAdditionalProperties[key] = value
			}
		}
		c.AdditionalProperties = decryptedAdditionalProperties

		decryptedCredentials = append(decryptedCredentials, c)
	}
	builderConfig.Credentials = decryptedCredentials

	// bootstrap
	obfuscator := NewObfuscator(secretHelper)
	envvarHelper := NewEnvvarHelper("ESTAFETTE_", secretHelper, obfuscator)
	whenEvaluator := NewWhenEvaluator(envvarHelper)
	dockerRunner := NewDockerRunner(envvarHelper, obfuscator, *runAsJob, builderConfig, cancellationChannel)
	pipelineRunner := NewPipelineRunner(envvarHelper, whenEvaluator, dockerRunner, *runAsJob, cancellationChannel)
	endOfLifeHelper := NewEndOfLifeHelper(*runAsJob, builderConfig)

	// detect controlling server
	ciServer := envvarHelper.getCiServer()

	if ciServer == "estafette" {
		// unset all ESTAFETTE_ envvars so they don't get abused by non-estafette components
		envvarHelper.unsetEstafetteEnvvars()
	}

	if ciServer == "gocd" {

		fatalHandler := NewGocdFatalHandler()

		// pretty print for go.cd integration
		log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().
			Timestamp().
			Logger()

		stdlog.SetFlags(0)
		stdlog.SetOutput(log.Logger)

		// log startup message
		log.Info().
			Str("branch", branch).
			Str("revision", revision).
			Str("buildDate", buildDate).
			Str("goVersion", goVersion).
			Msgf("Starting %v version %v...", app, version)

		// create docker client
		_, err := dockerRunner.createDockerClient()
		if err != nil {
			fatalHandler.handleGocdFatal(err, "Failed creating a docker client")
		}

		// read yaml
		manifest, err := manifest.ReadManifestFromFile(".estafette.yaml")
		if err != nil {
			fatalHandler.handleGocdFatal(err, "Reading .estafette.yaml manifest failed")
		}

		// initialize obfuscator
		err = obfuscator.CollectSecrets(manifest)
		if err != nil {
			fatalHandler.handleGocdFatal(err, "Collecting secrets to obfuscate failed")
		}

		// get current working directory
		dir, err := os.Getwd()
		if err != nil {
			fatalHandler.handleGocdFatal(err, "Getting current working directory failed")
		}

		log.Info().Msgf("Running %v stages", len(manifest.Stages))

		err = envvarHelper.setEstafetteGlobalEnvvars()
		if err != nil {
			fatalHandler.handleGocdFatal(err, "Setting global environment variables failed")
		}
		err = envvarHelper.setEstafetteStagesEnvvar(manifest.Stages)
		if err != nil {
			fatalHandler.handleGocdFatal(err, "Setting ESTAFETTE_STAGES environment variable failed")
		}

		// collect estafette and 'global' envvars from manifest
		estafetteEnvvars := envvarHelper.collectEstafetteEnvvarsAndLabels(manifest)
		globalEnvvars := envvarHelper.collectGlobalEnvvars(manifest)

		// merge estafette and global envvars
		envvars := envvarHelper.overrideEnvvars(estafetteEnvvars, globalEnvvars)

		// run stages
		result, err := pipelineRunner.runStages(context.Background(), manifest.Stages, dir, envvars)
		if err != nil {
			fatalHandler.handleGocdFatal(err, "Executing stages from manifest failed")
		}

		renderStats(result)

		handleExit(result)

	} else if ciServer == "estafette" {

		// log as severity for stackdriver logging to recognize the level
		zerolog.LevelFieldName = "severity"

		closer := initJaeger(app)
		defer closer.Close()

		envvarHelper.setEstafetteBuilderConfigEnvvars(builderConfig)

		buildLog := contracts.BuildLog{
			RepoSource:   builderConfig.Git.RepoSource,
			RepoOwner:    builderConfig.Git.RepoOwner,
			RepoName:     builderConfig.Git.RepoName,
			RepoBranch:   builderConfig.Git.RepoBranch,
			RepoRevision: builderConfig.Git.RepoRevision,
			Steps:        make([]contracts.BuildLogStep, 0),
		}

		// set some default fields added to all logs
		log.Logger = zerolog.New(os.Stdout).With().
			Timestamp().
			Str("app", app).
			Str("version", version).
			Str("jobName", *builderConfig.JobName).
			Interface("git", builderConfig.Git).
			Logger()

		stdlog.SetFlags(0)
		stdlog.SetOutput(log.Logger)

		// log startup message
		log.Info().
			Str("branch", branch).
			Str("revision", revision).
			Str("buildDate", buildDate).
			Str("goVersion", goVersion).
			Msgf("Starting %v version %v...", app, version)

		rootSpanName := "RunBuildJob"
		if *builderConfig.Action == "release" {
			rootSpanName = "RunReleaseJob"
		}

		rootSpan := opentracing.StartSpan(rootSpanName)
		defer rootSpan.Finish()

		ctx := context.Background()
		ctx = opentracing.ContextWithSpan(ctx, rootSpan)

		// start docker daemon
		dockerDaemonStartSpan, _ := opentracing.StartSpanFromContext(ctx, "StartDockerDaemon")
		err = dockerRunner.startDockerDaemon()
		if err != nil {
			endOfLifeHelper.handleFatal(ctx, buildLog, err, "Error starting docker daemon")
		}

		// wait for docker daemon to be ready for usage
		dockerRunner.waitForDockerDaemon()
		dockerDaemonStartSpan.Finish()

		// listen to cancellation in order to stop any running pipeline or container
		go pipelineRunner.stopPipelineOnCancellation()
		go dockerRunner.stopContainerOnCancellation()

		// get current working directory
		dir, err := os.Getwd()
		if err != nil {
			endOfLifeHelper.handleFatal(ctx, buildLog, err, "Getting current working directory failed")
		}

		// set some envvars
		err = envvarHelper.setEstafetteGlobalEnvvars()
		if err != nil {
			endOfLifeHelper.handleFatal(ctx, buildLog, err, "Setting global environment variables failed")
		}

		// initialize obfuscator
		err = obfuscator.CollectSecrets(*builderConfig.Manifest)
		if err != nil {
			endOfLifeHelper.handleFatal(ctx, buildLog, err, "Collecting secrets to obfuscate failed")
		}

		// check whether this is a regular build or a release
		stages := builderConfig.Manifest.Stages
		if *builderConfig.Action == "release" {
			// check if the release is defined
			releaseExists := false
			for _, r := range builderConfig.Manifest.Releases {
				if r.Name == builderConfig.ReleaseParams.ReleaseName {
					releaseExists = true
					stages = r.Stages
				}
			}
			if !releaseExists {
				endOfLifeHelper.handleFatal(ctx, buildLog, nil, fmt.Sprintf("Release %v does not exist", builderConfig.ReleaseParams.ReleaseName))
			}
			log.Info().Msgf("Starting release %v at version %v...", builderConfig.ReleaseParams.ReleaseName, builderConfig.BuildVersion.Version)
		} else {
			log.Info().Msgf("Starting build version %v...", builderConfig.BuildVersion.Version)
		}

		err = envvarHelper.setEstafetteStagesEnvvar(stages)
		if err != nil {
			endOfLifeHelper.handleFatal(ctx, buildLog, err, "Setting ESTAFETTE_STAGES environment variable failed")
		}

		// create docker client
		_, err = dockerRunner.createDockerClient()
		if err != nil {
			endOfLifeHelper.handleFatal(ctx, buildLog, err, "Failed creating a docker client")
		}

		// collect estafette envvars and run stages from manifest
		log.Info().Msgf("Running %v stages", len(stages))
		estafetteEnvvars := envvarHelper.collectEstafetteEnvvarsAndLabels(*builderConfig.Manifest)
		globalEnvvars := envvarHelper.collectGlobalEnvvars(*builderConfig.Manifest)
		envvars := envvarHelper.overrideEnvvars(estafetteEnvvars, globalEnvvars)

		// run stages
		result, err := pipelineRunner.runStages(ctx, stages, dir, envvars)
		if err != nil && !result.canceled {
			endOfLifeHelper.handleFatal(ctx, buildLog, err, "Executing stages from manifest failed")
		}

		// send result to ci-api
		log.Info().Interface("result", result).Msg("Finished running stages")
		buildLog.Steps = transformPipelineRunResultToBuildLogSteps(estafetteEnvvars, result)
		buildStatus := "succeeded"
		if result.HasAggregatedErrors() {
			buildStatus = "failed"
		}
		if result.canceled {
			buildStatus = "canceled"
		}

		_ = endOfLifeHelper.sendBuildFinishedEvent(ctx, buildStatus)
		_ = endOfLifeHelper.sendBuildJobLogEvent(ctx, buildLog)
		_ = endOfLifeHelper.sendBuildCleanEvent(ctx, buildStatus)

		// finish and flush so it gets sent to the tracing backend
		rootSpan.Finish()
		closer.Close()

		if *runAsJob {
			os.Exit(0)
		} else {
			handleExit(result)
		}

	} else {
		// Set up a simple console logger
		log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().
			Timestamp().
			Logger()

		log.Warn().Msgf("The CI Server (\"%s\") is not recognized, exiting.", ciServer)
	}
}

// initJaeger returns an instance of Jaeger Tracer that can be configured with environment variables
// https://github.com/jaegertracing/jaeger-client-go#environment-variables
func initJaeger(service string) io.Closer {

	cfg, err := jaegercfg.FromEnv()
	if err != nil {
		log.Fatal().Err(err).Msg("Generating Jaeger config from environment variables failed")
	}

	closer, err := cfg.InitGlobalTracer(service, jaegercfg.Logger(jaeger.StdLogger))

	if err != nil {
		log.Fatal().Err(err).Msg("Generating Jaeger tracer failed")
	}

	return closer
}
<commit_msg>prepare for reading builder config from mounted file<commit_after>package main

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	stdlog "log"
	"os"
	"os/signal"
	"runtime"
	"syscall"

	"github.com/alecthomas/kingpin"
	contracts "github.com/estafette/estafette-ci-contracts"
	crypt "github.com/estafette/estafette-ci-crypt"
	manifest "github.com/estafette/estafette-ci-manifest"
	"github.com/opentracing/opentracing-go"
	"github.com/rs/zerolog"
	"github.com/rs/zerolog/log"
	"github.com/uber/jaeger-client-go"
	jaegercfg "github.com/uber/jaeger-client-go/config"
)

var (
	app       string
	version   string
	branch    string
	revision  string
	buildDate string
	goVersion = runtime.Version()

	builderConfigFlag         = kingpin.Flag("builder-config", "The Estafette server passes in this json structure to parameterize the build, set trusted images and inject credentials.").Envar("BUILDER_CONFIG").String()
	builderConfigPath         = kingpin.Flag("builder-config-path", "The path to the builder config json stored in a mounted file, to parameterize the build, set trusted images and inject credentials.").Envar("BUILDER_CONFIG_PATH").String()
	secretDecryptionKey       = kingpin.Flag("secret-decryption-key", "The AES-256 key used to decrypt secrets that have been encrypted with it.").Envar("SECRET_DECRYPTION_KEY").String()
	secretDecryptionKeyBase64 = kingpin.Flag("secret-decryption-key-base64", "The base64 encoded AES-256 key used to decrypt secrets that have been encrypted with it.").Envar("SECRET_DECRYPTION_KEY_BASE64").String()
	runAsJob                  = kingpin.Flag("run-as-job", "To run the builder as a job and prevent build failures to fail the job.").Default("false").OverrideDefaultFromEnvar("RUN_AS_JOB").Bool()
)

func main() {

	// parse command line parameters
	kingpin.Parse()

	// define channel to catch SIGTERM and send out cancellation to stop further execution of stages and send the final state and logs to the ci server
	osSignals := make(chan os.Signal, 1)
	signal.Notify(osSignals, os.Interrupt, syscall.SIGTERM)
	cancellationChannel := make(chan struct{})
	go func(osSignals chan os.Signal, cancellationChannel chan struct{}) {
		// wait for sigterm
		<-osSignals
		// broadcast a cancellation
		close(cancellationChannel)
	}(osSignals, cancellationChannel)

	// support both base64 encoded decryption key and non-encoded
	secretDecryptionKeyBase64Encoded := *secretDecryptionKeyBase64 != ""
	decryptionKey := *secretDecryptionKey
	if secretDecryptionKeyBase64Encoded {
		decryptionKey = *secretDecryptionKeyBase64
	}

	secretHelper := crypt.NewSecretHelper(decryptionKey, secretDecryptionKeyBase64Encoded)

	// read builder config either from file or envvar
	var builderConfigJSON []byte

	if *builderConfigPath != "" {

		log.Info().Msgf("Reading builder config from file %v...", *builderConfigPath)

		var err error
		builderConfigJSON, err = ioutil.ReadFile(*builderConfigPath)
		if err != nil {
			log.Fatal().Err(err).Interface("builderConfigJSON", builderConfigJSON).Msgf("Failed to read builder config file at %v", *builderConfigPath)
		}

	} else if *builderConfigFlag != "" {

		log.Info().Msg("Reading builder config from envvar BUILDER_CONFIG...")

		builderConfigJSON = []byte(*builderConfigFlag)
		os.Unsetenv("BUILDER_CONFIG")

	} else {

		log.Fatal().Msg("Neither BUILDER_CONFIG_PATH nor BUILDER_CONFIG envvar is not set; one of them is required")

	}

	// unmarshal builder config
	var builderConfig contracts.BuilderConfig
	err := json.Unmarshal(builderConfigJSON, &builderConfig)
	if err != nil {
		log.Fatal().Err(err).Interface("builderConfigJSON", builderConfigJSON).Msg("Failed to unmarshal builder config")
	}

	// decrypt all credentials
	decryptedCredentials := []*contracts.CredentialConfig{}
	for _, c := range builderConfig.Credentials {

		// loop all additional properties and decrypt
		decryptedAdditionalProperties := map[string]interface{}{}
		for key, value := range c.AdditionalProperties {
			if s, isString := value.(string); isString {
				decryptedAdditionalProperties[key], err = secretHelper.DecryptAllEnvelopes(s)
				if err != nil {
					log.Fatal().Err(err).Msgf("Failed decrypting credential %v property %v", c.Name, key)
				}
			} else {
				decryptedAdditionalProperties[key] = value
			}
		}
		c.AdditionalProperties = decryptedAdditionalProperties

		decryptedCredentials = append(decryptedCredentials, c)
	}
	builderConfig.Credentials = decryptedCredentials

	// bootstrap
	obfuscator := NewObfuscator(secretHelper)
	envvarHelper := NewEnvvarHelper("ESTAFETTE_", secretHelper, obfuscator)
	whenEvaluator := NewWhenEvaluator(envvarHelper)
	dockerRunner := NewDockerRunner(envvarHelper, obfuscator, *runAsJob, builderConfig, cancellationChannel)
	pipelineRunner := NewPipelineRunner(envvarHelper, whenEvaluator, dockerRunner, *runAsJob, cancellationChannel)
	endOfLifeHelper := NewEndOfLifeHelper(*runAsJob, builderConfig)

	// detect controlling server
	ciServer := envvarHelper.getCiServer()

	if ciServer == "estafette" {
		// unset all ESTAFETTE_ envvars so they don't get abused by non-estafette components
		envvarHelper.unsetEstafetteEnvvars()
	}

	if ciServer == "gocd" {

		fatalHandler := NewGocdFatalHandler()

		// pretty print for go.cd integration
		log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().
			Timestamp().
			Logger()

		stdlog.SetFlags(0)
		stdlog.SetOutput(log.Logger)

		// log startup message
		log.Info().
			Str("branch", branch).
			Str("revision", revision).
			Str("buildDate", buildDate).
			Str("goVersion", goVersion).
			Msgf("Starting %v version %v...", app, version)

		// create docker client
		_, err := dockerRunner.createDockerClient()
		if err != nil {
			fatalHandler.handleGocdFatal(err, "Failed creating a docker client")
		}

		// read yaml
		manifest, err := manifest.ReadManifestFromFile(".estafette.yaml")
		if err != nil {
			fatalHandler.handleGocdFatal(err, "Reading .estafette.yaml manifest failed")
		}

		// initialize obfuscator
		err = obfuscator.CollectSecrets(manifest)
		if err != nil {
			fatalHandler.handleGocdFatal(err, "Collecting secrets to obfuscate failed")
		}

		// get current working directory
		dir, err := os.Getwd()
		if err != nil {
			fatalHandler.handleGocdFatal(err, "Getting current working directory failed")
		}

		log.Info().Msgf("Running %v stages", len(manifest.Stages))

		err = envvarHelper.setEstafetteGlobalEnvvars()
		if err != nil {
			fatalHandler.handleGocdFatal(err, "Setting global environment variables failed")
		}
		err = envvarHelper.setEstafetteStagesEnvvar(manifest.Stages)
		if err != nil {
			fatalHandler.handleGocdFatal(err, "Setting ESTAFETTE_STAGES environment variable failed")
		}

		// collect estafette and 'global' envvars from manifest
		estafetteEnvvars := envvarHelper.collectEstafetteEnvvarsAndLabels(manifest)
		globalEnvvars := envvarHelper.collectGlobalEnvvars(manifest)

		// merge estafette and global envvars
		envvars := envvarHelper.overrideEnvvars(estafetteEnvvars, globalEnvvars)

		// run stages
		result, err := pipelineRunner.runStages(context.Background(), manifest.Stages, dir, envvars)
		if err != nil {
			fatalHandler.handleGocdFatal(err, "Executing stages from manifest failed")
		}

		renderStats(result)

		handleExit(result)

	} else if ciServer == "estafette" {

		// log as severity for stackdriver logging to recognize the level
		zerolog.LevelFieldName = "severity"

		closer := initJaeger(app)
		defer closer.Close()

		envvarHelper.setEstafetteBuilderConfigEnvvars(builderConfig)

		buildLog := contracts.BuildLog{
			RepoSource:   builderConfig.Git.RepoSource,
			RepoOwner:    builderConfig.Git.RepoOwner,
			RepoName:     builderConfig.Git.RepoName,
			RepoBranch:   builderConfig.Git.RepoBranch,
			RepoRevision: builderConfig.Git.RepoRevision,
			Steps:        make([]contracts.BuildLogStep, 0),
		}

		// set some default fields added to all logs
		log.Logger = zerolog.New(os.Stdout).With().
			Timestamp().
			Str("app", app).
			Str("version", version).
			Str("jobName", *builderConfig.JobName).
			Interface("git", builderConfig.Git).
			Logger()

		stdlog.SetFlags(0)
		stdlog.SetOutput(log.Logger)

		// log startup message
		log.Info().
			Str("branch", branch).
			Str("revision", revision).
			Str("buildDate", buildDate).
			Str("goVersion", goVersion).
			Msgf("Starting %v version %v...", app, version)

		rootSpanName := "RunBuildJob"
		if *builderConfig.Action == "release" {
			rootSpanName = "RunReleaseJob"
		}

		rootSpan := opentracing.StartSpan(rootSpanName)
		defer rootSpan.Finish()

		ctx := context.Background()
		ctx = opentracing.ContextWithSpan(ctx, rootSpan)

		// start docker daemon
		dockerDaemonStartSpan, _ := opentracing.StartSpanFromContext(ctx, "StartDockerDaemon")
		err = dockerRunner.startDockerDaemon()
		if err != nil {
			endOfLifeHelper.handleFatal(ctx, buildLog, err, "Error starting docker daemon")
		}

		// wait for docker daemon to be ready for usage
		dockerRunner.waitForDockerDaemon()
		dockerDaemonStartSpan.Finish()

		// listen to cancellation in order to stop any running pipeline or container
		go pipelineRunner.stopPipelineOnCancellation()
		go dockerRunner.stopContainerOnCancellation()

		// get current working directory
		dir, err := os.Getwd()
		if err != nil {
			endOfLifeHelper.handleFatal(ctx, buildLog, err, "Getting current working directory failed")
		}

		// set some envvars
		err = envvarHelper.setEstafetteGlobalEnvvars()
		if err != nil {
			endOfLifeHelper.handleFatal(ctx, buildLog, err, "Setting global environment variables failed")
		}

		// initialize obfuscator
		err = obfuscator.CollectSecrets(*builderConfig.Manifest)
		if err != nil {
			endOfLifeHelper.handleFatal(ctx, buildLog, err, "Collecting secrets to obfuscate failed")
		}

		// check whether this is a regular build or a release
		stages := builderConfig.Manifest.Stages
		if *builderConfig.Action == "release" {
			// check if the release is defined
			releaseExists := false
			for _, r := range builderConfig.Manifest.Releases {
				if r.Name == builderConfig.ReleaseParams.ReleaseName {
					releaseExists = true
					stages = r.Stages
				}
			}
			if !releaseExists {
				endOfLifeHelper.handleFatal(ctx, buildLog, nil, fmt.Sprintf("Release %v does not exist", builderConfig.ReleaseParams.ReleaseName))
			}
			log.Info().Msgf("Starting release %v at version %v...", builderConfig.ReleaseParams.ReleaseName, builderConfig.BuildVersion.Version)
		} else {
			log.Info().Msgf("Starting build version %v...", builderConfig.BuildVersion.Version)
		}

		err = envvarHelper.setEstafetteStagesEnvvar(stages)
		if err != nil {
			endOfLifeHelper.handleFatal(ctx, buildLog, err, "Setting ESTAFETTE_STAGES environment variable failed")
		}

		// create docker client
		_, err = dockerRunner.createDockerClient()
		if err != nil {
			endOfLifeHelper.handleFatal(ctx, buildLog, err, "Failed creating a docker client")
		}

		// collect estafette envvars and run stages from manifest
		log.Info().Msgf("Running %v stages", len(stages))
		estafetteEnvvars := envvarHelper.collectEstafetteEnvvarsAndLabels(*builderConfig.Manifest)
		globalEnvvars := envvarHelper.collectGlobalEnvvars(*builderConfig.Manifest)
		envvars := envvarHelper.overrideEnvvars(estafetteEnvvars, globalEnvvars)

		// run stages
		result, err := pipelineRunner.runStages(ctx, stages, dir, envvars)
		if err != nil && !result.canceled {
			endOfLifeHelper.handleFatal(ctx, buildLog, err, "Executing stages from manifest failed")
		}

		// send result to ci-api
		log.Info().Interface("result", result).Msg("Finished running stages")
		buildLog.Steps = transformPipelineRunResultToBuildLogSteps(estafetteEnvvars, result)
		buildStatus := "succeeded"
		if result.HasAggregatedErrors() {
			buildStatus = "failed"
		}
		if result.canceled {
			buildStatus = "canceled"
		}

		_ = endOfLifeHelper.sendBuildFinishedEvent(ctx, buildStatus)
		_ = endOfLifeHelper.sendBuildJobLogEvent(ctx, buildLog)
		_ = endOfLifeHelper.sendBuildCleanEvent(ctx, buildStatus)

		// finish and flush so it gets sent to the tracing backend
		rootSpan.Finish()
		closer.Close()

		if *runAsJob {
			os.Exit(0)
		} else {
			handleExit(result)
		}

	} else {
		// Set up a simple console logger
		log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().
			Timestamp().
			Logger()

		log.Warn().Msgf("The CI Server (\"%s\") is not recognized, exiting.", ciServer)
	}
}

// initJaeger returns an instance of Jaeger Tracer that can be configured with environment variables
// https://github.com/jaegertracing/jaeger-client-go#environment-variables
func initJaeger(service string) io.Closer {

	cfg, err := jaegercfg.FromEnv()
	if err != nil {
		log.Fatal().Err(err).Msg("Generating Jaeger config from environment variables failed")
	}

	closer, err := cfg.InitGlobalTracer(service, jaegercfg.Logger(jaeger.StdLogger))

	if err != nil {
		log.Fatal().Err(err).Msg("Generating Jaeger tracer failed")
	}

	return closer
}
<|endoftext|>


<commit_before>package dns

// Implementation of TSIG: generation and validation
// RFC 2845 and RFC 4635
import (
	"io"
	"strconv"
	"strings"
	"crypto/hmac"
	"encoding/hex"
)

// HMAC hashing codes. These are transmitted as domain names.
const (
	HmacMD5    = "hmac-md5.sig-alg.reg.int"
	HmacSHA1   = "hmac-sha1"
	HmacSHA256 = "hmac-sha256"
)

type RR_TSIG struct {
	Hdr        RR_Header
	Algorithm  string "domain-name"
	TimeSigned uint64
	Fudge      uint16
	MACSize    uint16
	MAC        string "size-hex"
	OrigId     uint16
	Error      uint16
	OtherLen   uint16
	OtherData  string "size-hex"
}

func (rr *RR_TSIG) Header() *RR_Header {
	return &rr.Hdr
}

func (rr *RR_TSIG) Defaults() {
        rr.Header().Ttl = 0
        rr.Header().Class = ClassANY
        rr.Header().Rrtype = TypeTSIG
        rr.Fudge = 300
}

// TSIG has no official presentation format, but this will suffice.
func (rr *RR_TSIG) String() string {
	return rr.Hdr.String() +
		" " + rr.Algorithm +
		" " + tsigTimeToDate(rr.TimeSigned) +
		" " + strconv.Itoa(int(rr.Fudge)) +
		" " + strconv.Itoa(int(rr.MACSize)) +
		" " + rr.MAC +
		" " + strconv.Itoa(int(rr.OrigId)) +
		" " + strconv.Itoa(int(rr.Error)) +
		" " + strconv.Itoa(int(rr.OtherLen)) +
		" " + rr.OtherData
}

// The following values must be put in wireformat, so that the MAC can be calculated.
// RFC 2845, section 3.4.2. TSIG Variables.
type tsigWireFmt struct {
	// From RR_HEADER
	Name  string "domain-name"
	Class uint16
	Ttl   uint32
	// Rdata of the TSIG
	Algorithm  string "domain-name"
	TimeSigned uint64
	Fudge      uint16
	// MACSize, MAC and OrigId excluded
	Error     uint16
	OtherLen  uint16
	OtherData string "size-hex"
}

// Generate the HMAC for message. The TSIG RR is modified
// to include the MAC and MACSize. Note the the msg Id must
// already be set, otherwise the MAC will not be correct when
// the message is send.
// The string 'secret' must be encoded in base64.
func (t *RR_TSIG) Generate(m *Msg, secret string) bool {
	rawsecret, err := packBase64([]byte(secret))
        if err != nil {
                return false
        }
	t.OrigId = m.MsgHdr.Id

	buf, ok := tsigToBuf(t, m)
	h := hmac.NewMD5([]byte(rawsecret))
	io.WriteString(h, string(buf))

	t.MAC = strings.ToUpper(hex.EncodeToString(h.Sum()))
	t.MACSize = uint16(len(h.Sum()))        // Needs to be "on-the-wire" size.
	if !ok {
		return false
	}
	return true
}

// Verify a TSIG. The message should be the complete with
// the TSIG record still attached (as the last rr in the Additional
// section). Return true on success.
// The secret is a base64 encoded string with the secret.
func (t *RR_TSIG) Verify(m *Msg, secret string) bool {
	// copy the mesg, strip (and check) the tsig rr
	// perform the opposite of Generate() and then
	// verify the mac
	rawsecret, err := packBase64([]byte(secret))
        if err != nil {
                return false
        }

	msg2 := m // TODO deep copy TODO(mg)
	if len(msg2.Extra) < 1 {
		// nothing in additional
		return false
	}
        if t.Header().Rrtype != TypeTSIG {
                return false
        }
        msg2.MsgHdr.Id = t.OrigId
        msg2.Extra = msg2.Extra[:len(msg2.Extra)-1]     // Strip off the TSIG
        buf, ok := tsigToBuf(t, msg2)
        if !ok {
                return false
        }
        h := hmac.NewMD5([]byte(rawsecret))
        io.WriteString(h, string(buf))
        return strings.ToUpper(hex.EncodeToString(h.Sum())) == t.MAC
}

func tsigToBuf(rr *RR_TSIG, msg *Msg) ([]byte, bool) {
	// Fill the struct and generate the wiredata
	buf := make([]byte, DefaultMsgSize) // TODO(mg) bufsize!
	tsig := new(tsigWireFmt)
	tsig.Name = rr.Header().Name
	tsig.Class = rr.Header().Class
	tsig.Ttl = rr.Header().Ttl
	tsig.Algorithm = rr.Algorithm
	tsig.TimeSigned = rr.TimeSigned
	tsig.Fudge = rr.Fudge
	tsig.Error = rr.Error
	tsig.OtherLen = rr.OtherLen
	tsig.OtherData = rr.OtherData
	n, ok1 := packStruct(tsig, buf, 0)
	if !ok1 {
		return nil, false
	}
	buf = buf[:n]
	msgbuf, ok := msg.Pack()
	if !ok {
		return nil, false
	}
        // First the pkg, then the tsig wire fmt
	buf = append(msgbuf, buf...)
	return buf, true
}
<commit_msg>rename it<commit_after>package dns

// Implementation of TSIG: generation and validation
// RFC 2845 and RFC 4635
import (
	"io"
	"strconv"
	"strings"
	"crypto/hmac"
	"encoding/hex"
)

// HMAC hashing codes. These are transmitted as domain names.
const (
	HmacMD5    = "hmac-md5.sig-alg.reg.int"
	HmacSHA1   = "hmac-sha1"
	HmacSHA256 = "hmac-sha256"
)

type RR_TSIG struct {
	Hdr        RR_Header
	Algorithm  string "domain-name"
	TimeSigned uint64
	Fudge      uint16
	MACSize    uint16
	MAC        string "size-hex"
	OrigId     uint16
	Error      uint16
	OtherLen   uint16
	OtherData  string "size-hex"
}

func (rr *RR_TSIG) Header() *RR_Header {
	return &rr.Hdr
}

func (rr *RR_TSIG) SetDefaults() {
        rr.Header().Ttl = 0
        rr.Header().Class = ClassANY
        rr.Header().Rrtype = TypeTSIG
        rr.Fudge = 300
}

// TSIG has no official presentation format, but this will suffice.
func (rr *RR_TSIG) String() string {
	return rr.Hdr.String() +
		" " + rr.Algorithm +
		" " + tsigTimeToDate(rr.TimeSigned) +
		" " + strconv.Itoa(int(rr.Fudge)) +
		" " + strconv.Itoa(int(rr.MACSize)) +
		" " + rr.MAC +
		" " + strconv.Itoa(int(rr.OrigId)) +
		" " + strconv.Itoa(int(rr.Error)) +
		" " + strconv.Itoa(int(rr.OtherLen)) +
		" " + rr.OtherData
}

// The following values must be put in wireformat, so that the MAC can be calculated.
// RFC 2845, section 3.4.2. TSIG Variables.
type tsigWireFmt struct {
	// From RR_HEADER
	Name  string "domain-name"
	Class uint16
	Ttl   uint32
	// Rdata of the TSIG
	Algorithm  string "domain-name"
	TimeSigned uint64
	Fudge      uint16
	// MACSize, MAC and OrigId excluded
	Error     uint16
	OtherLen  uint16
	OtherData string "size-hex"
}

// Generate the HMAC for message. The TSIG RR is modified
// to include the MAC and MACSize. Note the the msg Id must
// already be set, otherwise the MAC will not be correct when
// the message is send.
// The string 'secret' must be encoded in base64.
func (t *RR_TSIG) Generate(m *Msg, secret string) bool {
	rawsecret, err := packBase64([]byte(secret))
        if err != nil {
                return false
        }
	t.OrigId = m.MsgHdr.Id

	buf, ok := tsigToBuf(t, m)
	h := hmac.NewMD5([]byte(rawsecret))
	io.WriteString(h, string(buf))

	t.MAC = strings.ToUpper(hex.EncodeToString(h.Sum()))
	t.MACSize = uint16(len(h.Sum()))        // Needs to be "on-the-wire" size.
	if !ok {
		return false
	}
	return true
}

// Verify a TSIG. The message should be the complete with
// the TSIG record still attached (as the last rr in the Additional
// section). Return true on success.
// The secret is a base64 encoded string with the secret.
func (t *RR_TSIG) Verify(m *Msg, secret string) bool {
	// copy the mesg, strip (and check) the tsig rr
	// perform the opposite of Generate() and then
	// verify the mac
	rawsecret, err := packBase64([]byte(secret))
        if err != nil {
                return false
        }

	msg2 := m // TODO deep copy TODO(mg)
	if len(msg2.Extra) < 1 {
		// nothing in additional
		return false
	}
        if t.Header().Rrtype != TypeTSIG {
                return false
        }
        msg2.MsgHdr.Id = t.OrigId
        msg2.Extra = msg2.Extra[:len(msg2.Extra)-1]     // Strip off the TSIG
        buf, ok := tsigToBuf(t, msg2)
        if !ok {
                return false
        }
        h := hmac.NewMD5([]byte(rawsecret))
        io.WriteString(h, string(buf))
        return strings.ToUpper(hex.EncodeToString(h.Sum())) == t.MAC
}

func tsigToBuf(rr *RR_TSIG, msg *Msg) ([]byte, bool) {
	// Fill the struct and generate the wiredata
	buf := make([]byte, DefaultMsgSize) // TODO(mg) bufsize!
	tsig := new(tsigWireFmt)
	tsig.Name = rr.Header().Name
	tsig.Class = rr.Header().Class
	tsig.Ttl = rr.Header().Ttl
	tsig.Algorithm = rr.Algorithm
	tsig.TimeSigned = rr.TimeSigned
	tsig.Fudge = rr.Fudge
	tsig.Error = rr.Error
	tsig.OtherLen = rr.OtherLen
	tsig.OtherData = rr.OtherData
	n, ok1 := packStruct(tsig, buf, 0)
	if !ok1 {
		return nil, false
	}
	buf = buf[:n]
	msgbuf, ok := msg.Pack()
	if !ok {
		return nil, false
	}
        // First the pkg, then the tsig wire fmt
	buf = append(msgbuf, buf...)
	return buf, true
}
<|endoftext|>


<commit_before>/*
 Copyright 2015 Patrick Moroney

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package main

import (
	"encoding/csv"
	"flag"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"github.com/quipo/statsd"
)

func getStats() {
	file, err := ioutil.TempFile("", "sar")
	if err != nil {
		log.Print(err)
		return
	}
	defer func() {
		err = os.Remove(file.Name())
		if err != nil {
			log.Print(err)
		}
	}()
	err = exec.Command("sar", "1", "1", "-A", "-o", file.Name()).Run()
	if err != nil {
		log.Print(err)
		return
	}
	cmd := exec.Command("sadf", "1", "1", "--", "-A", file.Name())
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		log.Print(err)
		return
	}
	if err := cmd.Start(); err != nil {
		log.Print(err)
		return
	}
	csv := csv.NewReader(stdout)
	csv.Comma = '	'
	records, err := csv.ReadAll()
	if err != nil {
		log.Print(err)
		return
	}
	if err := cmd.Wait(); err != nil {
		log.Print(err)
		return
	}
	for _, record := range records {
		value, err := strconv.ParseFloat(record[5], 64)
		if err != nil {
			//log.Print(err)
			continue
		}
		metric := record[4]
		if metric[:2] == "kb" {
			metric = metric[2:]
			value = value * 1024
		} else if len(metric) > 3 && metric[2:4] == "kB" {
			metric = metric[:2] + "bit" + metric[4:]
			value = value * 1024 * 8
		}
		if record[3] != "-" && record[3] != "all" {
			metric = record[3] + "." + metric
		}
		metric = strings.Replace(metric, "%", "pct_", -1)
		sd.FGauge(metric, value)
		//fmt.Print(metric, "=", value, "\n")
	}

}

var statsdServer = flag.String("d", ":8125", "Destination statsd server address")
var statsdPrefix = flag.String("p", "sar", "Statsd prefix for metrics")
var interval = flag.Duration("i", 10*time.Second, "Interval to send metrics")
var sd = statsd.NewStatsdClient(*statsdServer, *statsdPrefix+".")

func main() {
	flag.Parse()
	err := sd.CreateSocket()
	if err != nil {
		log.Fatal(err)
	}
	c := time.Tick(*interval)
	for _ = range c {
		go getStats()
	}
}
<commit_msg>Some cleanups<commit_after>/*
 Copyright 2015 Patrick Moroney

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package main

import (
	"encoding/csv"
	"flag"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"github.com/quipo/statsd"
)

func getStats() {
	file, err := ioutil.TempFile("", "sar")
	if err != nil {
		log.Print(err)
		return
	}

	defer func() {
		err = os.Remove(file.Name())
		if err != nil {
			log.Print(err)
		}
	}()

	err = exec.Command("sar", "1", "1", "-A", "-o", file.Name()).Run()
	if err != nil {
		log.Print(err)
		return
	}

	cmd := exec.Command("sadf", "1", "1", "--", "-A", file.Name())
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		log.Print(err)
		return
	}

	if err := cmd.Start(); err != nil {
		log.Print(err)
		return
	}

	csv := csv.NewReader(stdout)
	csv.Comma = '	'
	records, err := csv.ReadAll()
	if err != nil {
		log.Print(err)
		return
	}

	if err := cmd.Wait(); err != nil {
		log.Print(err)
		return
	}

	for _, record := range records {
		_ = sendRecord(record)
	}
}

func sendRecord(record []string) error {
	value, err := strconv.ParseFloat(record[5], 64)
	if err != nil {
		// We aren't interested in non-numeric values
		return err
	}

	metric := record[4]
	if metric[:2] == "kb" {
		// if metric is in kilobits, convert it into bits
		metric = metric[2:]
		value = value * 1024
	} else if len(metric) > 3 && metric[2:4] == "kB" {
		// if metric is in kilobytes, convert it into bits
		metric = metric[:2] + "bit" + metric[4:]
		value = value * 1024 * 8
	}

	if record[3] != "-" && record[3] != "all" {
		// add device name to metric
		metric = record[3] + "." + metric
	}

	metric = strings.Replace(metric, "%", "pct_", -1)
	sd.FGauge(metric, value)
	return nil
}

var statsdServer = flag.String("d", ":8125", "Destination statsd server address")
var statsdPrefix = flag.String("p", "sar", "Statsd prefix for metrics")
var interval = flag.Duration("i", 10*time.Second, "Interval to send metrics")
var sd = statsd.NewStatsdClient(*statsdServer, *statsdPrefix+".")

func main() {
	flag.Parse()
	err := sd.CreateSocket()
	if err != nil {
		log.Fatal(err)
	}
	c := time.Tick(*interval)
	for _ = range c {
		go getStats()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/jawher/mow.cli"
	"os"
)

const Version = "0.0.1"

// Singleton config object
var config *Config

func main() {
	app := cli.App("mesos-cli", "Alternative Apache Mesos CLI")
	app.Spec = "[OPTIONS]"
	app.Command("cat", "Output the contents of a file", cat)
	app.Command("local", "Launch a local Mesos cluster (requires Docker)", local)
	app.Command("ls", "List the sandbox directory of a task", ls)
	app.Command("ps", "List currently running tasks on a cluster", ps)
	app.Command("run", "Run arbitrary commands against a cluster", run)

	var (
		profile    = app.StringOpt("profile", "default", "Profile to load")
		configPath = app.StringOpt("config", fmt.Sprintf("%s/.mesos-cli.json", homeDir()),
			"Path to load config from")
		level = app.IntOpt("level", 0, "Level of verbosity")
		err   error
	)

	// This is done to satisfy the presumptuous golang/glog package
	// which assumes I am using flag and insists it be configured
	// with such. Since glog is used in go-mesos it is easiest to use
	// the same library for the moment.
	flag.CommandLine.Set("v", string(*level))
	flag.CommandLine.Set("logtostderr", "1")
	flag.CommandLine.Parse([]string{})

	app.Before = func() {
		config, err = LoadConfig(*configPath, *profile)
		failOnErr(err)
	}
	app.Run(os.Args)
}
<commit_msg>add version command<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/jawher/mow.cli"
	"os"
)

const Version = "undefined"

// Singleton config object
var config *Config

func main() {
	app := cli.App("mesos-cli", "Alternative Apache Mesos CLI")
	app.Spec = "[OPTIONS]"
	app.Command("cat", "Output the contents of a file", cat)
	app.Command("local", "Launch a local Mesos cluster (requires Docker)", local)
	app.Command("ls", "List the sandbox directory of a task", ls)
	app.Command("ps", "List currently running tasks on a cluster", ps)
	app.Command("run", "Run arbitrary commands against a cluster", run)

	var (
		profile    = app.StringOpt("profile", "default", "Profile to load")
		configPath = app.StringOpt("config", fmt.Sprintf("%s/.mesos-cli.json", homeDir()),
			"Path to load config from")
		level = app.IntOpt("level", 0, "Level of verbosity")
		err   error
	)

	app.Version("version", Version)

	// This is done to satisfy the presumptuous golang/glog package
	// which assumes I am using flag and insists it be configured
	// with such. Since glog is used in go-mesos it is easiest to use
	// the same library for the moment.
	flag.CommandLine.Set("v", string(*level))
	flag.CommandLine.Set("logtostderr", "1")
	flag.CommandLine.Parse([]string{})

	app.Before = func() {
		config, err = LoadConfig(*configPath, *profile)
		failOnErr(err)
	}
	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"net"
	"net/http"
	"sync"

	"github.com/ant0ine/go-json-rest/rest"
)

func main() {
	var port string
	flag.StringVar(&port, "port", "9999", "port to listen")
	flag.Parse()

	log.Println("Starting server on localhost:" + port)

	countries := Countries{
		Store: map[string]*Country{},
	}

	api := rest.NewApi()
	api.Use(rest.DefaultDevStack...)
	router, err := rest.MakeRouter(
		rest.Get("/", func(w rest.ResponseWriter, r *rest.Request) {
			if e := w.WriteJson(map[string]string{"Body": "Hello, World!"}); e != nil {
				log.Print(e)
			}
		}),
		rest.Get("/lookup/#host", func(w rest.ResponseWriter, r *rest.Request) {
			ip, e := net.LookupIP(r.PathParam("host"))
			if e != nil {
				rest.Error(w, e.Error(), http.StatusInternalServerError)
				return
			}
			if e := w.WriteJson(&ip); e != nil {
				log.Println(e)
			}
		}),
		rest.Get("/countries", countries.GetAllCountries),
		rest.Post("/countries", countries.PostCountry),
		rest.Get("/countries/:code", countries.GetCountry),
		rest.Delete("/countries/:code", countries.DeleteCountry),
	)
	if err != nil {
		log.Fatal(err)
	}
	api.SetApp(router)
	log.Fatal(http.ListenAndServe(":"+port, api.MakeHandler()))
}

// Country is a country.
type Country struct {
	Code string
	Name string
}

// Countries is a collection of countries.
type Countries struct {
	sync.RWMutex
	Store map[string]*Country
}

// GetCountry returns a country corresponding the given country code.
func (c *Countries) GetCountry(w rest.ResponseWriter, r *rest.Request) {
	code := r.PathParam("code")

	c.RLock()
	var country *Country
	if c.Store[code] != nil {
		country = &Country{}
		*country = *c.Store[code]
	}
	c.RUnlock()

	if country == nil {
		rest.NotFound(w, r)
		return
	}
	returnJSON(w, &country)
}

// GetAllCountries returns all countries registered.
func (c *Countries) GetAllCountries(w rest.ResponseWriter, r *rest.Request) {
	c.RLock()

	countries := make([]Country, len(c.Store))
	i := 0
	for _, country := range c.Store {
		countries[i] = *country
		i++
	}
	c.RUnlock()
	returnJSON(w, countries)
}

// PostCountry adds a posted code and countriy.
func (c *Countries) PostCountry(w rest.ResponseWriter, r *rest.Request) {
	country := Country{}
	if e := r.DecodeJsonPayload(&country); e != nil {
		rest.Error(w, e.Error(), http.StatusInternalServerError)
		return
	}

	if country.Code == "" {
		rest.Error(w, "country code required", 400)
		return
	}

	if country.Name == "" {
		rest.Error(w, "country name required", 400)
		return
	}

	c.Lock()
	c.Store[country.Code] = &country
	c.Unlock()

	returnJSON(w, &country)
}

// DeleteCountry deletes a specified country.
func (c *Countries) DeleteCountry(w rest.ResponseWriter, r *rest.Request) {
	code := r.PathParam("code")
	c.Lock()
	delete(c.Store, code)
	c.Unlock()
	w.WriteHeader(http.StatusOK)
}

func returnJSON(w rest.ResponseWriter, v interface{}) {
	if e := w.WriteJson(v); e != nil {
		log.Println(e)
	}
}
<commit_msg>main: add status middleware<commit_after>package main

import (
	"flag"
	"log"
	"net"
	"net/http"
	"sync"

	"github.com/ant0ine/go-json-rest/rest"
)

func main() {
	var port string
	flag.StringVar(&port, "port", "9999", "port to listen")
	flag.Parse()

	log.Println("Starting server on localhost:" + port)

	countries := Countries{
		Store: map[string]*Country{},
	}

	api := rest.NewApi()
	statusMW := &rest.StatusMiddleware{}
	api.Use(statusMW)
	api.Use(rest.DefaultDevStack...)
	router, err := rest.MakeRouter(
		rest.Get("/", func(w rest.ResponseWriter, r *rest.Request) {
			if e := w.WriteJson(map[string]string{"Body": "Hello, World!"}); e != nil {
				log.Println(e)
			}
		}),
		rest.Get("/lookup/#host", func(w rest.ResponseWriter, r *rest.Request) {
			ip, e := net.LookupIP(r.PathParam("host"))
			if e != nil {
				rest.Error(w, e.Error(), http.StatusInternalServerError)
				return
			}
			if e := w.WriteJson(&ip); e != nil {
				log.Println(e)
			}
		}),
		rest.Get("/countries", countries.GetAllCountries),
		rest.Post("/countries", countries.PostCountry),
		rest.Get("/countries/:code", countries.GetCountry),
		rest.Delete("/countries/:code", countries.DeleteCountry),
		rest.Get("/stats", func(w rest.ResponseWriter, r *rest.Request) {
			returnJSON(w, statusMW.GetStatus())
		}),
	)
	if err != nil {
		log.Fatal(err)
	}

	http.Handle("/api/", http.StripPrefix("/api", api.MakeHandler()))
	http.Handle("/static/", http.StripPrefix("/static", http.FileServer(http.Dir("."))))

	api.SetApp(router)

	log.Fatal(http.ListenAndServe(":"+port, api.MakeHandler()))
}

// Country is a country.
type Country struct {
	Code string
	Name string
}

// Countries is a collection of countries.
type Countries struct {
	sync.RWMutex
	Store map[string]*Country
}

// GetCountry returns a country corresponding the given country code.
func (c *Countries) GetCountry(w rest.ResponseWriter, r *rest.Request) {
	code := r.PathParam("code")

	c.RLock()
	var country *Country
	if c.Store[code] != nil {
		country = &Country{}
		*country = *c.Store[code]
	}
	c.RUnlock()

	if country == nil {
		rest.NotFound(w, r)
		return
	}
	returnJSON(w, &country)
}

// GetAllCountries returns all countries registered.
func (c *Countries) GetAllCountries(w rest.ResponseWriter, r *rest.Request) {
	c.RLock()

	countries := make([]Country, len(c.Store))
	i := 0
	for _, country := range c.Store {
		countries[i] = *country
		i++
	}
	c.RUnlock()
	returnJSON(w, countries)
}

// PostCountry adds a posted code and countriy.
func (c *Countries) PostCountry(w rest.ResponseWriter, r *rest.Request) {
	country := Country{}
	if e := r.DecodeJsonPayload(&country); e != nil {
		rest.Error(w, e.Error(), http.StatusInternalServerError)
		return
	}

	if country.Code == "" {
		rest.Error(w, "country code required", 400)
		return
	}

	if country.Name == "" {
		rest.Error(w, "country name required", 400)
		return
	}

	c.Lock()
	c.Store[country.Code] = &country
	c.Unlock()

	returnJSON(w, &country)
}

// DeleteCountry deletes a specified country.
func (c *Countries) DeleteCountry(w rest.ResponseWriter, r *rest.Request) {
	code := r.PathParam("code")
	c.Lock()
	delete(c.Store, code)
	c.Unlock()
	w.WriteHeader(http.StatusOK)
}

func returnJSON(w rest.ResponseWriter, v interface{}) {
	if e := w.WriteJson(v); e != nil {
		log.Println(e)
	}
}
<|endoftext|>


<commit_before>package web

import "strings"

const (
	defaultHTTPSuccess int = 200
	defaultHTTPError   int = 400
)

// Controller interface
type Controller interface {
	Index(ctx *Context)
	Create(ctx *Context)
	Detail(ctx *Context)
	Update(ctx *Context)
	Destroy(ctx *Context)
}

// Middleware struct
type Middleware struct {
	Path     string
	Callback Callback
}

// Middlewares Middleware list
type Middlewares []Middleware

func (m Middlewares) exec(path string, ctx *Context) {
	for i := range m {
		if strings.HasPrefix(path, m[i].Path) {
			if m[i].Callback != nil {
				m[i].Callback(ctx)
			}
		}
	}
}

// Validation interface
type Validation interface {
	Validate() error
}

// responseData struct
type responseData struct {
	Success bool        `json:"success"`
	Code    int         `json:"code"`
	Result  interface{} `json:"result"`
	Error   error       `json:"error"`
}

// ValidationError struct
type ValidationError struct {
	Name    string `json:"name"`
	Message string `json:"message"`
}

func (r *ValidationError) Error() string {
	return r.Message
}

// Error struct
type Error struct {
	Message string `json:"message"`
}

func (r *Error) Error() string {
	return r.Message
}
<commit_msg>clean code<commit_after>package web

import "strings"

const (
	defaultHTTPSuccess int = 200
	defaultHTTPError   int = 400
)

// Controller interface
type Controller interface {
	Index(ctx *Context)
	Create(ctx *Context)
	Detail(ctx *Context)
	Update(ctx *Context)
	Destroy(ctx *Context)
}

// Middleware struct
type Middleware struct {
	Path     string
	Callback Callback
}

// Middlewares Middleware list
type Middlewares []Middleware

func (m Middlewares) exec(path string, ctx *Context) {
	for i := range m {
		if strings.HasPrefix(path, m[i].Path) {
			if m[i].Callback != nil {
				m[i].Callback(ctx)
			}
		}
	}
}

// responseData struct
type responseData struct {
	Success bool        `json:"success"`
	Code    int         `json:"code"`
	Result  interface{} `json:"result"`
	Error   error       `json:"error"`
}

// ValidationError struct
type ValidationError struct {
	Name    string `json:"name"`
	Message string `json:"message"`
}

func (r *ValidationError) Error() string {
	return r.Message
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"time"

	log "github.com/Sirupsen/logrus"
	"gopkg.in/alecthomas/kingpin.v2"

	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/pkg/api/unversioned"
	"k8s.io/client-go/pkg/api/v1"
	"k8s.io/client-go/pkg/apis/extensions/v1beta1"
	"k8s.io/client-go/pkg/labels"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"

	"github.com/linki/chaoskube/chaoskube"
	"github.com/linki/chaoskube/util"
)

const (
	appName = "chaoskube"
	image   = "quay.io/linki/chaoskube"
	version = "v0.5.0"
)

var (
	labelString string
	annString   string
	nsString    string
	kubeconfig  string
	interval    time.Duration
	inCluster   bool
	deploy      bool
	dryRun      bool
	debug       bool
)

func init() {
	kingpin.Flag("labels", "A set of labels to restrict the list of affected pods. Defaults to everything.").Default(labels.Everything().String()).StringVar(&labelString)
	kingpin.Flag("annotations", "A set of annotations to restrict the list of affected pods. Defaults to everything.").Default(labels.Everything().String()).StringVar(&annString)
	kingpin.Flag("namespaces", "A set of namespaces to restrict the list of affected pods. Defaults to everything.").Default(v1.NamespaceAll).StringVar(&nsString)
	kingpin.Flag("kubeconfig", "Path to a kubeconfig file").Default(clientcmd.RecommendedHomeFile).StringVar(&kubeconfig)
	kingpin.Flag("interval", "Interval between Pod terminations").Short('i').Default("10m").DurationVar(&interval)
	kingpin.Flag("in-cluster", "If true, finds the Kubernetes cluster from the environment").Short('c').BoolVar(&inCluster)
	kingpin.Flag("deploy", "If true, deploys chaoskube in the current cluster with the provided configuration").Short('d').BoolVar(&deploy)
	kingpin.Flag("dry-run", "If true, don't actually do anything.").Default("true").BoolVar(&dryRun)
	kingpin.Flag("debug", "Enable debug logging.").BoolVar(&debug)
}

func main() {
	kingpin.Version(version)
	kingpin.Parse()

	if debug {
		log.SetLevel(log.DebugLevel)
	}

	if dryRun {
		log.Infof("Dry run enabled. I won't kill anything. Use --no-dry-run when you're ready.")
	}

	client, err := newClient()
	if err != nil {
		log.Fatal(err)
	}

	if deploy {
		log.Debugf("Deploying %s:%s", image, version)

		manifest := generateManifest()

		deployment := client.Extensions().Deployments(manifest.Namespace)

		_, err := deployment.Get(manifest.Name)
		if err != nil {
			_, err = deployment.Create(manifest)
		} else {
			_, err = deployment.Update(manifest)
		}
		if err != nil {
			log.Fatal(err)
		}

		log.Infof("Deployed %s:%s", image, version)
		os.Exit(0)
	}

	labelSelector, err := labels.Parse(labelString)
	if err != nil {
		log.Fatal(err)
	}

	if !labelSelector.Empty() {
		log.Infof("Filtering pods by labels: %s", labelSelector.String())
	}

	annotations, err := labels.Parse(annString)
	if err != nil {
		log.Fatal(err)
	}

	if !annotations.Empty() {
		log.Infof("Filtering pods by annotations: %s", annotations.String())
	}

	namespaces, err := labels.Parse(nsString)
	if err != nil {
		log.Fatal(err)
	}

	if !namespaces.Empty() {
		log.Infof("Filtering pods by namespaces: %s", namespaces.String())
	}

	chaoskube := chaoskube.New(client, labelSelector, annotations, namespaces, log.StandardLogger(), dryRun, time.Now().UTC().UnixNano())

	for {
		if err := chaoskube.TerminateVictim(); err != nil {
			log.Fatal(err)
		}

		log.Debugf("Sleeping for %s...", interval)
		time.Sleep(interval)
	}
}

func newClient() (*kubernetes.Clientset, error) {
	var (
		config *rest.Config
		err    error
	)

	if inCluster {
		config, err = rest.InClusterConfig()
		log.Debug("Using in-cluster config.")
	} else {
		config, err = clientcmd.BuildConfigFromFlags("", kubeconfig)
		log.Debugf("Using current context from kubeconfig at %s.", kubeconfig)
	}
	if err != nil {
		return nil, err
	}
	log.Infof("Targeting cluster at %s", config.Host)

	client, err := kubernetes.NewForConfig(config)
	if err != nil {
		return nil, err
	}

	return client, nil
}

func generateManifest() *v1beta1.Deployment {
	// modifies flags for deployment
	args := append(os.Args[1:], "--in-cluster")
	args = util.StripElements(args, "--kubeconfig", "--deploy")

	return &v1beta1.Deployment{
		TypeMeta: unversioned.TypeMeta{
			APIVersion: "extensions/v1beta1",
			Kind:       "Deployment",
		},
		ObjectMeta: v1.ObjectMeta{
			Name:      appName,
			Namespace: v1.NamespaceDefault,
			Labels: map[string]string{
				"app":      appName,
				"heritage": appName,
			},
		},
		Spec: v1beta1.DeploymentSpec{
			Template: v1.PodTemplateSpec{
				ObjectMeta: v1.ObjectMeta{
					Labels: map[string]string{
						"app": appName,
					},
				},
				Spec: v1.PodSpec{
					Containers: []v1.Container{
						v1.Container{
							Name:  appName,
							Image: image + ":" + version,
							Args:  args,
						},
					},
				},
			},
		},
	}
}
<commit_msg>feat: drop --in-cluster flag for cluster auto-detection<commit_after>package main

import (
	"os"
	"time"

	log "github.com/Sirupsen/logrus"
	"gopkg.in/alecthomas/kingpin.v2"

	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/pkg/api/unversioned"
	"k8s.io/client-go/pkg/api/v1"
	"k8s.io/client-go/pkg/apis/extensions/v1beta1"
	"k8s.io/client-go/pkg/labels"
	"k8s.io/client-go/tools/clientcmd"

	"github.com/linki/chaoskube/chaoskube"
	"github.com/linki/chaoskube/util"
)

const (
	appName = "chaoskube"
	image   = "quay.io/linki/chaoskube"
	version = "v0.5.0"
)

var (
	labelString string
	annString   string
	nsString    string
	master      string
	kubeconfig  string
	interval    time.Duration
	inCluster   bool
	deploy      bool
	dryRun      bool
	debug       bool
)

func init() {
	kingpin.Flag("labels", "A set of labels to restrict the list of affected pods. Defaults to everything.").Default(labels.Everything().String()).StringVar(&labelString)
	kingpin.Flag("annotations", "A set of annotations to restrict the list of affected pods. Defaults to everything.").Default(labels.Everything().String()).StringVar(&annString)
	kingpin.Flag("namespaces", "A set of namespaces to restrict the list of affected pods. Defaults to everything.").Default(v1.NamespaceAll).StringVar(&nsString)
	kingpin.Flag("master", "The address of the Kubernetes cluster to target").StringVar(&master)
	kingpin.Flag("kubeconfig", "Path to a kubeconfig file").StringVar(&kubeconfig)
	kingpin.Flag("interval", "Interval between Pod terminations").Short('i').Default("10m").DurationVar(&interval)
	kingpin.Flag("deploy", "If true, deploys chaoskube in the current cluster with the provided configuration").Short('d').BoolVar(&deploy)
	kingpin.Flag("dry-run", "If true, don't actually do anything.").Default("true").BoolVar(&dryRun)
	kingpin.Flag("debug", "Enable debug logging.").BoolVar(&debug)
}

func main() {
	kingpin.Version(version)
	kingpin.Parse()

	if debug {
		log.SetLevel(log.DebugLevel)
	}

	if dryRun {
		log.Infof("Dry run enabled. I won't kill anything. Use --no-dry-run when you're ready.")
	}

	client, err := newClient()
	if err != nil {
		log.Fatal(err)
	}

	if deploy {
		log.Debugf("Deploying %s:%s", image, version)

		manifest := generateManifest()

		deployment := client.Extensions().Deployments(manifest.Namespace)

		_, err := deployment.Get(manifest.Name)
		if err != nil {
			_, err = deployment.Create(manifest)
		} else {
			_, err = deployment.Update(manifest)
		}
		if err != nil {
			log.Fatal(err)
		}

		log.Infof("Deployed %s:%s", image, version)
		os.Exit(0)
	}

	labelSelector, err := labels.Parse(labelString)
	if err != nil {
		log.Fatal(err)
	}

	if !labelSelector.Empty() {
		log.Infof("Filtering pods by labels: %s", labelSelector.String())
	}

	annotations, err := labels.Parse(annString)
	if err != nil {
		log.Fatal(err)
	}

	if !annotations.Empty() {
		log.Infof("Filtering pods by annotations: %s", annotations.String())
	}

	namespaces, err := labels.Parse(nsString)
	if err != nil {
		log.Fatal(err)
	}

	if !namespaces.Empty() {
		log.Infof("Filtering pods by namespaces: %s", namespaces.String())
	}

	chaoskube := chaoskube.New(client, labelSelector, annotations, namespaces, log.StandardLogger(), dryRun, time.Now().UTC().UnixNano())

	for {
		if err := chaoskube.TerminateVictim(); err != nil {
			log.Fatal(err)
		}

		log.Debugf("Sleeping for %s...", interval)
		time.Sleep(interval)
	}
}

func newClient() (*kubernetes.Clientset, error) {
	if kubeconfig == "" {
		if _, err := os.Stat(clientcmd.RecommendedHomeFile); err == nil {
			kubeconfig = clientcmd.RecommendedHomeFile
		}
	}

	config, err := clientcmd.BuildConfigFromFlags(master, kubeconfig)
	if err != nil {
		return nil, err
	}

	log.Infof("Targeting cluster at %s", config.Host)

	client, err := kubernetes.NewForConfig(config)
	if err != nil {
		return nil, err
	}

	return client, nil
}

func generateManifest() *v1beta1.Deployment {
	// modifies flags for deployment
	args := append(os.Args[1:], "--in-cluster")
	args = util.StripElements(args, "--kubeconfig", "--deploy")

	return &v1beta1.Deployment{
		TypeMeta: unversioned.TypeMeta{
			APIVersion: "extensions/v1beta1",
			Kind:       "Deployment",
		},
		ObjectMeta: v1.ObjectMeta{
			Name:      appName,
			Namespace: v1.NamespaceDefault,
			Labels: map[string]string{
				"app":      appName,
				"heritage": appName,
			},
		},
		Spec: v1beta1.DeploymentSpec{
			Template: v1.PodTemplateSpec{
				ObjectMeta: v1.ObjectMeta{
					Labels: map[string]string{
						"app": appName,
					},
				},
				Spec: v1.PodSpec{
					Containers: []v1.Container{
						v1.Container{
							Name:  appName,
							Image: image + ":" + version,
							Args:  args,
						},
					},
				},
			},
		},
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
)

type column struct {
	parent string
	name   string
}

func newColumn(name string) *column {
	return &column{parent: "", name: name}
}

type tuple struct {
	values []interface{}
}

func newTuple(vals []interface{}) *tuple {
	return &tuple{values: vals}
}

type relation struct {
	columns []*column
	tuples  []*tuple
}

func (r *relation) String() string {
	var buf bytes.Buffer
	for _, c := range r.columns {
		buf.WriteByte('|')
		if c.parent != "" {
			buf.WriteString(c.parent)
			buf.WriteByte('.')
		}
		buf.WriteString(c.name)
	}
	buf.WriteString("|\n")
	for _, t := range r.tuples {
		for _, v := range t.values {
			buf.WriteByte('|')
			buf.WriteString(fmt.Sprint(v))
		}
		buf.WriteString("|\n")
	}
	return buf.String()
}

func (r *relation) findColumn(name string) int {
	for i, c := range r.columns {
		if c.name == name {
			return i
		}
	}
	return -1
}

func main() {

}
<commit_msg>Define tables and relevant functions<commit_after>package main

import (
	"bytes"
	"fmt"
)

type column struct {
	parent string
	name   string
}

func newColumn(name string) *column {
	return &column{parent: "", name: name}
}

type tuple struct {
	values []interface{}
}

func newTuple(vals []interface{}) *tuple {
	return &tuple{values: vals}
}

type relation struct {
	columns []*column
	tuples  []*tuple
}

func (r *relation) String() string {
	var buf bytes.Buffer
	for _, c := range r.columns {
		buf.WriteByte('|')
		if c.parent != "" {
			buf.WriteString(c.parent)
			buf.WriteByte('.')
		}
		buf.WriteString(c.name)
	}
	buf.WriteString("|\n")
	for _, t := range r.tuples {
		for _, v := range t.values {
			buf.WriteByte('|')
			buf.WriteString(fmt.Sprint(v))
		}
		buf.WriteString("|\n")
	}
	return buf.String()
}

func (r *relation) findColumn(name string) int {
	for i, c := range r.columns {
		if c.name == name {
			return i
		}
	}
	return -1
}

type table struct {
	relation
	name string
}

func newTable(name string, cols []*column) *table {
	var t *table
	t.name = name
	t.columns = cols
	t.tuples = []*tuple{}
	return t
}

func create(name string, colNames []string) *table {
	cols := []*column{}
	for _, cn := range colNames {
		cols = append(cols, newColumn(cn))
	}
	// FIXME: register table to a global dictionary
	return newTable(name, cols)
}

func (t *table) insert(vals []interface{}) *table {
	t.tuples = append(t.tuples, newTuple(vals))
	return t
}

func main() {

}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"

	"github.com/waltzofpearls/relay-api/rapi"
)

func main() {
	var configPath string

	flag.StringVar(&configPath, "c", "config.json", "Path to config file")
	flag.Parse()
	flag.Visit(func(v *flag.Flag) {
		fmt.Printf("%s - %s: %s\n", v.Usage, v.Name, v.Value)
	})

	config := rapi.NewConfigFile(configPath)
	api := rapi.New(config)

	api.NewEndpoint("GET", "/users")
	api.NewEndpoint("GET", "/invoices").
		TransformResponse(&InvoiceInternal{}, &InvoiceExternal{})
	api.NewEndpoint("GET", "/invoices/{Id:[A-Z0-9]+}").
		InternalPath("/invoices/{{.Id}}").
		TransformResponse(&InvoiceInternal{}, &InvoiceExternal{})

	api.Run()
}
<commit_msg>Added login endpoint with passthru handler<commit_after>package main

import (
	"flag"
	"fmt"

	"github.com/waltzofpearls/relay-api/rapi"
)

func main() {
	var configPath string

	flag.StringVar(&configPath, "c", "config.json", "Path to config file")
	flag.Parse()
	flag.Visit(func(v *flag.Flag) {
		fmt.Printf("%s - %s: %s\n", v.Usage, v.Name, v.Value)
	})

	config := rapi.NewConfigFile(configPath)
	api := rapi.New(config)

	api.NewEndpoint("POST", "/auth/login")
	api.NewEndpoint("GET", "/users")
	api.NewEndpoint("GET", "/invoices").
		TransformResponse(&InvoiceInternal{}, &InvoiceExternal{})
	api.NewEndpoint("GET", "/invoices/{Id:[A-Z0-9]+}").
		InternalPath("/invoices/{{.Id}}").
		TransformResponse(&InvoiceInternal{}, &InvoiceExternal{})

	api.Run()
}
<|endoftext|>


<commit_before>// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
// All rights reserved. Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"encoding/binary"
	"encoding/hex"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/golang/groupcache/lru"
	"github.com/juju/ratelimit"
	"github.com/syncthing/syncthing/discover"
	"github.com/syncthing/syncthing/protocol"
	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/opt"
)

const cacheLimitSeconds = 3600

var (
	lock       sync.Mutex
	queries    = 0
	announces  = 0
	answered   = 0
	limited    = 0
	unknowns   = 0
	debug      = false
	lruSize    = 1024
	limitAvg   = 1
	limitBurst = 10
	limiter    *lru.Cache
)

func main() {
	var listen string
	var timestamp bool
	var statsIntv int
	var statsFile string
	var dbDir string

	flag.StringVar(&listen, "listen", ":22026", "Listen address")
	flag.BoolVar(&debug, "debug", false, "Enable debug output")
	flag.BoolVar(&timestamp, "timestamp", true, "Timestamp the log output")
	flag.IntVar(&statsIntv, "stats-intv", 0, "Statistics output interval (s)")
	flag.StringVar(&statsFile, "stats-file", "/var/discosrv/stats", "Statistics file name")
	flag.IntVar(&lruSize, "limit-cache", lruSize, "Limiter cache entries")
	flag.IntVar(&limitAvg, "limit-avg", limitAvg, "Allowed average package rate, per 10 s")
	flag.IntVar(&limitBurst, "limit-burst", limitBurst, "Allowed burst size, packets")
	flag.StringVar(&dbDir, "db-dir", "/var/discosrv/db", "Database directory")
	flag.Parse()

	limiter = lru.New(lruSize)

	log.SetOutput(os.Stdout)
	if !timestamp {
		log.SetFlags(0)
	}

	addr, _ := net.ResolveUDPAddr("udp", listen)
	conn, err := net.ListenUDP("udp", addr)
	if err != nil {
		log.Fatal(err)
	}

	parentDir := filepath.Dir(dbDir)
	if _, err := os.Stat(parentDir); err != nil && os.IsNotExist(err) {
		err = os.MkdirAll(parentDir, 0755)
		if err != nil {
			log.Fatal(err)
		}
	}

	db, err := leveldb.OpenFile(dbDir, &opt.Options{CachedOpenFiles: 32})
	if err != nil {
		log.Fatal(err)
	}

	statsLog, err := os.OpenFile(statsFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
	if err != nil {
		log.Fatal(err)
	}

	if statsIntv > 0 {
		go logStats(statsLog, statsIntv)
	}

	go clean(statsLog, db)

	var buf = make([]byte, 1024)
	for {
		buf = buf[:cap(buf)]
		n, addr, err := conn.ReadFromUDP(buf)

		if limit(addr) {
			// Rate limit in effect for source
			continue
		}

		if err != nil {
			log.Fatal(err)
		}

		if n < 4 {
			log.Printf("Received short packet (%d bytes)", n)
			continue
		}

		buf = buf[:n]
		magic := binary.BigEndian.Uint32(buf)

		switch magic {
		case discover.AnnouncementMagic:
			handleAnnounceV2(db, addr, buf)

		case discover.QueryMagic:
			handleQueryV2(db, conn, addr, buf)

		default:
			lock.Lock()
			unknowns++
			lock.Unlock()
		}
	}
}

func limit(addr *net.UDPAddr) bool {
	key := addr.IP.String()

	lock.Lock()
	defer lock.Unlock()

	bkt, ok := limiter.Get(key)
	if ok {
		bkt := bkt.(*ratelimit.Bucket)
		if bkt.TakeAvailable(1) != 1 {
			// Rate limit exceeded; ignore packet
			if debug {
				log.Println("Rate limit exceeded for", key)
			}
			limited++
			return true
		}
	} else {
		if debug {
			log.Println("New limiter for", key)
		}
		// One packet per ten seconds average rate, burst ten packets
		limiter.Add(key, ratelimit.NewBucket(10*time.Second/time.Duration(limitAvg), int64(limitBurst)))
	}

	return false
}

func handleAnnounceV2(db *leveldb.DB, addr *net.UDPAddr, buf []byte) {
	var pkt discover.Announce
	err := pkt.UnmarshalXDR(buf)
	if err != nil && err != io.EOF {
		log.Println("AnnounceV2 Unmarshal:", err)
		log.Println(hex.Dump(buf))
		return
	}
	if debug {
		log.Printf("<- %v %#v", addr, pkt)
	}

	lock.Lock()
	announces++
	lock.Unlock()

	ip := addr.IP.To4()
	if ip == nil {
		ip = addr.IP.To16()
	}

	var addrs []address
	now := time.Now().Unix()
	for _, addr := range pkt.This.Addresses {
		tip := addr.IP
		if len(tip) == 0 {
			tip = ip
		}
		addrs = append(addrs, address{
			ip:   tip,
			port: addr.Port,
			seen: now,
		})
	}

	var id protocol.NodeID
	if len(pkt.This.ID) == 32 {
		// Raw node ID
		copy(id[:], pkt.This.ID)
	} else {
		id.UnmarshalText(pkt.This.ID)
	}

	update(db, id, addrs)
}

func handleQueryV2(db *leveldb.DB, conn *net.UDPConn, addr *net.UDPAddr, buf []byte) {
	var pkt discover.Query
	err := pkt.UnmarshalXDR(buf)
	if err != nil {
		log.Println("QueryV2 Unmarshal:", err)
		log.Println(hex.Dump(buf))
		return
	}
	if debug {
		log.Printf("<- %v %#v", addr, pkt)
	}

	var id protocol.NodeID
	if len(pkt.NodeID) == 32 {
		// Raw node ID
		copy(id[:], pkt.NodeID)
	} else {
		id.UnmarshalText(pkt.NodeID)
	}

	lock.Lock()
	queries++
	lock.Unlock()

	addrs := get(db, id)

	now := time.Now().Unix()
	if len(addrs) > 0 {
		ann := discover.Announce{
			Magic: discover.AnnouncementMagic,
			This: discover.Node{
				ID: pkt.NodeID,
			},
		}
		for _, addr := range addrs {
			if now-addr.seen > cacheLimitSeconds {
				continue
			}
			ann.This.Addresses = append(ann.This.Addresses, discover.Address{IP: addr.ip, Port: addr.port})
		}
		if debug {
			log.Printf("-> %v %#v", addr, pkt)
		}

		if len(ann.This.Addresses) == 0 {
			return
		}

		tb := ann.MarshalXDR()
		_, _, err = conn.WriteMsgUDP(tb, nil, addr)
		if err != nil {
			log.Println("QueryV2 response write:", err)
		}

		lock.Lock()
		answered++
		lock.Unlock()
	}
}

func next(intv int) time.Time {
	d := time.Duration(intv) * time.Second
	t0 := time.Now()
	t1 := t0.Add(d).Truncate(d)
	time.Sleep(t1.Sub(t0))
	return t1
}

func logStats(statsLog io.Writer, intv int) {
	for {
		t := next(intv)

		lock.Lock()

		fmt.Fprintf(statsLog, "%d Queries:%d Answered:%d Announces:%d Unknown:%d Limited:%d\n",
			t.Unix(), queries, answered, announces, unknowns, limited)

		queries = 0
		announces = 0
		answered = 0
		limited = 0
		unknowns = 0

		lock.Unlock()
	}
}

func get(db *leveldb.DB, id protocol.NodeID) []address {
	var addrs addressList
	val, err := db.Get(id[:], nil)
	if err == nil {
		addrs.UnmarshalXDR(val)
	}
	return addrs.addresses
}

func update(db *leveldb.DB, id protocol.NodeID, addrs []address) {
	var newAddrs addressList

	val, err := db.Get(id[:], nil)
	if err == nil {
		newAddrs.UnmarshalXDR(val)
	}

nextAddr:
	for _, newAddr := range addrs {
		for i, exAddr := range newAddrs.addresses {
			if bytes.Compare(newAddr.ip, exAddr.ip) == 0 {
				newAddrs.addresses[i] = newAddr
				continue nextAddr
			}
		}
		newAddrs.addresses = append(newAddrs.addresses, newAddr)
	}

	db.Put(id[:], newAddrs.MarshalXDR(), nil)
}

func clean(statsLog io.Writer, db *leveldb.DB) {
	for {
		now := next(cacheLimitSeconds)
		nowSecs := now.Unix()

		var kept, deleted int64
		iter := db.NewIterator(nil, nil)
		for iter.Next() {
			var addrs addressList
			addrs.UnmarshalXDR(iter.Value())

			// Remove expired addresses
			newAddrs := addrs.addresses
			for i := 0; i < len(newAddrs); i++ {
				if nowSecs-newAddrs[i].seen > cacheLimitSeconds {
					newAddrs[i] = newAddrs[len(newAddrs)-1]
					newAddrs = newAddrs[:len(newAddrs)-1]
				}
			}

			// Delete empty records
			if len(newAddrs) == 0 {
				db.Delete(iter.Key(), nil)
				deleted++
				continue
			}

			// Update changed records
			if len(newAddrs) != len(addrs.addresses) {
				addrs.addresses = newAddrs
				db.Put(iter.Key(), addrs.MarshalXDR(), nil)
			}
			kept++
		}
		iter.Release()

		fmt.Fprintf(statsLog, "%d Kept:%d Deleted:%d Took:%0.04fs\n", nowSecs, kept, deleted, time.Since(now).Seconds())
	}
}
<commit_msg>Optionally log unknown packet data (for debugging)<commit_after>// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
// All rights reserved. Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"encoding/binary"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/golang/groupcache/lru"
	"github.com/juju/ratelimit"
	"github.com/syncthing/syncthing/discover"
	"github.com/syncthing/syncthing/protocol"
	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/opt"
)

const cacheLimitSeconds = 3600

var (
	lock       sync.Mutex
	queries    = 0
	announces  = 0
	answered   = 0
	limited    = 0
	unknowns   = 0
	debug      = false
	lruSize    = 1024
	limitAvg   = 1
	limitBurst = 10
	limiter    *lru.Cache
)

func main() {
	var listen string
	var timestamp bool
	var statsIntv int
	var statsFile string
	var unknownFile string
	var dbDir string

	flag.StringVar(&listen, "listen", ":22026", "Listen address")
	flag.BoolVar(&debug, "debug", false, "Enable debug output")
	flag.BoolVar(&timestamp, "timestamp", true, "Timestamp the log output")
	flag.IntVar(&statsIntv, "stats-intv", 0, "Statistics output interval (s)")
	flag.StringVar(&statsFile, "stats-file", "/var/discosrv/stats", "Statistics file name")
	flag.StringVar(&unknownFile, "unknown-file", "", "Unknown packet log file name")
	flag.IntVar(&lruSize, "limit-cache", lruSize, "Limiter cache entries")
	flag.IntVar(&limitAvg, "limit-avg", limitAvg, "Allowed average package rate, per 10 s")
	flag.IntVar(&limitBurst, "limit-burst", limitBurst, "Allowed burst size, packets")
	flag.StringVar(&dbDir, "db-dir", "/var/discosrv/db", "Database directory")
	flag.Parse()

	limiter = lru.New(lruSize)

	log.SetOutput(os.Stdout)
	if !timestamp {
		log.SetFlags(0)
	}

	addr, _ := net.ResolveUDPAddr("udp", listen)
	conn, err := net.ListenUDP("udp", addr)
	if err != nil {
		log.Fatal(err)
	}

	parentDir := filepath.Dir(dbDir)
	if _, err := os.Stat(parentDir); err != nil && os.IsNotExist(err) {
		err = os.MkdirAll(parentDir, 0755)
		if err != nil {
			log.Fatal(err)
		}
	}

	db, err := leveldb.OpenFile(dbDir, &opt.Options{CachedOpenFiles: 32})
	if err != nil {
		log.Fatal(err)
	}

	statsLog, err := os.OpenFile(statsFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
	if err != nil {
		log.Fatal(err)
	}

	var unknownLog io.Writer
	if unknownFile != "" {
		unknownLog, err = os.OpenFile(unknownFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
		if err != nil {
			log.Fatal(err)
		}
	}

	if statsIntv > 0 {
		go logStats(statsLog, statsIntv)
	}

	go clean(statsLog, db)

	var buf = make([]byte, 1024)
	for {
		buf = buf[:cap(buf)]
		n, addr, err := conn.ReadFromUDP(buf)

		if limit(addr) {
			// Rate limit in effect for source
			continue
		}

		if err != nil {
			log.Fatal(err)
		}

		if n < 4 {
			log.Printf("Received short packet (%d bytes)", n)
			continue
		}

		buf = buf[:n]
		magic := binary.BigEndian.Uint32(buf)

		switch magic {
		case discover.AnnouncementMagic:
			err := handleAnnounceV2(db, addr, buf)
			if err != nil && unknownLog != nil {
				fmt.Fprintf(unknownLog, "AE %d %v %x\n", time.Now().Unix(), addr, buf)
			}

		case discover.QueryMagic:
			err := handleQueryV2(db, conn, addr, buf)
			if err != nil && unknownLog != nil {
				fmt.Fprintf(unknownLog, "QE %d %v %x\n", time.Now().Unix(), addr, buf)
			}

		default:
			lock.Lock()
			unknowns++
			lock.Unlock()
			if unknownLog != nil {
				fmt.Fprintf(unknownLog, "UN %d %v %x\n", time.Now().Unix(), addr, buf)
			}
		}
	}
}

func limit(addr *net.UDPAddr) bool {
	key := addr.IP.String()

	lock.Lock()
	defer lock.Unlock()

	bkt, ok := limiter.Get(key)
	if ok {
		bkt := bkt.(*ratelimit.Bucket)
		if bkt.TakeAvailable(1) != 1 {
			// Rate limit exceeded; ignore packet
			if debug {
				log.Println("Rate limit exceeded for", key)
			}
			limited++
			return true
		}
	} else {
		if debug {
			log.Println("New limiter for", key)
		}
		// One packet per ten seconds average rate, burst ten packets
		limiter.Add(key, ratelimit.NewBucket(10*time.Second/time.Duration(limitAvg), int64(limitBurst)))
	}

	return false
}

func handleAnnounceV2(db *leveldb.DB, addr *net.UDPAddr, buf []byte) error {
	var pkt discover.Announce
	err := pkt.UnmarshalXDR(buf)
	if err != nil && err != io.EOF {
		return err
	}
	if debug {
		log.Printf("<- %v %#v", addr, pkt)
	}

	lock.Lock()
	announces++
	lock.Unlock()

	ip := addr.IP.To4()
	if ip == nil {
		ip = addr.IP.To16()
	}

	var addrs []address
	now := time.Now().Unix()
	for _, addr := range pkt.This.Addresses {
		tip := addr.IP
		if len(tip) == 0 {
			tip = ip
		}
		addrs = append(addrs, address{
			ip:   tip,
			port: addr.Port,
			seen: now,
		})
	}

	var id protocol.NodeID
	if len(pkt.This.ID) == 32 {
		// Raw node ID
		copy(id[:], pkt.This.ID)
	} else {
		err = id.UnmarshalText(pkt.This.ID)
		if err != nil {
			return err
		}
	}

	update(db, id, addrs)
	return nil
}

func handleQueryV2(db *leveldb.DB, conn *net.UDPConn, addr *net.UDPAddr, buf []byte) error {
	var pkt discover.Query
	err := pkt.UnmarshalXDR(buf)
	if err != nil {
		return err
	}
	if debug {
		log.Printf("<- %v %#v", addr, pkt)
	}

	var id protocol.NodeID
	if len(pkt.NodeID) == 32 {
		// Raw node ID
		copy(id[:], pkt.NodeID)
	} else {
		err = id.UnmarshalText(pkt.NodeID)
		if err != nil {
			return err
		}
	}

	lock.Lock()
	queries++
	lock.Unlock()

	addrs := get(db, id)

	now := time.Now().Unix()
	if len(addrs) > 0 {
		ann := discover.Announce{
			Magic: discover.AnnouncementMagic,
			This: discover.Node{
				ID: pkt.NodeID,
			},
		}
		for _, addr := range addrs {
			if now-addr.seen > cacheLimitSeconds {
				continue
			}
			ann.This.Addresses = append(ann.This.Addresses, discover.Address{IP: addr.ip, Port: addr.port})
		}
		if debug {
			log.Printf("-> %v %#v", addr, pkt)
		}

		if len(ann.This.Addresses) == 0 {
			return nil
		}

		tb := ann.MarshalXDR()
		_, _, err = conn.WriteMsgUDP(tb, nil, addr)
		if err != nil {
			log.Println("QueryV2 response write:", err)
			return nil
		}

		lock.Lock()
		answered++
		lock.Unlock()
	}
	return nil
}

func next(intv int) time.Time {
	d := time.Duration(intv) * time.Second
	t0 := time.Now()
	t1 := t0.Add(d).Truncate(d)
	time.Sleep(t1.Sub(t0))
	return t1
}

func logStats(statsLog io.Writer, intv int) {
	for {
		t := next(intv)

		lock.Lock()

		fmt.Fprintf(statsLog, "%d Queries:%d Answered:%d Announces:%d Unknown:%d Limited:%d\n",
			t.Unix(), queries, answered, announces, unknowns, limited)

		queries = 0
		announces = 0
		answered = 0
		limited = 0
		unknowns = 0

		lock.Unlock()
	}
}

func get(db *leveldb.DB, id protocol.NodeID) []address {
	var addrs addressList
	val, err := db.Get(id[:], nil)
	if err == nil {
		addrs.UnmarshalXDR(val)
	}
	return addrs.addresses
}

func update(db *leveldb.DB, id protocol.NodeID, addrs []address) {
	var newAddrs addressList

	val, err := db.Get(id[:], nil)
	if err == nil {
		newAddrs.UnmarshalXDR(val)
	}

nextAddr:
	for _, newAddr := range addrs {
		for i, exAddr := range newAddrs.addresses {
			if bytes.Compare(newAddr.ip, exAddr.ip) == 0 {
				newAddrs.addresses[i] = newAddr
				continue nextAddr
			}
		}
		newAddrs.addresses = append(newAddrs.addresses, newAddr)
	}

	db.Put(id[:], newAddrs.MarshalXDR(), nil)
}

func clean(statsLog io.Writer, db *leveldb.DB) {
	for {
		now := next(cacheLimitSeconds)
		nowSecs := now.Unix()

		var kept, deleted int64
		iter := db.NewIterator(nil, nil)
		for iter.Next() {
			var addrs addressList
			addrs.UnmarshalXDR(iter.Value())

			// Remove expired addresses
			newAddrs := addrs.addresses
			for i := 0; i < len(newAddrs); i++ {
				if nowSecs-newAddrs[i].seen > cacheLimitSeconds {
					newAddrs[i] = newAddrs[len(newAddrs)-1]
					newAddrs = newAddrs[:len(newAddrs)-1]
				}
			}

			// Delete empty records
			if len(newAddrs) == 0 {
				db.Delete(iter.Key(), nil)
				deleted++
				continue
			}

			// Update changed records
			if len(newAddrs) != len(addrs.addresses) {
				addrs.addresses = newAddrs
				db.Put(iter.Key(), addrs.MarshalXDR(), nil)
			}
			kept++
		}
		iter.Release()

		fmt.Fprintf(statsLog, "%d Kept:%d Deleted:%d Took:%0.04fs\n", nowSecs, kept, deleted, time.Since(now).Seconds())
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"io"
	"os"

	// Plugins
	"github.com/hansrodtang/runcom/core"
	"github.com/hansrodtang/runcom/plugins"
	_ "github.com/hansrodtang/runcom/plugins/dotfiles"
	_ "github.com/hansrodtang/runcom/plugins/packages/homebrew"
	// Other dependencies
	"github.com/hansrodtang/viper"
	"github.com/spf13/cobra"
	"golang.org/x/crypto/ssh/terminal"
)

const defaultBackend = "directory"

func main() {
	viper.SetDefault("backend", defaultBackend)

	var mainCmd = &cobra.Command{
		Use:   core.BinaryName,
		Short: core.Name + " is a simple and flexible configuration manager",
		Long: `A Simple and Flexible Configuration Manager..
Complete documentation is available at http:/runcom.github.io`,
	}

	mainCmd.PersistentFlags().String("backend", defaultBackend, "Storage backend to use for saving configuration")
	viper.BindPFlag("backend", mainCmd.PersistentFlags().Lookup("backend"))

	plugs := plugins.GetAll()

	for _, plugin := range plugs {
		mainCmd.AddCommand(plugin.Command)
	}

	mainCmd.AddCommand(
		core.ConnectCommand,
		core.PushCommand,
		core.GetCommand,
		core.UnpackCommand,
		core.PackCommand)

	if !terminal.IsTerminal(int(os.Stdout.Fd())) {
		var test io.Writer
		test = bufio.NewWriter(test)
		mainCmd.SetOutput(test)
	}

	mainCmd.Execute()
}
<commit_msg>Added backend flag selection.<commit_after>package main

import (
	"bufio"
	"fmt"
	"io"
	"os"

	"github.com/hansrodtang/runcom/backends"
	"github.com/hansrodtang/runcom/core"
	"github.com/hansrodtang/runcom/plugins"
	// Backend
	_ "github.com/hansrodtang/runcom/backends/directory"
	// Plugins
	_ "github.com/hansrodtang/runcom/plugins/dotfiles"
	_ "github.com/hansrodtang/runcom/plugins/packages/homebrew"
	// Other dependencies
	"github.com/hansrodtang/viper"
	"github.com/spf13/cobra"
	"golang.org/x/crypto/ssh/terminal"
)

const defaultBackend = "directory"

func main() {
	viper.SetDefault("backend", defaultBackend)

	var mainCmd = &cobra.Command{
		Use:   core.BinaryName,
		Short: core.Name + " is a simple and flexible configuration manager",
		Long: `A Simple and Flexible Configuration Manager..
Complete documentation is available at http:/runcom.github.io`,
	}

	mainCmd.PersistentFlags().String("backend", defaultBackend, "Storage backend to use for saving configuration")
	viper.BindPFlag("backend", mainCmd.PersistentFlags().Lookup("backend"))

	print := core.NewPrinter(core.BinaryName)

	backendErr := backends.Select(viper.GetString("backend"))
	if backendErr != nil {
		answer := print.Ask("Backend '"+viper.GetString("backend")+"' not found. Use default? ["+defaultBackend+"]", true)
		if !answer {
			fmt.Println("No storage backend chosen. Exiting")
			os.Exit(0)
		}
		backendErr = backends.Select(defaultBackend)
		if backendErr != nil {
			panic("Could not load default backend.")
		}

	}

	plugs := plugins.GetAll()

	for _, plugin := range plugs {
		mainCmd.AddCommand(plugin.Command)
	}

	mainCmd.AddCommand(
		core.ConnectCommand,
		core.PushCommand,
		core.GetCommand,
		core.UnpackCommand,
		core.PackCommand)

	if !terminal.IsTerminal(int(os.Stdout.Fd())) {
		var test io.Writer
		test = bufio.NewWriter(test)
		mainCmd.SetOutput(test)
	}

	mainCmd.Execute()
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"launchpad.net/goyaml"
	"log"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"runtime"
	"strings"
	"syscall"
)

var (
	upgrade   = flag.Bool("upgrade", false, "force upgrade even if older version exists")
	gopath    = flag.Bool("gopath", false, "use GOPATH from environment instead of downloading all dependencies")
	onlyBuild = flag.Bool("build", false, "only build, do not run command (can pass multiple spec files)")
)

func getCacheDir() (string, error) {
	var cache_dir string
	cache_dir = os.Getenv("DEMAND_CACHE_DIR")
	if cache_dir == "" {
		u, err := user.Current()
		if err != nil {
			return "", fmt.Errorf("cannot determine home directory: %v", err)
		}
		cache_dir = filepath.Join(u.HomeDir, ".cache/demand")
	}
	return cache_dir, nil
}

// Create directories if they don't exist.
func maybeMkdirs(perm os.FileMode, paths ...string) error {
	for _, path := range paths {
		err := os.Mkdir(path, perm)
		if err != nil && !os.IsExist(err) {
			return err
		}
	}
	return nil
}

// copy environment, but override GOPATH
func copyEnvWithGopath(gopath string) []string {
	old := os.Environ()
	env := make([]string, 0, len(old))
	for _, kv := range old {
		if strings.HasPrefix(kv, "GOPATH=") {
			continue
		}
		env = append(env, kv)
	}

	env = append(env, "GOPATH="+gopath)
	return env
}

func runBinary(binary string, args []string, env []string) error {
	// can't use os/exec etc, we want exec not fork+exec, and we want
	// to pass all fds to the child
	return syscall.Exec(binary, args, env)
}

type specification struct {
	Go struct {
		Import string
	}
}

func build(cache_dir, cache_bin_dir, cache_bin_arch_dir string,
	spec_path string, spec_file io.Reader, binary string) error {
	err := maybeMkdirs(0750, cache_dir, cache_bin_dir, cache_bin_arch_dir)
	if err != nil {
		return fmt.Errorf("cannot create cache directory: %v", err)
	}

	var spec specification
	spec_data, err := ioutil.ReadAll(spec_file)
	err = goyaml.Unmarshal(spec_data, &spec)
	if err != nil {
		return fmt.Errorf("cannot parse spec file: %v", err)
	}
	if spec.Go.Import == "" {
		return fmt.Errorf("spec file does not specify import path: %s", spec_path)
	}

	tmp_gopath, err := ioutil.TempDir("", "demand-gopath-")
	if err != nil {
		return fmt.Errorf("cannot create temp directory: %v", err)
	}
	defer func() {
		err := os.RemoveAll(tmp_gopath)
		if err != nil {
			log.Printf("tempdir cleanup failed: %v", err)
		}
	}()

	env_gopath := tmp_gopath
	if *gopath {
		old_gopath := os.Getenv("GOPATH")
		if old_gopath != "" {
			env_gopath = env_gopath + string(filepath.ListSeparator) + old_gopath
		}
	}
	env := copyEnvWithGopath(env_gopath)

	// TODO -upgrade should be handled just by the fact that we have a
	// clean GOPATH, but double check what happens on -gopath -upgrade

	// need to do this in two steps, as "go get" won't let us control
	// destination
	cmd := exec.Command("go", "get", "-d", "--", spec.Go.Import)
	cmd.Dir = tmp_gopath
	cmd.Stdout = os.Stderr
	cmd.Stderr = os.Stderr
	cmd.Env = env
	err = cmd.Run()
	if err != nil {
		return fmt.Errorf("could not get go package: %v", err)
	}

	// poor man's tempfile atomicity; go build complains if
	// destination exists
	tmp_bin := fmt.Sprintf("%s.%d.tmp", binary, os.Getpid())
	defer func() {
		err := os.Remove(tmp_bin)
		if err != nil && !os.IsNotExist(err) {
			log.Printf("temp binary cleanup failed: %v", err)
		}
	}()
	cmd = exec.Command("go", "build", "-o", tmp_bin, "--", spec.Go.Import)
	cmd.Dir = tmp_gopath
	cmd.Stdout = os.Stderr
	cmd.Stderr = os.Stderr
	cmd.Env = env
	err = cmd.Run()
	if err != nil {
		return fmt.Errorf("could not build go package: %v", err)
	}

	err = os.Rename(tmp_bin, binary)
	if err != nil {
		return fmt.Errorf("could put new binary in place: %v", err)
	}
	return nil
}

func doit(args []string) error {
	spec_path := args[0]

	spec_base := filepath.Base(spec_path)
	if spec_base[0] == '.' {
		return fmt.Errorf("refusing to run hidden spec file: %s", spec_path)
	}

	// open it here to guard against typos; we don't need to read
	// until we know it's a cache miss
	spec_file, err := os.Open(spec_path)
	if err != nil {
		return fmt.Errorf("cannot open spec file: %v", err)
	}
	// we rely on Go's automatic use of O_CLOEXEC to close this in
	// syscall.Exec, this defer is a nice gesture for error cases
	defer func() {
		// silence errcheck
		_ = spec_file.Close()
	}()

	cache_dir, err := getCacheDir()
	if err != nil {
		return err
	}
	cache_bin_dir := filepath.Join(cache_dir, "bin")
	arch := fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH)
	cache_bin_arch_dir := filepath.Join(cache_bin_dir, arch)

	binary := filepath.Join(cache_bin_arch_dir, spec_base)

	if !*onlyBuild && !*upgrade {
		err = runBinary(binary, flag.Args(), os.Environ())
		if err != nil && !os.IsNotExist(err) {
			return fmt.Errorf("cannot exec %s: %v", binary, err)
		}
	}

	// if we're still here, we don't have a cached binary, or we're
	// upgrading
	err = build(cache_dir, cache_bin_dir, cache_bin_arch_dir, spec_path, spec_file, binary)
	if err != nil {
		return err
	}

	if !*onlyBuild {
		// now run it (again); this time ENOENT means trouble
		err = runBinary(binary, flag.Args(), os.Environ())
		if err != nil {
			return fmt.Errorf("cannot exec %s: %v", binary, err)
		}
	}
	return nil
}

func dobuild(specs []string) error {
	var err error
	for i, _ := range specs {
		args := specs[i : i+1]
		err = doit(args)
		if err != nil {
			return err
		}
	}
	return nil
}

var prog = filepath.Base(os.Args[0])

func usage() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", prog)
	fmt.Fprintf(os.Stderr, "  %s [OPTS] SPEC_PATH [ARGS..]\n", prog)
	fmt.Fprintf(os.Stderr, "\n")
	fmt.Fprintf(os.Stderr, "Only build, do not run command:\n")
	fmt.Fprintf(os.Stderr, "  %s -build [OPTS] SPEC_PATH..\n", prog)
	fmt.Fprintf(os.Stderr, "\n")
	fmt.Fprintf(os.Stderr, "Options:\n")
	flag.PrintDefaults()
	fmt.Fprintf(os.Stderr, "\n")
	fmt.Fprintf(os.Stderr, "Use as an interpreter:\n")
	fmt.Fprintf(os.Stderr, "  #!/usr/bin/env demand\n")
	fmt.Fprintf(os.Stderr, "  go:\n")
	fmt.Fprintf(os.Stderr, "    import: GO_IMPORT_PATH_HERE\n")
}

func main() {
	log.SetFlags(0)
	log.SetPrefix(prog + ": ")

	flag.Usage = usage
	flag.Parse()
	if flag.NArg() == 0 {
		flag.Usage()
		os.Exit(2)
	}

	var err error
	if *onlyBuild {
		err = dobuild(flag.Args())
	} else {
		err = doit(flag.Args())
	}

	if err != nil {
		log.Print(err)
		os.Exit(1)
	}
}
<commit_msg>golint cleanup<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"launchpad.net/goyaml"
	"log"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"runtime"
	"strings"
	"syscall"
)

var (
	upgrade   = flag.Bool("upgrade", false, "force upgrade even if older version exists")
	gopath    = flag.Bool("gopath", false, "use GOPATH from environment instead of downloading all dependencies")
	onlyBuild = flag.Bool("build", false, "only build, do not run command (can pass multiple spec files)")
)

func getCacheDir() (string, error) {
	var cacheDir string
	cacheDir = os.Getenv("DEMAND_CACHE_DIR")
	if cacheDir == "" {
		u, err := user.Current()
		if err != nil {
			return "", fmt.Errorf("cannot determine home directory: %v", err)
		}
		cacheDir = filepath.Join(u.HomeDir, ".cache/demand")
	}
	return cacheDir, nil
}

// Create directories if they don't exist.
func maybeMkdirs(perm os.FileMode, paths ...string) error {
	for _, path := range paths {
		err := os.Mkdir(path, perm)
		if err != nil && !os.IsExist(err) {
			return err
		}
	}
	return nil
}

// copy environment, but override GOPATH
func copyEnvWithGopath(gopath string) []string {
	old := os.Environ()
	env := make([]string, 0, len(old))
	for _, kv := range old {
		if strings.HasPrefix(kv, "GOPATH=") {
			continue
		}
		env = append(env, kv)
	}

	env = append(env, "GOPATH="+gopath)
	return env
}

func runBinary(binary string, args []string, env []string) error {
	// can't use os/exec etc, we want exec not fork+exec, and we want
	// to pass all fds to the child
	return syscall.Exec(binary, args, env)
}

type specification struct {
	Go struct {
		Import string
	}
}

func build(cacheDir, cacheBinDir, cacheBinArchDir string,
	specPath string, specFile io.Reader, binary string) error {
	err := maybeMkdirs(0750, cacheDir, cacheBinDir, cacheBinArchDir)
	if err != nil {
		return fmt.Errorf("cannot create cache directory: %v", err)
	}

	var spec specification
	specData, err := ioutil.ReadAll(specFile)
	err = goyaml.Unmarshal(specData, &spec)
	if err != nil {
		return fmt.Errorf("cannot parse spec file: %v", err)
	}
	if spec.Go.Import == "" {
		return fmt.Errorf("spec file does not specify import path: %s", specPath)
	}

	tmpGopath, err := ioutil.TempDir("", "demand-gopath-")
	if err != nil {
		return fmt.Errorf("cannot create temp directory: %v", err)
	}
	defer func() {
		err := os.RemoveAll(tmpGopath)
		if err != nil {
			log.Printf("tempdir cleanup failed: %v", err)
		}
	}()

	envGopath := tmpGopath
	if *gopath {
		oldGopath := os.Getenv("GOPATH")
		if oldGopath != "" {
			envGopath = envGopath + string(filepath.ListSeparator) + oldGopath
		}
	}
	env := copyEnvWithGopath(envGopath)

	// TODO -upgrade should be handled just by the fact that we have a
	// clean GOPATH, but double check what happens on -gopath -upgrade

	// need to do this in two steps, as "go get" won't let us control
	// destination
	cmd := exec.Command("go", "get", "-d", "--", spec.Go.Import)
	cmd.Dir = tmpGopath
	cmd.Stdout = os.Stderr
	cmd.Stderr = os.Stderr
	cmd.Env = env
	err = cmd.Run()
	if err != nil {
		return fmt.Errorf("could not get go package: %v", err)
	}

	// poor man's tempfile atomicity; go build complains if
	// destination exists
	tmpBin := fmt.Sprintf("%s.%d.tmp", binary, os.Getpid())
	defer func() {
		err := os.Remove(tmpBin)
		if err != nil && !os.IsNotExist(err) {
			log.Printf("temp binary cleanup failed: %v", err)
		}
	}()
	cmd = exec.Command("go", "build", "-o", tmpBin, "--", spec.Go.Import)
	cmd.Dir = tmpGopath
	cmd.Stdout = os.Stderr
	cmd.Stderr = os.Stderr
	cmd.Env = env
	err = cmd.Run()
	if err != nil {
		return fmt.Errorf("could not build go package: %v", err)
	}

	err = os.Rename(tmpBin, binary)
	if err != nil {
		return fmt.Errorf("could put new binary in place: %v", err)
	}
	return nil
}

func doit(args []string) error {
	specPath := args[0]

	specBase := filepath.Base(specPath)
	if specBase[0] == '.' {
		return fmt.Errorf("refusing to run hidden spec file: %s", specPath)
	}

	// open it here to guard against typos; we don't need to read
	// until we know it's a cache miss
	specFile, err := os.Open(specPath)
	if err != nil {
		return fmt.Errorf("cannot open spec file: %v", err)
	}
	// we rely on Go's automatic use of O_CLOEXEC to close this in
	// syscall.Exec, this defer is a nice gesture for error cases
	defer func() {
		// silence errcheck
		_ = specFile.Close()
	}()

	cacheDir, err := getCacheDir()
	if err != nil {
		return err
	}
	cacheBinDir := filepath.Join(cacheDir, "bin")
	arch := fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH)
	cacheBinArchDir := filepath.Join(cacheBinDir, arch)

	binary := filepath.Join(cacheBinArchDir, specBase)

	if !*onlyBuild && !*upgrade {
		err = runBinary(binary, flag.Args(), os.Environ())
		if err != nil && !os.IsNotExist(err) {
			return fmt.Errorf("cannot exec %s: %v", binary, err)
		}
	}

	// if we're still here, we don't have a cached binary, or we're
	// upgrading
	err = build(cacheDir, cacheBinDir, cacheBinArchDir, specPath, specFile, binary)
	if err != nil {
		return err
	}

	if !*onlyBuild {
		// now run it (again); this time ENOENT means trouble
		err = runBinary(binary, flag.Args(), os.Environ())
		if err != nil {
			return fmt.Errorf("cannot exec %s: %v", binary, err)
		}
	}
	return nil
}

func dobuild(specs []string) error {
	var err error
	for i := range specs {
		args := specs[i : i+1]
		err = doit(args)
		if err != nil {
			return err
		}
	}
	return nil
}

var prog = filepath.Base(os.Args[0])

func usage() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", prog)
	fmt.Fprintf(os.Stderr, "  %s [OPTS] SPEC_PATH [ARGS..]\n", prog)
	fmt.Fprintf(os.Stderr, "\n")
	fmt.Fprintf(os.Stderr, "Only build, do not run command:\n")
	fmt.Fprintf(os.Stderr, "  %s -build [OPTS] SPEC_PATH..\n", prog)
	fmt.Fprintf(os.Stderr, "\n")
	fmt.Fprintf(os.Stderr, "Options:\n")
	flag.PrintDefaults()
	fmt.Fprintf(os.Stderr, "\n")
	fmt.Fprintf(os.Stderr, "Use as an interpreter:\n")
	fmt.Fprintf(os.Stderr, "  #!/usr/bin/env demand\n")
	fmt.Fprintf(os.Stderr, "  go:\n")
	fmt.Fprintf(os.Stderr, "    import: GO_IMPORT_PATH_HERE\n")
}

func main() {
	log.SetFlags(0)
	log.SetPrefix(prog + ": ")

	flag.Usage = usage
	flag.Parse()
	if flag.NArg() == 0 {
		flag.Usage()
		os.Exit(2)
	}

	var err error
	if *onlyBuild {
		err = dobuild(flag.Args())
	} else {
		err = doit(flag.Args())
	}

	if err != nil {
		log.Print(err)
		os.Exit(1)
	}
}
<|endoftext|>