<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"time"

	"github.com/cloudfoundry-community/go-cfclient"
	"github.com/SumoLogic/sumologic-cloudfoundry-nozzle/caching"
	"github.com/SumoLogic/sumologic-cloudfoundry-nozzle/eventQueue"
	"github.com/SumoLogic/sumologic-cloudfoundry-nozzle/eventRouting"
	"github.com/SumoLogic/sumologic-cloudfoundry-nozzle/events"
	"github.com/SumoLogic/sumologic-cloudfoundry-nozzle/firehoseclient"
	"github.com/SumoLogic/sumologic-cloudfoundry-nozzle/logging"
	"github.com/SumoLogic/sumologic-cloudfoundry-nozzle/sumoCFFirehose"
	"gopkg.in/alecthomas/kingpin.v2"
)

var (
	apiEndpoint                 = kingpin.Flag("api-endpoint", "URL to CF API Endpoint").OverrideDefaultFromEnvar("API_ENDPOINT").String()
	sumoEndpoint                = kingpin.Flag("sumo-endpoint", "SUMO-ENDPOINT Complete URL for the endpoint, copied from the Sumo Logic HTTP Source configuration").OverrideDefaultFromEnvar("SUMO_ENDPOINT").String()
	subscriptionId              = kingpin.Flag("subscription-id", "Cloud Foundry ID for the subscription.").Default("firehose").OverrideDefaultFromEnvar("FIREHOSE_SUBSCRIPTION_ID").String()
	user                        = kingpin.Flag("cloudfoundry-user", "Cloud Foundry User").OverrideDefaultFromEnvar("CLOUDFOUNDRY_USER").String() //user created in CF, authorized to connect the firehose
	password                    = kingpin.Flag("cloudfoundry-password", "Cloud Foundry Password").OverrideDefaultFromEnvar("CLOUDFOUNDRY_PASSWORD").String()
	keepAlive		            = time.ParseDuration("25s") //default Error,ContainerMetric,HttpStart,HttpStop,HttpStartStop,LogMessage,ValueMetric,CounterEvent
	wantedEvents                = kingpin.Flag("events", fmt.Sprintf("Comma separated list of events you would like. Valid options are %s", eventRouting.GetListAuthorizedEventEvents())).Default("LogMessage").OverrideDefaultFromEnvar("EVENTS").String()
	boltDatabasePath            = "event.db"
	skipSSLValidation           = kingpin.Flag("skip-ssl-validation", "Skip SSL validation (to allow things like self-signed certs). Do not set to true in production").Default("false").OverrideDefaultFromEnvar("SKIP_SSL_VALIDATION").Bool()
	tickerTime                  = kingpin.Flag("nozzle-polling-period", "How frequently this Nozzle polls the CF Firehose for data").Default("15s").OverrideDefaultFromEnvar("NOZZLE_POLLING_PERIOD").Duration()
	eventsBatchSize             = kingpin.Flag("log-events-batch-size", "When number of messages in the buffer is equal to this flag, send those to Sumo Logic").Default("500").OverrideDefaultFromEnvar("LOG_EVENTS_BATCH_SIZE").Int()
	sumoPostMinimumDelay        = kingpin.Flag("sumo-post-minimum-delay", "Minimum time between HTTP POST to Sumo Logic").Default("2000ms").OverrideDefaultFromEnvar("SUMO_POST_MINIMUM_DELAY").Duration()
	sumoCategory                = kingpin.Flag("sumo-category", "This value overrides the default 'Source Category' associated with the configured Sumo Logic HTTP Source").Default("").OverrideDefaultFromEnvar("SUMO_CATEGORY").String()
	sumoName                    = kingpin.Flag("sumo-name", "This value overrides the default 'Source Name' associated with the configured Sumo Logic HTTP Source").Default("").OverrideDefaultFromEnvar("SUMO_NAME").String()
	sumoHost                    = kingpin.Flag("sumo-host", "This value overrides the default 'Source Host' associated with the configured Sumo Logic HTTP Source").Default("").OverrideDefaultFromEnvar("SUMO_HOST").String()
	verboseLogMessages          = kingpin.Flag("verbose-log-messages", "Enable Verbose in 'LogMessage' Event. If this flag NOT present, the LogMessage will contain ONLY the fields: tiemstamp, cf_app_guid, Msg").Default("false").OverrideDefaultFromEnvar("VERBOSE_LOG_MESSAGES").Bool()
	customMetadata              = kingpin.Flag("custom-metadata", "Use this flag for addingCustom Metadata to the JSON (key1:value1,key2:value2, etc...)").Default("").OverrideDefaultFromEnvar("CUSTOM_METADATA").String()
	includeOnlyMatchingFilter   = kingpin.Flag("include-only-matching-filter", "Adds an 'Include only' filter to Events content (key1:value1,key2:value2, etc...)").Default("").OverrideDefaultFromEnvar("INCLUDE_ONLY_MATCHING_FILTER").String()
	excludeAlwaysMatchingFilter = kingpin.Flag("exclude-always-matching-filter", "Adds an 'Exclude always' filter to Events content (key1:value1,key2:value2, etc...)").Default("").OverrideDefaultFromEnvar("EXCLUDE_ALWAYS_MATCHING_FILTER").String()
)

var (
	version = "0.1.0"
)

func main() {
	//logging init
	logging.Init(ioutil.Discard, os.Stdout, os.Stdout, os.Stderr)

	kingpin.Version(version)
	kingpin.Parse()

	logging.Info.Println("Set Configurations:")
	logging.Info.Println("CF API Endpoint: " + *apiEndpoint)
	logging.Info.Println("Sumo Logic Endpoint: " + *sumoEndpoint)
	logging.Info.Println("Cloud Foundry Nozzle Subscription ID: " + *subscriptionId)
	logging.Info.Println("Cloud Foundry User: " + *user)
	logging.Info.Println("Events Selected: " + *wantedEvents)
	logging.Info.Printf("Skip SSL Validation: %v", *skipSSLValidation)
	logging.Info.Printf("Nozzle Polling Period: %v", *tickerTime)
	logging.Info.Printf("Log Events Batch Size: [%d]", *eventsBatchSize)
	logging.Info.Printf("Sumo Logic HTTP Post Minimum Delay: %v", *sumoPostMinimumDelay)
	if *sumoName != "" {
		logging.Info.Println("Sumo Logic Name: " + *sumoName)
	}
	if *sumoHost != "" {
		logging.Info.Println("Sumo Logic Host: " + *sumoHost)
	}
	if *sumoCategory != "" {
		logging.Info.Println("Sumo Logic Category: " + *sumoCategory)
	}
	logging.Info.Printf("Verbose Log Messages: %v\n", *verboseLogMessages)
	logging.Info.Println("Starting Sumo Logic Nozzle " + version)

	c := cfclient.Config{
		ApiAddress:        *apiEndpoint,
		Username:          *user,
		Password:          *password,
		SkipSslValidation: *skipSSLValidation,
	}
	cfClient, errCfClient := cfclient.NewClient(&c)

	if errCfClient != nil {
		logging.Error.Fatal("Error setting up CF Client: ", errCfClient)
		os.Exit(1)
	}

	//Creating Caching
	var cachingClient caching.Caching
	if caching.IsNeeded(*wantedEvents) {
		cachingClient = caching.NewCachingBolt(cfClient, boltDatabasePath)
	} else {
		cachingClient = caching.NewCachingEmpty()
	}

	logging.Info.Println("Creating queue")
	queue := eventQueue.NewQueue(make([]*events.Event, 100))
	loggingClientSumo := sumoCFFirehose.NewSumoLogicAppender(*sumoEndpoint, 5000, &queue, *eventsBatchSize, *sumoPostMinimumDelay, *sumoCategory, *sumoName, *sumoHost, *verboseLogMessages, *customMetadata, *includeOnlyMatchingFilter, *excludeAlwaysMatchingFilter, version)
	go loggingClientSumo.Start() //multi

	logging.Info.Println("Creating Events")
	events := eventRouting.NewEventRouting(cachingClient, &queue)
	err := events.SetupEventRouting(*wantedEvents)
	if err != nil {
		logging.Error.Fatal("Error setting up event routing: ", err)
		os.Exit(1)

	}

	// Parse extra fields from cmd call
	cachingClient.CreateBucket()
	//Let's Update the database the first time
	logging.Info.Printf("Start filling app/space/org cache.\n")
	apps := cachingClient.GetAllApp()
	logging.Info.Printf("Done filling cache! Found [%d] Apps \n", len(apps))

	logging.Info.Println("Apps found: ")
	for i := 0; i < len(apps); i++ {
		logging.Info.Printf("[%d] "+apps[i].Name+" GUID: "+apps[i].Guid, i+1)
	}
	cachingClient.PerformPoollingCaching(*tickerTime)

	firehoseConfig := &firehoseclient.FirehoseConfig{
		TrafficControllerURL:   cfClient.Endpoint.DopplerEndpoint,
		InsecureSSLSkipVerify:  *skipSSLValidation,
		IdleTimeoutSeconds:     keepAlive,
		FirehoseSubscriptionID: *subscriptionId,
	}

	logging.Info.Printf("Connecting to Firehose... \n")
	firehoseClient := firehoseclient.NewFirehoseNozzle(cfClient, events, firehoseConfig)
	errFirehose := firehoseClient.Start()
	logging.Info.Printf("FirehoseClient Error: %v", errFirehose)
	defer cachingClient.Close()

}
<commit_msg>Amenzhinsky did a pull request to update deprecation notices. doing same in Sumo Logic code<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"time"

	"github.com/cloudfoundry-community/go-cfclient"
	"github.com/SumoLogic/sumologic-cloudfoundry-nozzle/caching"
	"github.com/SumoLogic/sumologic-cloudfoundry-nozzle/eventQueue"
	"github.com/SumoLogic/sumologic-cloudfoundry-nozzle/eventRouting"
	"github.com/SumoLogic/sumologic-cloudfoundry-nozzle/events"
	"github.com/SumoLogic/sumologic-cloudfoundry-nozzle/firehoseclient"
	"github.com/SumoLogic/sumologic-cloudfoundry-nozzle/logging"
	"github.com/SumoLogic/sumologic-cloudfoundry-nozzle/sumoCFFirehose"
	"gopkg.in/alecthomas/kingpin.v2"
)

var (
	apiEndpoint                 = kingpin.Flag("api-endpoint", "URL to CF API Endpoint").Envar("API_ENDPOINT").String()
	sumoEndpoint                = kingpin.Flag("sumo-endpoint", "SUMO-ENDPOINT Complete URL for the endpoint, copied from the Sumo Logic HTTP Source configuration").Envar("SUMO_ENDPOINT").String()
	subscriptionId              = kingpin.Flag("subscription-id", "Cloud Foundry ID for the subscription.").Default("firehose").Envar("FIREHOSE_SUBSCRIPTION_ID").String()
	user                        = kingpin.Flag("cloudfoundry-user", "Cloud Foundry User").Envar("CLOUDFOUNDRY_USER").String() //user created in CF, authorized to connect the firehose
	password                    = kingpin.Flag("cloudfoundry-password", "Cloud Foundry Password").Envar("CLOUDFOUNDRY_PASSWORD").String()
	keepAlive		            = time.ParseDuration("25s") //default Error,ContainerMetric,HttpStart,HttpStop,HttpStartStop,LogMessage,ValueMetric,CounterEvent
	wantedEvents                = kingpin.Flag("events", fmt.Sprintf("Comma separated list of events you would like. Valid options are %s", eventRouting.GetListAuthorizedEventEvents())).Default("LogMessage").Envar("EVENTS").String()
	boltDatabasePath            = "event.db"
	skipSSLValidation           = kingpin.Flag("skip-ssl-validation", "Skip SSL validation (to allow things like self-signed certs). Do not set to true in production").Default("false").Envar("SKIP_SSL_VALIDATION").Bool()
	tickerTime                  = kingpin.Flag("nozzle-polling-period", "How frequently this Nozzle polls the CF Firehose for data").Default("15s").Envar("NOZZLE_POLLING_PERIOD").Duration()
	eventsBatchSize             = kingpin.Flag("log-events-batch-size", "When number of messages in the buffer is equal to this flag, send those to Sumo Logic").Default("500").Envar("LOG_EVENTS_BATCH_SIZE").Int()
	sumoPostMinimumDelay        = kingpin.Flag("sumo-post-minimum-delay", "Minimum time between HTTP POST to Sumo Logic").Default("2000ms").Envar("SUMO_POST_MINIMUM_DELAY").Duration()
	sumoCategory                = kingpin.Flag("sumo-category", "This value overrides the default 'Source Category' associated with the configured Sumo Logic HTTP Source").Default("").Envar("SUMO_CATEGORY").String()
	sumoName                    = kingpin.Flag("sumo-name", "This value overrides the default 'Source Name' associated with the configured Sumo Logic HTTP Source").Default("").Envar("SUMO_NAME").String()
	sumoHost                    = kingpin.Flag("sumo-host", "This value overrides the default 'Source Host' associated with the configured Sumo Logic HTTP Source").Default("").Envar("SUMO_HOST").String()
	verboseLogMessages          = kingpin.Flag("verbose-log-messages", "Enable Verbose in 'LogMessage' Event. If this flag NOT present, the LogMessage will contain ONLY the fields: tiemstamp, cf_app_guid, Msg").Default("false").Envar("VERBOSE_LOG_MESSAGES").Bool()
	customMetadata              = kingpin.Flag("custom-metadata", "Use this flag for addingCustom Metadata to the JSON (key1:value1,key2:value2, etc...)").Default("").Envar("CUSTOM_METADATA").String()
	includeOnlyMatchingFilter   = kingpin.Flag("include-only-matching-filter", "Adds an 'Include only' filter to Events content (key1:value1,key2:value2, etc...)").Default("").Envar("INCLUDE_ONLY_MATCHING_FILTER").String()
	excludeAlwaysMatchingFilter = kingpin.Flag("exclude-always-matching-filter", "Adds an 'Exclude always' filter to Events content (key1:value1,key2:value2, etc...)").Default("").Envar("EXCLUDE_ALWAYS_MATCHING_FILTER").String()
)

var (
	version = "0.1.0"
)

func main() {
	//logging init
	logging.Init(ioutil.Discard, os.Stdout, os.Stdout, os.Stderr)

	kingpin.Version(version)
	kingpin.Parse()

	logging.Info.Println("Set Configurations:")
	logging.Info.Println("CF API Endpoint: " + *apiEndpoint)
	logging.Info.Println("Sumo Logic Endpoint: " + *sumoEndpoint)
	logging.Info.Println("Cloud Foundry Nozzle Subscription ID: " + *subscriptionId)
	logging.Info.Println("Cloud Foundry User: " + *user)
	logging.Info.Println("Events Selected: " + *wantedEvents)
	logging.Info.Printf("Skip SSL Validation: %v", *skipSSLValidation)
	logging.Info.Printf("Nozzle Polling Period: %v", *tickerTime)
	logging.Info.Printf("Log Events Batch Size: [%d]", *eventsBatchSize)
	logging.Info.Printf("Sumo Logic HTTP Post Minimum Delay: %v", *sumoPostMinimumDelay)
	if *sumoName != "" {
		logging.Info.Println("Sumo Logic Name: " + *sumoName)
	}
	if *sumoHost != "" {
		logging.Info.Println("Sumo Logic Host: " + *sumoHost)
	}
	if *sumoCategory != "" {
		logging.Info.Println("Sumo Logic Category: " + *sumoCategory)
	}
	logging.Info.Printf("Verbose Log Messages: %v\n", *verboseLogMessages)
	logging.Info.Println("Starting Sumo Logic Nozzle " + version)

	c := cfclient.Config{
		ApiAddress:        *apiEndpoint,
		Username:          *user,
		Password:          *password,
		SkipSslValidation: *skipSSLValidation,
	}
	cfClient, errCfClient := cfclient.NewClient(&c)

	if errCfClient != nil {
		logging.Error.Fatal("Error setting up CF Client: ", errCfClient)
		os.Exit(1)
	}

	//Creating Caching
	var cachingClient caching.Caching
	if caching.IsNeeded(*wantedEvents) {
		cachingClient = caching.NewCachingBolt(cfClient, boltDatabasePath)
	} else {
		cachingClient = caching.NewCachingEmpty()
	}

	logging.Info.Println("Creating queue")
	queue := eventQueue.NewQueue(make([]*events.Event, 100))
	loggingClientSumo := sumoCFFirehose.NewSumoLogicAppender(*sumoEndpoint, 5000, &queue, *eventsBatchSize, *sumoPostMinimumDelay, *sumoCategory, *sumoName, *sumoHost, *verboseLogMessages, *customMetadata, *includeOnlyMatchingFilter, *excludeAlwaysMatchingFilter, version)
	go loggingClientSumo.Start() //multi

	logging.Info.Println("Creating Events")
	events := eventRouting.NewEventRouting(cachingClient, &queue)
	err := events.SetupEventRouting(*wantedEvents)
	if err != nil {
		logging.Error.Fatal("Error setting up event routing: ", err)
		os.Exit(1)

	}

	// Parse extra fields from cmd call
	cachingClient.CreateBucket()
	//Let's Update the database the first time
	logging.Info.Printf("Start filling app/space/org cache.\n")
	apps := cachingClient.GetAllApp()
	logging.Info.Printf("Done filling cache! Found [%d] Apps \n", len(apps))

	logging.Info.Println("Apps found: ")
	for i := 0; i < len(apps); i++ {
		logging.Info.Printf("[%d] "+apps[i].Name+" GUID: "+apps[i].Guid, i+1)
	}
	cachingClient.PerformPoollingCaching(*tickerTime)

	firehoseConfig := &firehoseclient.FirehoseConfig{
		TrafficControllerURL:   cfClient.Endpoint.DopplerEndpoint,
		InsecureSSLSkipVerify:  *skipSSLValidation,
		IdleTimeoutSeconds:     keepAlive,
		FirehoseSubscriptionID: *subscriptionId,
	}

	logging.Info.Printf("Connecting to Firehose... \n")
	firehoseClient := firehoseclient.NewFirehoseNozzle(cfClient, events, firehoseConfig)
	errFirehose := firehoseClient.Start()
	logging.Info.Printf("FirehoseClient Error: %v", errFirehose)
	defer cachingClient.Close()

}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"runtime"
	"strings"

	"github.com/bgentry/heroku-go"
	flag "github.com/bgentry/pflag"
	"github.com/heroku/hk/hkclient"
	"github.com/heroku/hk/postgresql"
	"github.com/heroku/hk/rollbar"
	"github.com/heroku/hk/term"
	"github.com/mgutz/ansi"
)

var (
	apiURL = "https://api.heroku.com"
	stdin  = bufio.NewReader(os.Stdin)
)

type Command struct {
	// args does not include the command name
	Run      func(cmd *Command, args []string)
	Flag     flag.FlagSet
	NeedsApp bool

	Usage    string // first word is the command name
	Category string // i.e. "App", "Account", etc.
	Short    string // `hk help` output
	Long     string // `hk help cmd` output
}

func (c *Command) PrintUsage() {
	if c.Runnable() {
		fmt.Fprintf(os.Stderr, "Usage: hk %s\n", c.FullUsage())
	}
	fmt.Fprintf(os.Stderr, "Use 'hk help %s' for more information.\n", c.Name())
}

func (c *Command) PrintLongUsage() {
	if c.Runnable() {
		fmt.Printf("Usage: hk %s\n\n", c.FullUsage())
	}
	fmt.Println(strings.Trim(c.Long, "\n"))
}

func (c *Command) FullUsage() string {
	if c.NeedsApp {
		return c.Name() + " [-a <app or remote>]" + strings.TrimPrefix(c.Usage, c.Name())
	}
	return c.Usage
}

func (c *Command) Name() string {
	name := c.Usage
	i := strings.Index(name, " ")
	if i >= 0 {
		name = name[:i]
	}
	return name
}

func (c *Command) Runnable() bool {
	return c.Run != nil
}

const extra = " (extra)"

func (c *Command) List() bool {
	return c.Short != "" && !strings.HasSuffix(c.Short, extra)
}

func (c *Command) ListAsExtra() bool {
	return c.Short != "" && strings.HasSuffix(c.Short, extra)
}

func (c *Command) ShortExtra() string {
	return c.Short[:len(c.Short)-len(extra)]
}

// Running `hk help` will list commands in this order.
var commands = []*Command{
	cmdCreate,
	cmdApps,
	cmdDynos,
	cmdReleases,
	cmdReleaseInfo,
	cmdRollback,
	cmdAddons,
	cmdAddonAdd,
	cmdAddonDestroy,
	cmdScale,
	cmdRestart,
	cmdSet,
	cmdUnset,
	cmdEnv,
	cmdRun,
	cmdLog,
	cmdInfo,
	cmdRename,
	cmdDestroy,
	cmdDomains,
	cmdDomainAdd,
	cmdDomainRemove,
	cmdVersion,
	cmdHelp,

	helpCommands,
	helpEnviron,
	helpPlugins,
	helpMore,
	helpAbout,

	// listed by hk help more
	cmdAccess,
	cmdAccessAdd,
	cmdAccessRemove,
	cmdAccountFeatures,
	cmdAccountFeatureInfo,
	cmdAccountFeatureEnable,
	cmdAccountFeatureDisable,
	cmdAddonOpen,
	cmdAddonPlan,
	cmdAddonPlans,
	cmdAddonServices,
	cmdAPI,
	cmdCreds,
	cmdDrains,
	cmdDrainInfo,
	cmdDrainAdd,
	cmdDrainRemove,
	cmdFeatures,
	cmdFeatureInfo,
	cmdFeatureEnable,
	cmdFeatureDisable,
	cmdGet,
	cmdKeys,
	cmdKeyAdd,
	cmdKeyRemove,
	cmdLogin,
	cmdLogout,
	cmdMaintenance,
	cmdMaintenanceEnable,
	cmdMaintenanceDisable,
	cmdMembers,
	cmdOpen,
	cmdOrgs,
	cmdPgList,
	cmdPgInfo,
	cmdPgUnfollow,
	cmdPsql,
	cmdRegions,
	cmdSSL,
	cmdSSLCertAdd,
	cmdSSLCertRollback,
	cmdSSLDestroy,
	cmdStatus,
	cmdTransfer,
	cmdTransfers,
	cmdTransferAccept,
	cmdTransferDecline,
	cmdTransferCancel,
	cmdURL,
	cmdWhichApp,

	// unlisted
	cmdUpdate,
}

var (
	flagApp   string
	client    *heroku.Client
	pgclient  *postgresql.Client
	hkAgent   = "hk/" + Version + " (" + runtime.GOOS + "; " + runtime.GOARCH + ")"
	userAgent = hkAgent + " " + heroku.DefaultUserAgent
)

func initClients() {
	loadNetrc()
	suite, err := hkclient.New(nrc, hkAgent)
	if err != nil {
		printError(err.Error())
	}

	client = suite.Client
	pgclient = suite.PgClient

}

func main() {
	log.SetFlags(0)

	// make sure command is specified, disallow global args
	args := os.Args[1:]
	if len(args) < 1 || strings.IndexRune(args[0], '-') == 0 {
		printUsageTo(os.Stderr)
		os.Exit(2)
	}

	// Run the update command as early as possible to avoid the possibility of
	// installations being stranded without updates due to errors in other code
	if args[0] == cmdUpdate.Name() {
		cmdUpdate.Run(cmdUpdate, args)
		return
	} else if updater != nil {
		defer updater.backgroundRun() // doesn't run if os.Exit is called
	}

	if !term.IsANSI(os.Stdout) {
		ansi.DisableColors(true)
	}

	initClients()

	for _, cmd := range commands {
		if cmd.Name() == args[0] && cmd.Run != nil {
			defer recoverPanic()

			cmd.Flag.SetDisableDuplicates(true) // disallow duplicate flag options
			if !gitConfigBool("hk.strict-flag-ordering") {
				cmd.Flag.SetInterspersed(true) // allow flags & non-flag args to mix
			}
			cmd.Flag.Usage = func() {
				cmd.PrintUsage()
			}
			if cmd.NeedsApp {
				cmd.Flag.StringVarP(&flagApp, "app", "a", "", "app name")
			}
			if err := cmd.Flag.Parse(args[1:]); err == flag.ErrHelp {
				cmdHelp.Run(cmdHelp, args[:1])
				return
			} else if err != nil {
				printError(err.Error())
				os.Exit(2)
			}
			if flagApp != "" {
				if gitRemoteApp, err := appFromGitRemote(flagApp); err == nil {
					flagApp = gitRemoteApp
				}
			}
			if cmd.NeedsApp {
				a, err := app()
				switch {
				case err == errMultipleHerokuRemotes, err == nil && a == "":
					msg := "no app specified"
					if err != nil {
						msg = err.Error()
					}
					printError(msg)
					cmd.PrintUsage()
					os.Exit(2)
				case err != nil:
					printFatal(err.Error())
				}
			}
			cmd.Run(cmd, cmd.Flag.Args())
			return
		}
	}

	path := findPlugin(args[0])
	if path == "" {
		fmt.Fprintf(os.Stderr, "Unknown command: %s\n", args[0])
		if g := suggest(args[0]); len(g) > 0 {
			fmt.Fprintf(os.Stderr, "Possible alternatives: %v\n", strings.Join(g, " "))
		}
		fmt.Fprintf(os.Stderr, "Run 'hk help' for usage.\n")
		os.Exit(2)
	}
	err := execPlugin(path, args)
	printFatal("exec error: %s", err)
}

var rollbarClient = &rollbar.Client{
	AppName:    "hk",
	AppVersion: Version,
	Endpoint:   "https://api.rollbar.com/api/1/item/",
	Token:      "d344db7a09fa481e983694bfa326e6d9",
}

func recoverPanic() {
	if Version != "dev" {
		if rec := recover(); rec != nil {
			message := ""
			switch rec := rec.(type) {
			case error:
				message = rec.Error()
			default:
				message = fmt.Sprintf("%v", rec)
			}
			if err := rollbarClient.Report(message); err != nil {
				printError("reporting crash failed: %s", err.Error())
				panic(rec)
			}
			printFatal("hk encountered and reported an internal client error")
		}
	}
}

func app() (string, error) {
	if flagApp != "" {
		return flagApp, nil
	}

	if app := os.Getenv("HKAPP"); app != "" {
		return app, nil
	}

	return appFromGitRemote(remoteFromGitConfig())
}

func mustApp() string {
	name, err := app()
	if err != nil {
		printFatal(err.Error())
	}
	return name
}
<commit_msg>printFatal on hkclient.New() error.<commit_after>package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"runtime"
	"strings"

	"github.com/bgentry/heroku-go"
	flag "github.com/bgentry/pflag"
	"github.com/heroku/hk/hkclient"
	"github.com/heroku/hk/postgresql"
	"github.com/heroku/hk/rollbar"
	"github.com/heroku/hk/term"
	"github.com/mgutz/ansi"
)

var (
	apiURL = "https://api.heroku.com"
	stdin  = bufio.NewReader(os.Stdin)
)

type Command struct {
	// args does not include the command name
	Run      func(cmd *Command, args []string)
	Flag     flag.FlagSet
	NeedsApp bool

	Usage    string // first word is the command name
	Category string // i.e. "App", "Account", etc.
	Short    string // `hk help` output
	Long     string // `hk help cmd` output
}

func (c *Command) PrintUsage() {
	if c.Runnable() {
		fmt.Fprintf(os.Stderr, "Usage: hk %s\n", c.FullUsage())
	}
	fmt.Fprintf(os.Stderr, "Use 'hk help %s' for more information.\n", c.Name())
}

func (c *Command) PrintLongUsage() {
	if c.Runnable() {
		fmt.Printf("Usage: hk %s\n\n", c.FullUsage())
	}
	fmt.Println(strings.Trim(c.Long, "\n"))
}

func (c *Command) FullUsage() string {
	if c.NeedsApp {
		return c.Name() + " [-a <app or remote>]" + strings.TrimPrefix(c.Usage, c.Name())
	}
	return c.Usage
}

func (c *Command) Name() string {
	name := c.Usage
	i := strings.Index(name, " ")
	if i >= 0 {
		name = name[:i]
	}
	return name
}

func (c *Command) Runnable() bool {
	return c.Run != nil
}

const extra = " (extra)"

func (c *Command) List() bool {
	return c.Short != "" && !strings.HasSuffix(c.Short, extra)
}

func (c *Command) ListAsExtra() bool {
	return c.Short != "" && strings.HasSuffix(c.Short, extra)
}

func (c *Command) ShortExtra() string {
	return c.Short[:len(c.Short)-len(extra)]
}

// Running `hk help` will list commands in this order.
var commands = []*Command{
	cmdCreate,
	cmdApps,
	cmdDynos,
	cmdReleases,
	cmdReleaseInfo,
	cmdRollback,
	cmdAddons,
	cmdAddonAdd,
	cmdAddonDestroy,
	cmdScale,
	cmdRestart,
	cmdSet,
	cmdUnset,
	cmdEnv,
	cmdRun,
	cmdLog,
	cmdInfo,
	cmdRename,
	cmdDestroy,
	cmdDomains,
	cmdDomainAdd,
	cmdDomainRemove,
	cmdVersion,
	cmdHelp,

	helpCommands,
	helpEnviron,
	helpPlugins,
	helpMore,
	helpAbout,

	// listed by hk help more
	cmdAccess,
	cmdAccessAdd,
	cmdAccessRemove,
	cmdAccountFeatures,
	cmdAccountFeatureInfo,
	cmdAccountFeatureEnable,
	cmdAccountFeatureDisable,
	cmdAddonOpen,
	cmdAddonPlan,
	cmdAddonPlans,
	cmdAddonServices,
	cmdAPI,
	cmdCreds,
	cmdDrains,
	cmdDrainInfo,
	cmdDrainAdd,
	cmdDrainRemove,
	cmdFeatures,
	cmdFeatureInfo,
	cmdFeatureEnable,
	cmdFeatureDisable,
	cmdGet,
	cmdKeys,
	cmdKeyAdd,
	cmdKeyRemove,
	cmdLogin,
	cmdLogout,
	cmdMaintenance,
	cmdMaintenanceEnable,
	cmdMaintenanceDisable,
	cmdMembers,
	cmdOpen,
	cmdOrgs,
	cmdPgList,
	cmdPgInfo,
	cmdPgUnfollow,
	cmdPsql,
	cmdRegions,
	cmdSSL,
	cmdSSLCertAdd,
	cmdSSLCertRollback,
	cmdSSLDestroy,
	cmdStatus,
	cmdTransfer,
	cmdTransfers,
	cmdTransferAccept,
	cmdTransferDecline,
	cmdTransferCancel,
	cmdURL,
	cmdWhichApp,

	// unlisted
	cmdUpdate,
}

var (
	flagApp   string
	client    *heroku.Client
	pgclient  *postgresql.Client
	hkAgent   = "hk/" + Version + " (" + runtime.GOOS + "; " + runtime.GOARCH + ")"
	userAgent = hkAgent + " " + heroku.DefaultUserAgent
)

func initClients() {
	loadNetrc()
	suite, err := hkclient.New(nrc, hkAgent)
	if err != nil {
		printFatal(err.Error())
	}

	client = suite.Client
	pgclient = suite.PgClient

}

func main() {
	log.SetFlags(0)

	// make sure command is specified, disallow global args
	args := os.Args[1:]
	if len(args) < 1 || strings.IndexRune(args[0], '-') == 0 {
		printUsageTo(os.Stderr)
		os.Exit(2)
	}

	// Run the update command as early as possible to avoid the possibility of
	// installations being stranded without updates due to errors in other code
	if args[0] == cmdUpdate.Name() {
		cmdUpdate.Run(cmdUpdate, args)
		return
	} else if updater != nil {
		defer updater.backgroundRun() // doesn't run if os.Exit is called
	}

	if !term.IsANSI(os.Stdout) {
		ansi.DisableColors(true)
	}

	initClients()

	for _, cmd := range commands {
		if cmd.Name() == args[0] && cmd.Run != nil {
			defer recoverPanic()

			cmd.Flag.SetDisableDuplicates(true) // disallow duplicate flag options
			if !gitConfigBool("hk.strict-flag-ordering") {
				cmd.Flag.SetInterspersed(true) // allow flags & non-flag args to mix
			}
			cmd.Flag.Usage = func() {
				cmd.PrintUsage()
			}
			if cmd.NeedsApp {
				cmd.Flag.StringVarP(&flagApp, "app", "a", "", "app name")
			}
			if err := cmd.Flag.Parse(args[1:]); err == flag.ErrHelp {
				cmdHelp.Run(cmdHelp, args[:1])
				return
			} else if err != nil {
				printError(err.Error())
				os.Exit(2)
			}
			if flagApp != "" {
				if gitRemoteApp, err := appFromGitRemote(flagApp); err == nil {
					flagApp = gitRemoteApp
				}
			}
			if cmd.NeedsApp {
				a, err := app()
				switch {
				case err == errMultipleHerokuRemotes, err == nil && a == "":
					msg := "no app specified"
					if err != nil {
						msg = err.Error()
					}
					printError(msg)
					cmd.PrintUsage()
					os.Exit(2)
				case err != nil:
					printFatal(err.Error())
				}
			}
			cmd.Run(cmd, cmd.Flag.Args())
			return
		}
	}

	path := findPlugin(args[0])
	if path == "" {
		fmt.Fprintf(os.Stderr, "Unknown command: %s\n", args[0])
		if g := suggest(args[0]); len(g) > 0 {
			fmt.Fprintf(os.Stderr, "Possible alternatives: %v\n", strings.Join(g, " "))
		}
		fmt.Fprintf(os.Stderr, "Run 'hk help' for usage.\n")
		os.Exit(2)
	}
	err := execPlugin(path, args)
	printFatal("exec error: %s", err)
}

var rollbarClient = &rollbar.Client{
	AppName:    "hk",
	AppVersion: Version,
	Endpoint:   "https://api.rollbar.com/api/1/item/",
	Token:      "d344db7a09fa481e983694bfa326e6d9",
}

func recoverPanic() {
	if Version != "dev" {
		if rec := recover(); rec != nil {
			message := ""
			switch rec := rec.(type) {
			case error:
				message = rec.Error()
			default:
				message = fmt.Sprintf("%v", rec)
			}
			if err := rollbarClient.Report(message); err != nil {
				printError("reporting crash failed: %s", err.Error())
				panic(rec)
			}
			printFatal("hk encountered and reported an internal client error")
		}
	}
}

func app() (string, error) {
	if flagApp != "" {
		return flagApp, nil
	}

	if app := os.Getenv("HKAPP"); app != "" {
		return app, nil
	}

	return appFromGitRemote(remoteFromGitConfig())
}

func mustApp() string {
	name, err := app()
	if err != nil {
		printFatal(err.Error())
	}
	return name
}
<|endoftext|>


<commit_before>/*
Copyright 2015 Kelsey Hightower All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/docker/libcompose/project"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
)

var (
	composeFile string
	outputDir   string
)

func init() {
	flag.StringVar(&composeFile, "compose-file", "docker-compose.yml", "Specify an alternate compose `file`")
	flag.StringVar(&outputDir, "output-dir", "output", "Kubernetes configs output `directory`")
}

func main() {
	flag.Parse()

	p := project.NewProject(&project.Context{
		ProjectName: "kube",
		ComposeFile: composeFile,
	})

	if err := p.Parse(); err != nil {
		log.Fatalf("Failed to parse the compose project from %s: %v", composeFile, err)
	}
	if err := os.MkdirAll(outputDir, 0755); err != nil {
		log.Fatalf("Failed to create the output directory %s: %v", outputDir, err)
	}

	for name, service := range p.Configs {
		rc := &api.ReplicationController{
			TypeMeta: unversioned.TypeMeta{
				Kind:       "ReplicationController",
				APIVersion: "v1",
			},
			ObjectMeta: api.ObjectMeta{
				Name:   name,
				Labels: map[string]string{"service": name},
			},
			Spec: api.ReplicationControllerSpec{
				Replicas: 1,
				Selector: map[string]string{"service": name},
				Template: &api.PodTemplateSpec{
					ObjectMeta: api.ObjectMeta{
						Labels: map[string]string{"service": name},
					},
					Spec: api.PodSpec{
						Containers: []api.Container{
							{
								Name:  name,
								Image: service.Image,
							},
						},
					},
				},
			},
		}

		// Configure the container ports.
		var ports []api.ContainerPort
		for _, port := range service.Ports {
			// Check if we have to deal with a mapped port
			if strings.Contains(port, ":") {
				parts := strings.Split(port, ":")
				port = parts[1]
			}
			portNumber, err := strconv.Atoi(port)
			if err != nil {
				log.Fatalf("Invalid container port %s for service %s", port, name)
			}
			ports = append(ports, api.ContainerPort{ContainerPort: portNumber})
		}

		rc.Spec.Template.Spec.Containers[0].Ports = ports

		// Configure the container restart policy.
		switch service.Restart {
		case "", "always":
			rc.Spec.Template.Spec.RestartPolicy = api.RestartPolicyAlways
		case "no":
			rc.Spec.Template.Spec.RestartPolicy = api.RestartPolicyNever
		case "on-failure":
			rc.Spec.Template.Spec.RestartPolicy = api.RestartPolicyOnFailure
		default:
			log.Fatalf("Unknown restart policy %s for service %s", service.Restart, name)
		}

		data, err := json.MarshalIndent(rc, "", "  ")
		if err != nil {
			log.Fatalf("Failed to marshal the replication controller: %v", err)
		}

		// Save the replication controller for the Docker compose service to the
		// configs directory.
		outputFileName := fmt.Sprintf("%s-rc.yaml", name)
		outputFilePath := filepath.Join(outputDir, outputFileName)
		if err := ioutil.WriteFile(outputFilePath, data, 0644); err != nil {
			log.Fatalf("Failed to write replication controller %s: %v", outputFileName, err)
		}
		fmt.Println(outputFilePath)
	}
}
<commit_msg>fixes #17 - reflect libcompose project change in type Context struct<commit_after>/*
Copyright 2015 Kelsey Hightower All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/docker/libcompose/project"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
)

var (
	composeFile string
	outputDir   string
)

func init() {
	flag.StringVar(&composeFile, "compose-file", "docker-compose.yml", "Specify an alternate compose `file`")
	flag.StringVar(&outputDir, "output-dir", "output", "Kubernetes configs output `directory`")
}

func main() {
	flag.Parse()

	p := project.NewProject(&project.Context{
		ProjectName: "kube",
		ComposeFiles: []string{composeFile},
	})

	if err := p.Parse(); err != nil {
		log.Fatalf("Failed to parse the compose project from %s: %v", composeFile, err)
	}
	if err := os.MkdirAll(outputDir, 0755); err != nil {
		log.Fatalf("Failed to create the output directory %s: %v", outputDir, err)
	}

	for name, service := range p.Configs {
		rc := &api.ReplicationController{
			TypeMeta: unversioned.TypeMeta{
				Kind:       "ReplicationController",
				APIVersion: "v1",
			},
			ObjectMeta: api.ObjectMeta{
				Name:   name,
				Labels: map[string]string{"service": name},
			},
			Spec: api.ReplicationControllerSpec{
				Replicas: 1,
				Selector: map[string]string{"service": name},
				Template: &api.PodTemplateSpec{
					ObjectMeta: api.ObjectMeta{
						Labels: map[string]string{"service": name},
					},
					Spec: api.PodSpec{
						Containers: []api.Container{
							{
								Name:  name,
								Image: service.Image,
							},
						},
					},
				},
			},
		}

		// Configure the container ports.
		var ports []api.ContainerPort
		for _, port := range service.Ports {
			// Check if we have to deal with a mapped port
			if strings.Contains(port, ":") {
				parts := strings.Split(port, ":")
				port = parts[1]
			}
			portNumber, err := strconv.Atoi(port)
			if err != nil {
				log.Fatalf("Invalid container port %s for service %s", port, name)
			}
			ports = append(ports, api.ContainerPort{ContainerPort: portNumber})
		}

		rc.Spec.Template.Spec.Containers[0].Ports = ports

		// Configure the container restart policy.
		switch service.Restart {
		case "", "always":
			rc.Spec.Template.Spec.RestartPolicy = api.RestartPolicyAlways
		case "no":
			rc.Spec.Template.Spec.RestartPolicy = api.RestartPolicyNever
		case "on-failure":
			rc.Spec.Template.Spec.RestartPolicy = api.RestartPolicyOnFailure
		default:
			log.Fatalf("Unknown restart policy %s for service %s", service.Restart, name)
		}

		data, err := json.MarshalIndent(rc, "", "  ")
		if err != nil {
			log.Fatalf("Failed to marshal the replication controller: %v", err)
		}

		// Save the replication controller for the Docker compose service to the
		// configs directory.
		outputFileName := fmt.Sprintf("%s-rc.yaml", name)
		outputFilePath := filepath.Join(outputDir, outputFileName)
		if err := ioutil.WriteFile(outputFilePath, data, 0644); err != nil {
			log.Fatalf("Failed to write replication controller %s: %v", outputFileName, err)
		}
		fmt.Println(outputFilePath)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Marc-Antoine Ruel. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

// sci is a shameful CI.
//
// It is a simple Github webhook that runs a Go build and an hardcoded
// command upon PR or push from whitelisted users.
//
// It posts the stdout to a Github gist and updates the PR status.
//
// It doesn't stream data so it cannot be used for slow task.
package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"reflect"
	"runtime"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/bugsnag/osext"
	"github.com/google/go-github/github"
	"golang.org/x/oauth2"
)

type config struct {
	Port              int        // TCP port number for HTTP server.
	WebHookSecret     string     // https://developer.github.com/webhooks/
	Oauth2AccessToken string     // https://github.com/settings/tokens, check "repo:status" and "gist"
	UseSSH            bool       // Use ssh (instead of https) for checkout. Required for private repositories.
	Name              string     // Display name to use in the status report on Github.
	Checks            [][]string // Commands to run to test the repository. They are run one after the other from the repository's root.
}

func loadConfig() (*config, error) {
	c := &config{
		Port:              8080,
		WebHookSecret:     "Create a secret and set it at github.com/'name'/'repo'/settings/hooks",
		Oauth2AccessToken: "Get one at https://github.com/settings/tokens",
		UseSSH:            false,
		Name:              "sci",
		Checks:            [][]string{{"go", "test", "./..."}},
	}
	b, err := ioutil.ReadFile("sci.json")
	if err != nil {
		b, err = json.MarshalIndent(c, "", "  ")
		if err != nil {
			return nil, err
		}
		if err = ioutil.WriteFile("sci.json", b, 0600); err != nil {
			return nil, err
		}
		return nil, errors.New("wrote new sci.json")
	}
	if err = json.Unmarshal(b, c); err != nil {
		return nil, err
	}
	d, err := json.MarshalIndent(c, "", "  ")
	if err != nil {
		return nil, err
	}
	if !bytes.Equal(b, d) {
		log.Printf("Updating sci.json in canonical format")
		if err := ioutil.WriteFile("sci.json", d, 0600); err != nil {
			return nil, err
		}
	}
	return c, nil
}

func run(cwd string, cmd ...string) (string, bool) {
	cmds := strings.Join(cmd, " ")
	log.Printf("- cwd=%s : %s", cwd, cmds)
	c := exec.Command(cmd[0], cmd[1:]...)
	c.Dir = cwd
	start := time.Now()
	out, err := c.CombinedOutput()
	duration := time.Since(start)
	// Assumes UTF-8.
	return fmt.Sprintf("$ %s  (%s)\n%s", cmds, duration, string(out)), err == nil
}

// runChecks syncs then runs the checks and returns task's results.
func runChecks(cmds [][]string, repoName string, useSSH bool, commit, gopath string) (map[string]string, bool) {
	out := map[string]string{
		"metadata": fmt.Sprintf(
			"Commit: %s\nVersion: %s\nGOROOT: %s\nGOPATH: %s\nCPUs: %d",
			commit, runtime.Version(), runtime.GOROOT(), gopath, runtime.NumCPU()),
		"setup": "",
	}
	repoPath := "github.com/" + repoName
	base := filepath.Join(gopath, "src", repoPath)
	if _, err := os.Stat(base); err != nil {
		up := path.Dir(base)
		if err := os.MkdirAll(up, 0700); err != nil && !os.IsExist(err) {
			log.Printf("- %v", err)
		}
		url := "https://" + repoPath
		if useSSH {
			url = "git@github.com:" + repoName
		}
		stdout, ok := run(up, "git", "clone", "--quiet", url)
		out["setup"] = stdout
		if !ok {
			return out, ok
		}
	} else {
		stdout, ok := run(base, "git", "fetch", "--prune", "--quiet")
		out["setup"] = stdout
		if !ok {
			return out, ok
		}
	}
	stdout, ok := run(base, "git", "checkout", "--quiet", commit)
	out["setup"] += stdout
	if ok {
		// TODO(maruel): update dependencies manually!
		stdout, ok = run(base, "go", "get", "-v", "-d", "-t", "./...")
		out["setup"] += stdout
		if ok {
			// Precompilation has a dramatic effect on a Raspberry Pi.
			stdout, ok = run(base, "go", "test", "-i", "./...")
			out["setup"] += stdout
			if ok {
				// Finally run the checks!
				for i, cmd := range cmds {
					ok2 := true
					if out[fmt.Sprintf("cmd%d", i+1)], ok2 = run(base, cmd...); !ok2 {
						ok = false
					}
				}
			}
		}
	}
	return out, ok
}

type server struct {
	c       *config
	client  *github.Client
	gopath  string
	mu      sync.Mutex
	collabs map[string]map[string]bool
}

func (s *server) canCollab(owner, repo, user string) bool {
	key := owner + "/" + repo
	s.mu.Lock()
	defer s.mu.Unlock()
	if _, ok := s.collabs[key]; !ok {
		s.collabs[key] = map[string]bool{}
	}
	if v, ok := s.collabs[key][user]; ok {
		return v
	}
	v, _, _ := s.client.Repositories.IsCollaborator(owner, repo, user)
	if v {
		// Only cache hits because otherwise adding a collaborator would mean
		// restarting every sci instances.
		s.collabs[key][user] = v
	}
	log.Printf("- %s: %s access: %t", key, user, v)
	return v
}

func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	log.Printf("HTTP: %s %s", r.RemoteAddr, r.URL.Path)
	defer r.Body.Close()
	if r.Method != "POST" {
		http.Error(w, "Invalid method", http.StatusMethodNotAllowed)
		log.Printf("- invalid method")
		return
	}
	payload, err := github.ValidatePayload(r, []byte(s.c.WebHookSecret))
	if err != nil {
		http.Error(w, "Invalid secret", http.StatusUnauthorized)
		log.Printf("- invalid secret")
		return
	}
	if t := github.WebHookType(r); t != "ping" {
		event, err := github.ParseWebHook(t, payload)
		if err != nil {
			http.Error(w, "Invalid payload", http.StatusBadRequest)
			log.Printf("- invalid payload")
			return
		}
		// Process the rest asynchronously so the hook doesn't take too long.
		go func() {
			switch event := event.(type) {
			// TODO(maruel): For *github.CommitCommentEvent and
			// *github.IssueCommentEvent, when the comment is 'run tests' from a
			// collaborator, run the tests.
			case *github.PullRequestEvent:
				log.Printf("- PR %s #%d %s %s", *event.Repo.FullName, *event.PullRequest.ID, *event.Sender.Login, *event.Action)
				if *event.Action != "opened" && *event.Action != "synchronized" {
					log.Printf("- ignoring action %q for PR from %q", *event.Action, *event.Sender.Login)
				} else if !s.canCollab(*event.Repo.Owner.Login, *event.Repo.Name, *event.Sender.Login) {
					log.Printf("- ignoring owner %q for PR", *event.Sender.Login)
				} else if err = s.runCheck(*event.Repo.FullName, *event.PullRequest.Head.SHA); err != nil {
					log.Printf("- %v", err)
				}
			case *github.PushEvent:
				if event.HeadCommit == nil {
					log.Printf("- Push %s %s <deleted>", *event.Repo.FullName, *event.Ref)
				} else {
					log.Printf("- Push %s %s %s", *event.Repo.FullName, *event.Ref, *event.HeadCommit.ID)
					if !strings.HasPrefix(*event.Ref, "refs/heads/") {
						log.Printf("- ignoring branch %q for push", *event.Ref)
					} else if err = s.runCheck(*event.Repo.FullName, *event.HeadCommit.ID); err != nil {
						log.Printf("- %v", err)
					}
				}
			default:
				log.Printf("- ignoring hook type %s", reflect.TypeOf(event).Elem().Name())
			}
		}()
	}
	io.WriteString(w, "{}")
}

func (s *server) runCheck(repo, commit string) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	log.Printf("- Running test for %s at %s", repo, commit)
	// TODO(maruel): Update the gist as the task is running;
	// https://developer.github.com/v3/gists/#edit-a-gist
	out, success := runChecks(s.c.Checks, repo, s.c.UseSSH, commit, s.gopath)
	// https://developer.github.com/v3/gists/#create-a-gist
	// It is still accessible via the URL without authentication.
	gist := &github.Gist{
		Description: github.String("Output for https://github.com/" + repo + "/commit/" + commit),
		Public:      github.Bool(false),
		Files:       map[github.GistFilename]github.GistFile{},
	}
	for k, v := range out {
		if len(v) == 0 {
			v = "<missing>"
		}
		gist.Files[github.GistFilename(k)] = github.GistFile{Content: github.String(v)}
	}
	var err error
	if gist, _, err = s.client.Gists.Create(gist); err != nil {
		return err
	}
	log.Printf("- Gist at %s", *gist.HTMLURL)

	// https://developer.github.com/v3/repos/statuses/#create-a-status
	desc := "Ran:\n"
	for i, c := range s.c.Checks {
		if i != 0 {
			desc += "\n"
		}
		desc += "  " + strings.Join(c, " ")
	}
	status := &github.RepoStatus{
		State:       github.String("success"),
		TargetURL:   gist.HTMLURL,
		Description: &desc,
		Context:     github.String("sci"),
	}
	if !success {
		status.State = github.String("failure")
	}
	parts := strings.SplitN(repo, "/", 2)
	_, _, err = s.client.Repositories.CreateStatus(parts[0], parts[1], commit, status)
	return err
}

func mainImpl() error {
	test := flag.String("test", "", "runs a simulation locally, specify the git repository name (not URL) to test, e.g. 'maruel/sci'")
	commit := flag.String("commit", "HEAD", "commit ID to test and update; will only update if not 'HEAD'")
	flag.Parse()
	c, err := loadConfig()
	if err != nil {
		return err
	}
	wd, err := os.Getwd()
	if err != nil {
		return err
	}
	gopath := filepath.Join(wd, "sci-gopath")
	os.Setenv("GOPATH", gopath)
	tc := oauth2.NewClient(oauth2.NoContext, oauth2.StaticTokenSource(&oauth2.Token{AccessToken: c.Oauth2AccessToken}))
	s := server{c: c, client: github.NewClient(tc), gopath: gopath, collabs: map[string]map[string]bool{}}
	if len(*test) != 0 {
		if *commit == "HEAD" {
			// Only run locally.
			out, success := runChecks(c.Checks, *test, c.UseSSH, *commit, gopath)
			names := make([]string, 0, len(out))
			for k := range out {
				names = append(names, k)
			}
			sort.Strings(names)
			for _, k := range names {
				fmt.Printf("--- %s\n%s", k, out[k])
			}
			_, err := fmt.Printf("\nSuccess: %t\n", success)
			return err
		}
		return s.runCheck(*test, *commit)
	}
	http.Handle("/", &s)
	thisFile, err := osext.Executable()
	if err != nil {
		return err
	}
	log.Printf("Running in: %s", wd)
	log.Printf("Executable: %s", thisFile)
	ln, err := net.Listen("tcp", fmt.Sprintf(":%d", c.Port))
	if err != nil {
		return err
	}
	a := ln.Addr().String()
	ln.Close()
	log.Printf("Listening on: %s", a)
	go http.ListenAndServe(a, nil)
	// TODO(maruel): watch sci.json too.
	err = watchFile(thisFile)
	// Ensures no task is running.
	s.mu.Lock()
	return err
}

func main() {
	if err := mainImpl(); err != nil {
		fmt.Fprintf(os.Stderr, "sci: %s.\n", err)
		os.Exit(1)
	}
}
<commit_msg>Update the gist asynchronously<commit_after>// Copyright 2016 Marc-Antoine Ruel. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

// sci is a shameful CI.
//
// It is a simple Github webhook that runs a Go build and an hardcoded
// command upon PR or push from whitelisted users.
//
// It posts the stdout to a Github gist and updates the PR status.
//
// It doesn't stream data so it cannot be used for slow task.
package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"reflect"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/bugsnag/osext"
	"github.com/google/go-github/github"
	"golang.org/x/oauth2"
)

type config struct {
	Port              int        // TCP port number for HTTP server.
	WebHookSecret     string     // https://developer.github.com/webhooks/
	Oauth2AccessToken string     // https://github.com/settings/tokens, check "repo:status" and "gist"
	UseSSH            bool       // Use ssh (instead of https) for checkout. Required for private repositories.
	Name              string     // Display name to use in the status report on Github.
	Checks            [][]string // Commands to run to test the repository. They are run one after the other from the repository's root.
}

func loadConfig() (*config, error) {
	c := &config{
		Port:              8080,
		WebHookSecret:     "Create a secret and set it at github.com/'name'/'repo'/settings/hooks",
		Oauth2AccessToken: "Get one at https://github.com/settings/tokens",
		UseSSH:            false,
		Name:              "sci",
		Checks:            [][]string{{"go", "test", "./..."}},
	}
	b, err := ioutil.ReadFile("sci.json")
	if err != nil {
		b, err = json.MarshalIndent(c, "", "  ")
		if err != nil {
			return nil, err
		}
		if err = ioutil.WriteFile("sci.json", b, 0600); err != nil {
			return nil, err
		}
		return nil, errors.New("wrote new sci.json")
	}
	if err = json.Unmarshal(b, c); err != nil {
		return nil, err
	}
	d, err := json.MarshalIndent(c, "", "  ")
	if err != nil {
		return nil, err
	}
	if !bytes.Equal(b, d) {
		log.Printf("Updating sci.json in canonical format")
		if err := ioutil.WriteFile("sci.json", d, 0600); err != nil {
			return nil, err
		}
	}
	return c, nil
}

func run(cwd string, cmd ...string) (string, bool) {
	cmds := strings.Join(cmd, " ")
	log.Printf("- cwd=%s : %s", cwd, cmds)
	c := exec.Command(cmd[0], cmd[1:]...)
	c.Dir = cwd
	start := time.Now()
	out, err := c.CombinedOutput()
	duration := time.Since(start)
	// Assumes UTF-8.
	return fmt.Sprintf("$ %s  (%s)\n%s", cmds, duration, string(out)), err == nil
}

type file struct {
	name, content string
}

func metadata(commit, gopath string) string {
	return fmt.Sprintf(
		"Commit: %s\nVersion: %s\nGOROOT: %s\nGOPATH: %s\nCPUs: %d",
		commit, runtime.Version(), runtime.GOROOT(), gopath, runtime.NumCPU())
}

// runChecks syncs then runs the checks and returns task's results.
func runChecks(cmds [][]string, repoName string, useSSH bool, commit, gopath string, results chan<- file) bool {
	repoPath := "github.com/" + repoName
	base := filepath.Join(gopath, "src", repoPath)
	setup := ""
	if _, err := os.Stat(base); err != nil {
		up := path.Dir(base)
		if err := os.MkdirAll(up, 0700); err != nil && !os.IsExist(err) {
			log.Printf("- %v", err)
		}
		url := "https://" + repoPath
		if useSSH {
			url = "git@github.com:" + repoName
		}
		stdout, ok := run(up, "git", "clone", "--quiet", url)
		setup = stdout
		if !ok {
			results <- file{"setup", setup}
			return ok
		}
	} else {
		stdout, ok := run(base, "git", "fetch", "--prune", "--quiet")
		setup = stdout
		if !ok {
			results <- file{"setup", setup}
			return ok
		}
	}
	stdout, ok := run(base, "git", "checkout", "--quiet", commit)
	setup += stdout
	if ok {
		// TODO(maruel): update dependencies manually!
		stdout, ok = run(base, "go", "get", "-v", "-d", "-t", "./...")
		setup += stdout
		if ok {
			// Precompilation has a dramatic effect on a Raspberry Pi.
			stdout, ok = run(base, "go", "test", "-i", "./...")
			setup += stdout
		}
	}
	results <- file{"setup", setup}
	if ok {
		// Finally run the checks!
		for i, cmd := range cmds {
			stdout, ok2 := run(base, cmd...)
			results <- file{fmt.Sprintf("cmd%d", i+1), stdout}
			if !ok2 {
				ok = false
			}
		}
	}
	return ok
}

type server struct {
	c       *config
	client  *github.Client
	gopath  string
	mu      sync.Mutex
	collabs map[string]map[string]bool
}

func (s *server) canCollab(owner, repo, user string) bool {
	key := owner + "/" + repo
	s.mu.Lock()
	defer s.mu.Unlock()
	if _, ok := s.collabs[key]; !ok {
		s.collabs[key] = map[string]bool{}
	}
	if v, ok := s.collabs[key][user]; ok {
		return v
	}
	v, _, _ := s.client.Repositories.IsCollaborator(owner, repo, user)
	if v {
		// Only cache hits because otherwise adding a collaborator would mean
		// restarting every sci instances.
		s.collabs[key][user] = v
	}
	log.Printf("- %s: %s access: %t", key, user, v)
	return v
}

func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	log.Printf("HTTP: %s %s", r.RemoteAddr, r.URL.Path)
	defer r.Body.Close()
	if r.Method != "POST" {
		http.Error(w, "Invalid method", http.StatusMethodNotAllowed)
		log.Printf("- invalid method")
		return
	}
	payload, err := github.ValidatePayload(r, []byte(s.c.WebHookSecret))
	if err != nil {
		http.Error(w, "Invalid secret", http.StatusUnauthorized)
		log.Printf("- invalid secret")
		return
	}
	if t := github.WebHookType(r); t != "ping" {
		event, err := github.ParseWebHook(t, payload)
		if err != nil {
			http.Error(w, "Invalid payload", http.StatusBadRequest)
			log.Printf("- invalid payload")
			return
		}
		// Process the rest asynchronously so the hook doesn't take too long.
		go func() {
			switch event := event.(type) {
			// TODO(maruel): For *github.CommitCommentEvent and
			// *github.IssueCommentEvent, when the comment is 'run tests' from a
			// collaborator, run the tests.
			case *github.PullRequestEvent:
				log.Printf("- PR %s #%d %s %s", *event.Repo.FullName, *event.PullRequest.ID, *event.Sender.Login, *event.Action)
				if *event.Action != "opened" && *event.Action != "synchronized" {
					log.Printf("- ignoring action %q for PR from %q", *event.Action, *event.Sender.Login)
				} else if !s.canCollab(*event.Repo.Owner.Login, *event.Repo.Name, *event.Sender.Login) {
					log.Printf("- ignoring owner %q for PR", *event.Sender.Login)
				} else if err = s.runCheck(*event.Repo.FullName, *event.PullRequest.Head.SHA); err != nil {
					log.Printf("- %v", err)
				}
			case *github.PushEvent:
				if event.HeadCommit == nil {
					log.Printf("- Push %s %s <deleted>", *event.Repo.FullName, *event.Ref)
				} else {
					log.Printf("- Push %s %s %s", *event.Repo.FullName, *event.Ref, *event.HeadCommit.ID)
					if !strings.HasPrefix(*event.Ref, "refs/heads/") {
						log.Printf("- ignoring branch %q for push", *event.Ref)
					} else if err = s.runCheck(*event.Repo.FullName, *event.HeadCommit.ID); err != nil {
						log.Printf("- %v", err)
					}
				}
			default:
				log.Printf("- ignoring hook type %s", reflect.TypeOf(event).Elem().Name())
			}
		}()
	}
	io.WriteString(w, "{}")
}

func (s *server) runCheck(repo, commit string) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	log.Printf("- Running test for %s at %s", repo, commit)
	// https://developer.github.com/v3/gists/#create-a-gist
	// It is still accessible via the URL without authentication.
	gist := &github.Gist{
		Description: github.String("Output for https://github.com/" + repo + "/commit/" + commit),
		Public:      github.Bool(false),
		Files:       map[github.GistFilename]github.GistFile{"metadata": github.GistFile{Content: github.String(metadata(commit, s.gopath))}},
	}
	gist, _, err := s.client.Gists.Create(gist)
	if err != nil {
		// Don't bother running the tests.
		return err
	}
	log.Printf("- Gist at %s", *gist.HTMLURL)
	results := make(chan file)
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		for i := range results {
			// https://developer.github.com/v3/gists/#edit-a-gist
			if len(i.content) == 0 {
				i.content = "<missing>"
			}
			gist.Files = map[github.GistFilename]github.GistFile{github.GistFilename(i.name): github.GistFile{Content: &i.content}}
			if _, _, err = s.client.Gists.Edit(*gist.ID, gist); err != nil {
				// Just move on.
				log.Printf("- failed to update gist %v", err)
			}
		}
	}()
	success := runChecks(s.c.Checks, repo, s.c.UseSSH, commit, s.gopath, results)
	close(results)
	wg.Wait()

	// https://developer.github.com/v3/repos/statuses/#create-a-status
	desc := "Ran:\n"
	for i, c := range s.c.Checks {
		if i != 0 {
			desc += "\n"
		}
		desc += "  " + strings.Join(c, " ")
	}
	status := &github.RepoStatus{
		State:       github.String("success"),
		TargetURL:   gist.HTMLURL,
		Description: &desc,
		Context:     github.String("sci"),
	}
	if !success {
		status.State = github.String("failure")
	}
	parts := strings.SplitN(repo, "/", 2)
	_, _, err = s.client.Repositories.CreateStatus(parts[0], parts[1], commit, status)
	return err
}

func mainImpl() error {
	test := flag.String("test", "", "runs a simulation locally, specify the git repository name (not URL) to test, e.g. 'maruel/sci'")
	commit := flag.String("commit", "HEAD", "commit ID to test and update; will only update if not 'HEAD'")
	flag.Parse()
	c, err := loadConfig()
	if err != nil {
		return err
	}
	wd, err := os.Getwd()
	if err != nil {
		return err
	}
	gopath := filepath.Join(wd, "sci-gopath")
	os.Setenv("GOPATH", gopath)
	tc := oauth2.NewClient(oauth2.NoContext, oauth2.StaticTokenSource(&oauth2.Token{AccessToken: c.Oauth2AccessToken}))
	s := server{c: c, client: github.NewClient(tc), gopath: gopath, collabs: map[string]map[string]bool{}}
	if len(*test) != 0 {
		if *commit == "HEAD" {
			// Only run locally.
			results := make(chan file)
			var wg sync.WaitGroup
			wg.Add(1)
			go func() {
				defer wg.Done()
				for i := range results {
					fmt.Printf("--- %s\n%s", i.name, i.content)
				}
			}()
			fmt.Printf("--- metadata\n%s", metadata(*commit, gopath))
			success := runChecks(c.Checks, *test, c.UseSSH, *commit, gopath, results)
			close(results)
			wg.Wait()
			_, err := fmt.Printf("\nSuccess: %t\n", success)
			return err
		}
		return s.runCheck(*test, *commit)
	}
	http.Handle("/", &s)
	thisFile, err := osext.Executable()
	if err != nil {
		return err
	}
	log.Printf("Running in: %s", wd)
	log.Printf("Executable: %s", thisFile)
	ln, err := net.Listen("tcp", fmt.Sprintf(":%d", c.Port))
	if err != nil {
		return err
	}
	a := ln.Addr().String()
	ln.Close()
	log.Printf("Listening on: %s", a)
	go http.ListenAndServe(a, nil)
	// TODO(maruel): watch sci.json too.
	err = watchFile(thisFile)
	// Ensures no task is running.
	s.mu.Lock()
	return err
}

func main() {
	if err := mainImpl(); err != nil {
		fmt.Fprintf(os.Stderr, "sci: %s.\n", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"os/exec"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/samalba/dockerclient"
)

var (
	// TODO: should this be shared struct property?
	docker dockerclient.Client
)

func getNamesFromRawLinks(rawLinks []string) ([]string, error) {
	names := []string{}
	for _, rawLinks := range rawLinks {
		// ORIGINAL : "/redis:/adoring_stallman/redis"
		// DESIRED  : "redis" (the last bit of the string)
		// TODO: can this be simplified?
		name := strings.Split(strings.Split(rawLinks, ":")[1], "/")[2]
		names = append(names, name)
	}
	return names, nil
}

func getLinks(info dockerclient.ContainerInfo) ([]string, error) {
	rawLinks := info.HostConfig.Links
	links, err := getNamesFromRawLinks(rawLinks)
	if err != nil {
		return []string{}, err
	}
	log.Debugln("links for", info.Name, "are", links)
	return links, nil
}

func waitForDeps(deps []string, rollers chan string) {
	depRemaining := make(map[string]bool)
	for _, link := range deps {
		depRemaining[link] = true
	}
	for lastRestarted := range rollers {
		depRemaining[lastRestarted] = false
		log.Debugln("depRemaining is", depRemaining)
		for _, depRemains := range depRemaining {
			if depRemains {
				continue
			}
		}
		return
	}
}

func pollRestart(c dockerclient.Container, done chan string, rollers chan string) {
	log.Infoln("Initiating restart for ", c.Id)

	info, err := docker.InspectContainer(c.Id)
	if err != nil {
		log.Fatal(err)
	}

	deps, err := getLinks(*info)
	if err != nil {
		log.Fatal(err)
	}

	if len(deps) > 0 {
		waitForDeps(deps, rollers)
	}

	log.Debugln("Starting", c.Id)

	// Alright, all the deps are done restarting,
	// so let's rock and roll!
	if err := docker.StartContainer(c.Id, nil); err != nil {
		log.Fatal(err)
	}

	// lose the "/" on the beginning of container name
	done <- info.Name[1:]

	// Just catch the rest until the channel closes
	for lastRestarted := range rollers {
		log.Debugln("Read", lastRestarted, "from rollers in goroutine for", c.Id[:7])
	}
}

func main() {
	var (
		err error
	)
	start := time.Now()
	docker, err = dockerclient.NewDockerClient("unix:///var/run/docker.sock", nil)
	if err != nil {
		log.Fatal(err)
	}

	restartDone := make(chan string)

	// todo: should this be slice?
	// i.e. is there any reason to track the id in key?
	rMsgs := []chan string{}

	containers, err := docker.ListContainers(false, false, "")
	if err != nil {
		log.Fatal(err)
	}

	stopMsgs := make(chan bool)
	for _, c := range containers {
		rMsgs = append(rMsgs, make(chan string))
		go func(id string) {
			log.Infoln("Initiating stop for container", id)
			if err := docker.StopContainer(id, 30); err != nil {
				log.Fatal(err)
			}
			stopMsgs <- true
		}(c.Id)
	}

	// wait for all containers to be stopped
	for i := 0; i < len(containers); i++ {
		<-stopMsgs
	}
	close(stopMsgs)

	log.Info("All containers stopped")

	// restart the daemon
	log.Info("Restarting Docker...")
	if err := exec.Command("sudo", "service", "docker", "restart").Run(); err != nil {
		log.Fatal(err)
	}

	// TODO: Hack because the socket accepts
	// connections before the API is ready to
	// accept requests.
	for {
		_, err := docker.Info()
		if err != nil {
			log.Debug(err)
		} else {
			log.Info("Daemon restarted successfully")
			break
		}
		time.Sleep(100 * time.Millisecond)
	}

	// initiate each container polling for its deps to
	// know when it can restart
	for i, c := range containers {
		go pollRestart(c, restartDone, rMsgs[i])
	}

	for i := 0; i < len(containers); i++ {
		r := <-restartDone
		log.Debugln("Read from channel that", r, "started")
		for _, rMsg := range rMsgs {
			go func(rMsg chan string, r string) {
				log.Debugln("SENDING", r)
				rMsg <- r
			}(rMsg, r)
		}
	}

	close(restartDone)
	for _, rMsg := range rMsgs {
		close(rMsg)
	}

	log.Infof("Restarted the daemon and %d containers in %s", len(containers), time.Since(start))
}
<commit_msg>Specify channel directionality<commit_after>package main

import (
	"os/exec"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/samalba/dockerclient"
)

var (
	// TODO: should this be shared struct property?
	docker dockerclient.Client
)

func getNamesFromRawLinks(rawLinks []string) ([]string, error) {
	names := []string{}
	for _, rawLinks := range rawLinks {
		// ORIGINAL : "/redis:/adoring_stallman/redis"
		// DESIRED  : "redis" (the last bit of the string)
		// TODO: can this be simplified?
		name := strings.Split(strings.Split(rawLinks, ":")[1], "/")[2]
		names = append(names, name)
	}
	return names, nil
}

func getLinks(info dockerclient.ContainerInfo) ([]string, error) {
	rawLinks := info.HostConfig.Links
	links, err := getNamesFromRawLinks(rawLinks)
	if err != nil {
		return []string{}, err
	}
	log.Debugln("links for", info.Name, "are", links)
	return links, nil
}

func waitForDeps(deps []string, rollers <-chan string) {
	depRemaining := make(map[string]bool)
	for _, link := range deps {
		depRemaining[link] = true
	}
	for lastRestarted := range rollers {
		depRemaining[lastRestarted] = false
		log.Debugln("depRemaining is", depRemaining)
		for _, depRemains := range depRemaining {
			if depRemains {
				continue
			}
		}
		return
	}
}

func pollRestart(c dockerclient.Container, done chan<- string, rollers <-chan string) {
	log.Infoln("Initiating restart for ", c.Id)

	info, err := docker.InspectContainer(c.Id)
	if err != nil {
		log.Fatal(err)
	}

	deps, err := getLinks(*info)
	if err != nil {
		log.Fatal(err)
	}

	if len(deps) > 0 {
		waitForDeps(deps, rollers)
	}

	log.Debugln("Starting", c.Id)

	// Alright, all the deps are done restarting,
	// so let's rock and roll!
	if err := docker.StartContainer(c.Id, nil); err != nil {
		log.Fatal(err)
	}

	// lose the "/" on the beginning of container name
	done <- info.Name[1:]

	// Just catch the rest until the channel closes
	for lastRestarted := range rollers {
		log.Debugln("Read", lastRestarted, "from rollers in goroutine for", c.Id[:7])
	}
}

func main() {
	var (
		err error
	)
	start := time.Now()
	docker, err = dockerclient.NewDockerClient("unix:///var/run/docker.sock", nil)
	if err != nil {
		log.Fatal(err)
	}

	restartDone := make(chan string)

	// todo: should this be slice?
	// i.e. is there any reason to track the id in key?
	rMsgs := []chan string{}

	containers, err := docker.ListContainers(false, false, "")
	if err != nil {
		log.Fatal(err)
	}

	stopMsgs := make(chan bool)
	for _, c := range containers {
		rMsgs = append(rMsgs, make(chan string))
		go func(id string) {
			log.Infoln("Initiating stop for container", id)
			if err := docker.StopContainer(id, 30); err != nil {
				log.Fatal(err)
			}
			stopMsgs <- true
		}(c.Id)
	}

	// wait for all containers to be stopped
	for i := 0; i < len(containers); i++ {
		<-stopMsgs
	}
	close(stopMsgs)

	log.Info("All containers stopped")

	// restart the daemon
	log.Info("Restarting Docker...")
	if err := exec.Command("sudo", "service", "docker", "restart").Run(); err != nil {
		log.Fatal(err)
	}

	// TODO: Hack because the socket accepts
	// connections before the API is ready to
	// accept requests.
	for {
		_, err := docker.Info()
		if err != nil {
			log.Debug(err)
		} else {
			log.Info("Daemon restarted successfully")
			break
		}
		time.Sleep(100 * time.Millisecond)
	}

	// initiate each container polling for its deps to
	// know when it can restart
	for i, c := range containers {
		go pollRestart(c, restartDone, rMsgs[i])
	}

	for i := 0; i < len(containers); i++ {
		r := <-restartDone
		log.Debugln("Read from channel that", r, "started")
		for _, rMsg := range rMsgs {
			go func(rMsg chan<- string, r string) {
				log.Debugln("SENDING", r)
				rMsg <- r
			}(rMsg, r)
		}
	}

	close(restartDone)
	for _, rMsg := range rMsgs {
		close(rMsg)
	}

	log.Infof("Restarted the daemon and %d containers in %s", len(containers), time.Since(start))
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/csv"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"strings"
	"time"

	"github.com/fsnotify/fsnotify"
	"github.com/spf13/viper"
)

type Program struct {
	URL      string
	Duration time.Duration
}

// Loads a list of programs.
// A program consists of a list things to display on the rotator along
// with a number of seconds to display each one before moving on.
func loadProgramList() []Program {
	var list []Program

	filename := viper.GetString("program_file")
	bytes, err := ioutil.ReadFile(filename)
	if err != nil {
		log.Fatal(err)
	}
	webpages := string(bytes)

	r := csv.NewReader(strings.NewReader(webpages))

	for {
		var p Program
		record, err := r.Read()
		if err == io.EOF {
			fmt.Printf("Finished loading programs.\n\n")
			break
		}
		if err != nil {
			log.Fatal(err)
		}
		p.URL = record[0]
		p.Duration, err = time.ParseDuration(record[1])
		fmt.Printf("Loaded program %s to show for %s.\n", p.URL, p.Duration)
		list = append(list, p)
	}

	return list
}

func runProgram(program Program) {

	ip := viper.Get("IP")
	port := viper.GetInt("PORT")

	constr := fmt.Sprintf("%s:%d", ip, port)

	conn, err := net.Dial("tcp", constr)
	if err != nil {
		fmt.Printf("Error making network connection to: %s\n", constr)
		fmt.Println("Pausing for 30s")
		time.Sleep(30 * time.Second) // wait 30 seconds to slow retries
		return
	}

	fmt.Printf("Running program: %s for %s\n", program.URL, program.Duration)
	fmt.Fprintf(conn, "window.location='%s'\n", program.URL)
	status, err := bufio.NewReader(conn).ReadString('\n')
	fmt.Printf("%s", status)
	time.Sleep(program.Duration)
}

// InitializeConfig loads our configuration using Viper package.
func InitializeConfig() {

	viper.SetConfigType("yaml")
	// Set config file
	viper.SetConfigName("config")

	// Add config path
	//	viper.AddConfigPath("$HOME/.gorotator")
	viper.AddConfigPath(".")

	// Read in the config
	err := viper.ReadInConfig() // Find and read the config file
	if err != nil {             // Handle errors reading the config file
		panic(fmt.Errorf("Fatal error config file: %s \n", err))
	}

	// Load default settings
	viper.SetDefault("debug", false)

	viper.SetEnvPrefix("gorotator") // will be uppercased automatically
	viper.BindEnv("debug")
	viper.BindEnv("ip")
	viper.BindEnv("port")

	// Do some flag handling and any complicated config logic
	if !viper.IsSet("ip") || !viper.IsSet("port") {
		fmt.Println("Configuration error.  Both IP and PORT must be set via either config or environment.")
		os.Exit(1)
	}

	viper.WatchConfig()
	viper.OnConfigChange(func(e fsnotify.Event) {
		fmt.Println("Config file changed:", e.Name)
		fmt.Println("Content will change on the next loop.")
	})

}

func main() {

	InitializeConfig()

	for {
		pl := loadProgramList()
		for _, p := range pl {
			runProgram(p)
		}
		fmt.Printf("Looping back to beginning\n\n")
	}
}
<commit_msg>Replaced time.Sleep with select statement; 1st iteration interruptable<commit_after>package main

import (
	"bufio"
	"encoding/csv"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"strings"
	"time"

	"github.com/fsnotify/fsnotify"
	"github.com/spf13/viper"
)

type Program struct {
	URL      string
	Duration time.Duration
}

// Loads a list of programs.
// A program consists of a list things to display on the rotator along
// with a number of seconds to display each one before moving on.
func loadProgramList() []Program {
	var list []Program

	filename := viper.GetString("program_file")
	bytes, err := ioutil.ReadFile(filename)
	if err != nil {
		log.Fatal(err)
	}
	webpages := string(bytes)

	r := csv.NewReader(strings.NewReader(webpages))

	for {
		var p Program
		record, err := r.Read()
		if err == io.EOF {
			fmt.Printf("Finished loading programs.\n\n")
			break
		}
		if err != nil {
			log.Fatal(err)
		}
		p.URL = record[0]
		p.Duration, err = time.ParseDuration(record[1])
		fmt.Printf("Loaded program %s to show for %s.\n", p.URL, p.Duration)
		list = append(list, p)
	}

	return list
}

func runProgram(program Program) {

	ip := viper.Get("IP")
	port := viper.GetInt("PORT")

	constr := fmt.Sprintf("%s:%d", ip, port)

	conn, err := net.Dial("tcp", constr)
	if err != nil {
		fmt.Printf("Error making network connection to: %s\n", constr)
		fmt.Println("Pausing for 30s")
		time.Sleep(30 * time.Second) // wait 30 seconds to slow retries
		return
	}

	fmt.Printf("Running program: %s for %s\n", program.URL, program.Duration)
	fmt.Fprintf(conn, "window.location='%s'\n", program.URL)
	status, err := bufio.NewReader(conn).ReadString('\n')
	fmt.Printf("%s", status)
	// time.Sleep(program.Duration)

	abort := make(chan struct{})

	go func() {
		os.Stdin.Read(make([]byte, 1)) // read a single byte
		abort <- struct{}{}
	}()

	select {
	case <-time.After(program.Duration):
		// Do nothing.
	case <-abort:
		fmt.Println("Current program aborted")
		return
	}
}

// InitializeConfig loads our configuration using Viper package.
func InitializeConfig() {

	viper.SetConfigType("yaml")
	// Set config file
	viper.SetConfigName("config")

	// Add config path
	//	viper.AddConfigPath("$HOME/.gorotator")
	viper.AddConfigPath(".")

	// Read in the config
	err := viper.ReadInConfig() // Find and read the config file
	if err != nil {             // Handle errors reading the config file
		panic(fmt.Errorf("Fatal error config file: %s \n", err))
	}

	// Load default settings
	viper.SetDefault("debug", false)

	viper.SetEnvPrefix("gorotator") // will be uppercased automatically
	viper.BindEnv("debug")
	viper.BindEnv("ip")
	viper.BindEnv("port")

	// Do some flag handling and any complicated config logic
	if !viper.IsSet("ip") || !viper.IsSet("port") {
		fmt.Println("Configuration error.  Both IP and PORT must be set via either config or environment.")
		os.Exit(1)
	}

	viper.WatchConfig()
	viper.OnConfigChange(func(e fsnotify.Event) {
		fmt.Println("Config file changed:", e.Name)
		fmt.Println("Content will change on the next loop.")
	})

}

func main() {

	InitializeConfig()

	for {
		pl := loadProgramList()
		for _, p := range pl {
			runProgram(p)
		}
		fmt.Printf("Looping back to beginning\n\n")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/mibk/dupl/job"
	"github.com/mibk/dupl/printer"
	"github.com/mibk/dupl/syntax"
)

const defaultThreshold = 15

var (
	paths     = []string{"."}
	vendor    = flag.Bool("vendor", false, "")
	verbose   = flag.Bool("verbose", false, "")
	threshold = flag.Int("threshold", defaultThreshold, "")
	files     = flag.Bool("files", false, "")

	html     = flag.Bool("html", false, "")
	plumbing = flag.Bool("plumbing", false, "")
)

const (
	vendorDirPrefix = "vendor" + string(filepath.Separator)
	vendorDirInPath = string(filepath.Separator) + vendorDirPrefix
)

func init() {
	flag.BoolVar(verbose, "v", false, "alias for -verbose")
	flag.IntVar(threshold, "t", defaultThreshold, "alias for -threshold")
}

func main() {
	flag.Usage = usage
	flag.Parse()
	if *html && *plumbing {
		log.Fatal("you can have either plumbing or HTML output")
	}
	if flag.NArg() > 0 {
		paths = flag.Args()
	}

	if *verbose {
		log.Println("Building suffix tree")
	}
	schan := job.Parse(filesFeed())
	t, data, done := job.BuildTree(schan)
	<-done

	// finish stream
	t.Update(&syntax.Node{Type: -1})

	if *verbose {
		log.Println("Searching for clones")
	}
	mchan := t.FindDuplOver(*threshold)
	duplChan := make(chan syntax.Match)
	go func() {
		for m := range mchan {
			match := syntax.FindSyntaxUnits(*data, m, *threshold)
			if len(match.Frags) > 0 {
				duplChan <- match
			}
		}
		close(duplChan)
	}()

	newPrinter := printer.NewText
	if *html {
		newPrinter = printer.NewHTML
	} else if *plumbing {
		newPrinter = printer.NewPlumbing
	}
	p := newPrinter(os.Stdout, ioutil.ReadFile)
	if err := printDupls(p, duplChan); err != nil {
		log.Fatal(err)
	}
}

func filesFeed() chan string {
	if *files {
		fchan := make(chan string)
		go func() {
			s := bufio.NewScanner(os.Stdin)
			for s.Scan() {
				f := s.Text()
				fchan <- strings.TrimPrefix(f, "./")
			}
			close(fchan)
		}()
		return fchan
	}
	return crawlPaths(paths)
}

func crawlPaths(paths []string) chan string {
	fchan := make(chan string)
	go func() {
		for _, path := range paths {
			info, err := os.Lstat(path)
			if err != nil {
				log.Fatal(err)
			}
			if !info.IsDir() {
				fchan <- path
				continue
			}
			filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
				if !*vendor && (strings.HasPrefix(path, vendorDirPrefix) ||
					strings.Contains(path, vendorDirInPath)) {
					return nil
				}
				if !info.IsDir() && strings.HasSuffix(info.Name(), ".go") {
					fchan <- path
				}
				return nil
			})
		}
		close(fchan)
	}()
	return fchan
}

func printDupls(p printer.Printer, duplChan <-chan syntax.Match) error {
	groups := make(map[string][][]*syntax.Node)
	for dupl := range duplChan {
		groups[dupl.Hash] = append(groups[dupl.Hash], dupl.Frags...)
	}
	keys := make([]string, 0, len(groups))
	for k := range groups {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	if err := p.PrintHeader(); err != nil {
		return err
	}
	for _, k := range keys {
		uniq := unique(groups[k])
		if len(uniq) > 1 {
			if err := p.PrintClones(uniq); err != nil {
				return err
			}
		}
	}
	return p.PrintFooter()
}

func unique(group [][]*syntax.Node) [][]*syntax.Node {
	fileMap := make(map[string]map[int]struct{})

	var newGroup [][]*syntax.Node
	for _, seq := range group {
		node := seq[0]
		file, ok := fileMap[node.Filename]
		if !ok {
			file = make(map[int]struct{})
			fileMap[node.Filename] = file
		}
		if _, ok := file[node.Pos]; !ok {
			file[node.Pos] = struct{}{}
			newGroup = append(newGroup, seq)
		}
	}
	return newGroup
}

func usage() {
	fmt.Fprintln(os.Stderr, `Usage: dupl [flags] [paths]

Paths:
  If the given path is a file, dupl will use it regardless of
  the file extension. If it is a directory, it will recursively
  search for *.go files in that directory.

  If no path is given, dupl will recursively search for *.go
  files in the current directory.

Flags:
  -files
    	read file names from stdin one at each line
  -html
    	output the results as HTML, including duplicate code fragments
  -plumbing
    	plumbing (easy-to-parse) output for consumption by scripts or tools
  -t, -threshold size
    	minimum token sequence size as a clone (default 15)
  -vendor
    	check files in vendor directory
  -v, -verbose
    	explain what is being done

Examples:
  dupl -t 100
    	Search clones in the current directory of size at least
    	100 tokens.
  dupl $(find app/ -name '*_test.go')
    	Search for clones in tests in the app directory.
  find app/ -name '*_test.go' |dupl -files
    	The same as above.`)
	os.Exit(2)
}
<commit_msg>Checking returned error<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/mibk/dupl/job"
	"github.com/mibk/dupl/printer"
	"github.com/mibk/dupl/syntax"
)

const defaultThreshold = 15

var (
	paths     = []string{"."}
	vendor    = flag.Bool("vendor", false, "")
	verbose   = flag.Bool("verbose", false, "")
	threshold = flag.Int("threshold", defaultThreshold, "")
	files     = flag.Bool("files", false, "")

	html     = flag.Bool("html", false, "")
	plumbing = flag.Bool("plumbing", false, "")
)

const (
	vendorDirPrefix = "vendor" + string(filepath.Separator)
	vendorDirInPath = string(filepath.Separator) + vendorDirPrefix
)

func init() {
	flag.BoolVar(verbose, "v", false, "alias for -verbose")
	flag.IntVar(threshold, "t", defaultThreshold, "alias for -threshold")
}

func main() {
	flag.Usage = usage
	flag.Parse()
	if *html && *plumbing {
		log.Fatal("you can have either plumbing or HTML output")
	}
	if flag.NArg() > 0 {
		paths = flag.Args()
	}

	if *verbose {
		log.Println("Building suffix tree")
	}
	schan := job.Parse(filesFeed())
	t, data, done := job.BuildTree(schan)
	<-done

	// finish stream
	t.Update(&syntax.Node{Type: -1})

	if *verbose {
		log.Println("Searching for clones")
	}
	mchan := t.FindDuplOver(*threshold)
	duplChan := make(chan syntax.Match)
	go func() {
		for m := range mchan {
			match := syntax.FindSyntaxUnits(*data, m, *threshold)
			if len(match.Frags) > 0 {
				duplChan <- match
			}
		}
		close(duplChan)
	}()

	newPrinter := printer.NewText
	if *html {
		newPrinter = printer.NewHTML
	} else if *plumbing {
		newPrinter = printer.NewPlumbing
	}
	p := newPrinter(os.Stdout, ioutil.ReadFile)
	if err := printDupls(p, duplChan); err != nil {
		log.Fatal(err)
	}
}

func filesFeed() chan string {
	if *files {
		fchan := make(chan string)
		go func() {
			s := bufio.NewScanner(os.Stdin)
			for s.Scan() {
				f := s.Text()
				fchan <- strings.TrimPrefix(f, "./")
			}
			close(fchan)
		}()
		return fchan
	}
	return crawlPaths(paths)
}

func crawlPaths(paths []string) chan string {
	fchan := make(chan string)
	go func() {
		for _, path := range paths {
			info, err := os.Lstat(path)
			if err != nil {
				log.Fatal(err)
			}
			if !info.IsDir() {
				fchan <- path
				continue
			}
			err = filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
				if !*vendor && (strings.HasPrefix(path, vendorDirPrefix) ||
					strings.Contains(path, vendorDirInPath)) {
					return nil
				}
				if !info.IsDir() && strings.HasSuffix(info.Name(), ".go") {
					fchan <- path
				}
				return nil
			})
			if err != nil {
				log.Fatal(err)
			}
		}
		close(fchan)
	}()
	return fchan
}

func printDupls(p printer.Printer, duplChan <-chan syntax.Match) error {
	groups := make(map[string][][]*syntax.Node)
	for dupl := range duplChan {
		groups[dupl.Hash] = append(groups[dupl.Hash], dupl.Frags...)
	}
	keys := make([]string, 0, len(groups))
	for k := range groups {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	if err := p.PrintHeader(); err != nil {
		return err
	}
	for _, k := range keys {
		uniq := unique(groups[k])
		if len(uniq) > 1 {
			if err := p.PrintClones(uniq); err != nil {
				return err
			}
		}
	}
	return p.PrintFooter()
}

func unique(group [][]*syntax.Node) [][]*syntax.Node {
	fileMap := make(map[string]map[int]struct{})

	var newGroup [][]*syntax.Node
	for _, seq := range group {
		node := seq[0]
		file, ok := fileMap[node.Filename]
		if !ok {
			file = make(map[int]struct{})
			fileMap[node.Filename] = file
		}
		if _, ok := file[node.Pos]; !ok {
			file[node.Pos] = struct{}{}
			newGroup = append(newGroup, seq)
		}
	}
	return newGroup
}

func usage() {
	fmt.Fprintln(os.Stderr, `Usage: dupl [flags] [paths]

Paths:
  If the given path is a file, dupl will use it regardless of
  the file extension. If it is a directory, it will recursively
  search for *.go files in that directory.

  If no path is given, dupl will recursively search for *.go
  files in the current directory.

Flags:
  -files
    	read file names from stdin one at each line
  -html
    	output the results as HTML, including duplicate code fragments
  -plumbing
    	plumbing (easy-to-parse) output for consumption by scripts or tools
  -t, -threshold size
    	minimum token sequence size as a clone (default 15)
  -vendor
    	check files in vendor directory
  -v, -verbose
    	explain what is being done

Examples:
  dupl -t 100
    	Search clones in the current directory of size at least
    	100 tokens.
  dupl $(find app/ -name '*_test.go')
    	Search for clones in tests in the app directory.
  find app/ -name '*_test.go' |dupl -files
    	The same as above.`)
	os.Exit(2)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"os"
	"os/exec"
	"strings"
	"sync"
)

var BuildSha1Version string

func checkError(err error, prefix string) {
	if err != nil {
		panic(fmt.Sprintf("%s %s", prefix, err.Error()))
	}
}

func cleanFeedbackLine(s string) (out string) {
	out = s
	out = strings.TrimSpace(out)
	out = strings.Replace(out, "\r", "", -1)
	out = strings.Replace(out, "\n", "\\n", -1)
	return
}

func runCommand(wg *sync.WaitGroup, resultCollector *ResultCollector, commandIndex int, phase *nodeData, cmd *exec.Cmd) {
	if wg != nil {
		defer wg.Done()
	}

	defer func() {
		if r := recover(); r != nil {
			errMsg := fmt.Sprintf("%s", r)
			if !phase.ContinueOnFailure {
				logger.Fatallnf(errMsg)
			} else {
				logger.Errorlnf(errMsg)
			}

			resultCollector.AppendResult(&result{Cmd: cmd, Successful: false})
		} else {
			resultCollector.AppendResult(&result{Cmd: cmd, Successful: true})
		}
	}()

	stdout, err := cmd.StdoutPipe()
	checkError(err, "cmd.StdoutPipe")

	stderr, err := cmd.StderrPipe()
	checkError(err, "cmd.StderrPipe")

	outLines := []string{}
	stdoutScanner := bufio.NewScanner(stdout)
	go func() {
		for stdoutScanner.Scan() {
			txt := cleanFeedbackLine(stdoutScanner.Text())
			outLines = append(outLines, txt)
			logger.Infolnf("COMMAND %d STDOUT: %s", commandIndex, txt)
		}
	}()

	errLines := []string{}
	stderrScanner := bufio.NewScanner(stderr)
	go func() {
		for stderrScanner.Scan() {
			txt := cleanFeedbackLine(stderrScanner.Text())
			errLines = append(errLines, txt)
			logger.Warninglnf("COMMAND %d STDERR: %s", commandIndex, txt)
		}
	}()

	err = cmd.Start()
	checkError(err, "cmd.Start")

	err = cmd.Wait()
	if err != nil {
		newErr := fmt.Errorf("%s - lines: %s", err.Error(), strings.Join(errLines, "\\n"))
		outCombined := cleanFeedbackLine(strings.Join(outLines, "\n"))

		errMsg := fmt.Sprintf("FAILED COMMAND %d. Continue: %t. ERROR: %s. OUT: %s", commandIndex, phase.ContinueOnFailure, newErr.Error(), outCombined)
		panic(errMsg)
	}
}

func runPhase(setup *setup, phaseName string, phase *nodeData) {
	var wg sync.WaitGroup
	resultCollector := &ResultCollector{}

	cmds, err := phase.GetExecCommandsFromSteps(setup.StringVariablesOnly(), setup.Variables)
	if err != nil {
		logger.Fatallnf(err.Error())
	}

	if phase.RunParallel {
		wg.Add(len(cmds))
	}

	logger.Infolnf("Running step %s", phaseName)
	for ind, c := range cmds {
		logger.Infolnf(strings.TrimSpace(fmt.Sprintf(`INDEX %d = %s`, ind, execCommandToDisplayString(c))))

		var wgToUse *sync.WaitGroup = nil
		if phase.RunParallel {
			wgToUse = &wg
			go runCommand(wgToUse, resultCollector, ind, phase, c)
		} else {
			runCommand(wgToUse, resultCollector, ind, phase, c)
		}
	}

	if phase.RunParallel {
		wg.Wait()
	}

	if resultCollector.FailedCount() == 0 {
		logger.Infolnf("There were no failures - %d/%d successful", resultCollector.SuccessCount(), resultCollector.TotalCount())
	} else {
		logger.Errorlnf("There were %d/%d failures: ", resultCollector.FailedCount(), resultCollector.TotalCount())
		for _, failure := range resultCollector.FailedDisplayList() {
			logger.Errorlnf("- %s", failure)
		}
	}
}

func main() {
	logger.Infolnf("Running version '%s'", BuildSha1Version)

	if len(os.Args) < 2 {
		logger.Fatallnf("The first command-line argument must be the YAML file path.")
	}

	yamlFilePath := os.Args[1]

	setup, err := ParseYamlFile(yamlFilePath)
	if err != nil {
		logger.Fatallnf(err.Error())
	}

	for _, phase := range setup.Phases {
		fmt.Println(phase.Name)
	}
	return

	for _, phase := range setup.Phases {
		runPhase(setup, phase.Name, phase.Data)
	}
}
<commit_msg>Fix previous commit issue.<commit_after>package main

import (
	"bufio"
	"fmt"
	"os"
	"os/exec"
	"strings"
	"sync"
)

var BuildSha1Version string

func checkError(err error, prefix string) {
	if err != nil {
		panic(fmt.Sprintf("%s %s", prefix, err.Error()))
	}
}

func cleanFeedbackLine(s string) (out string) {
	out = s
	out = strings.TrimSpace(out)
	out = strings.Replace(out, "\r", "", -1)
	out = strings.Replace(out, "\n", "\\n", -1)
	return
}

func runCommand(wg *sync.WaitGroup, resultCollector *ResultCollector, commandIndex int, phase *nodeData, cmd *exec.Cmd) {
	if wg != nil {
		defer wg.Done()
	}

	defer func() {
		if r := recover(); r != nil {
			errMsg := fmt.Sprintf("%s", r)
			if !phase.ContinueOnFailure {
				logger.Fatallnf(errMsg)
			} else {
				logger.Errorlnf(errMsg)
			}

			resultCollector.AppendResult(&result{Cmd: cmd, Successful: false})
		} else {
			resultCollector.AppendResult(&result{Cmd: cmd, Successful: true})
		}
	}()

	stdout, err := cmd.StdoutPipe()
	checkError(err, "cmd.StdoutPipe")

	stderr, err := cmd.StderrPipe()
	checkError(err, "cmd.StderrPipe")

	outLines := []string{}
	stdoutScanner := bufio.NewScanner(stdout)
	go func() {
		for stdoutScanner.Scan() {
			txt := cleanFeedbackLine(stdoutScanner.Text())
			outLines = append(outLines, txt)
			logger.Tracelnf("COMMAND %d STDOUT: %s", commandIndex, txt)
		}
	}()

	errLines := []string{}
	stderrScanner := bufio.NewScanner(stderr)
	go func() {
		for stderrScanner.Scan() {
			txt := cleanFeedbackLine(stderrScanner.Text())
			errLines = append(errLines, txt)
			logger.Warninglnf("COMMAND %d STDERR: %s", commandIndex, txt)
		}
	}()

	err = cmd.Start()
	checkError(err, "cmd.Start")

	err = cmd.Wait()
	if err != nil {
		newErr := fmt.Errorf("%s - lines: %s", err.Error(), strings.Join(errLines, "\\n"))
		outCombined := cleanFeedbackLine(strings.Join(outLines, "\n"))

		errMsg := fmt.Sprintf("FAILED COMMAND %d. Continue: %t. ERROR: %s. OUT: %s", commandIndex, phase.ContinueOnFailure, newErr.Error(), outCombined)
		panic(errMsg)
	}
}

func runPhase(setup *setup, phaseName string, phase *nodeData) {
	var wg sync.WaitGroup
	resultCollector := &ResultCollector{}

	cmds, err := phase.GetExecCommandsFromSteps(setup.StringVariablesOnly(), setup.Variables)
	if err != nil {
		logger.Fatallnf(err.Error())
	}

	if phase.RunParallel {
		wg.Add(len(cmds))
	}

	logger.Infolnf("Running step %s", phaseName)
	for ind, c := range cmds {
		logger.Tracelnf(strings.TrimSpace(fmt.Sprintf(`INDEX %d = %s`, ind, execCommandToDisplayString(c))))

		var wgToUse *sync.WaitGroup = nil
		if phase.RunParallel {
			wgToUse = &wg
			go runCommand(wgToUse, resultCollector, ind, phase, c)
		} else {
			runCommand(wgToUse, resultCollector, ind, phase, c)
		}
	}

	if phase.RunParallel {
		wg.Wait()
	}

	if resultCollector.FailedCount() == 0 {
		logger.Infolnf("There were no failures - %d/%d successful", resultCollector.SuccessCount(), resultCollector.TotalCount())
	} else {
		logger.Errorlnf("There were %d/%d failures: ", resultCollector.FailedCount(), resultCollector.TotalCount())
		for _, failure := range resultCollector.FailedDisplayList() {
			logger.Errorlnf("- %s", failure)
		}
	}
}

func main() {
	logger.Infolnf("Running version '%s'", BuildSha1Version)

	if len(os.Args) < 2 {
		logger.Fatallnf("The first command-line argument must be the YAML file path.")
	}

	yamlFilePath := os.Args[1]

	setup, err := ParseYamlFile(yamlFilePath)
	if err != nil {
		logger.Fatallnf(err.Error())
	}

	for _, phase := range setup.Phases {
		runPhase(setup, phase.Name, phase.Data)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math/rand"
	"net"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/codegangsta/cli"
	"github.com/miekg/dns"
)

const (
	Version = "0.0.1"
)

var (
	root string
)

func rrsearch(ns string, q string, t uint16) *dns.Msg {
	c := new(dns.Client)
	m := new(dns.Msg)
	m.SetQuestion(dns.Fqdn(q), t)

	r, _, err := c.Exchange(m, net.JoinHostPort(ns, "53"))

	if err != nil {
		fmt.Println(err.Error())
		panic("rrsearch Error")
	}
	return r
}

func splitRR(rr dns.RR) []string {
	rrary := strings.SplitN(rr.String(), "\t", 5)
	return rrary
}

func setNS(rrs []string, r *dns.Msg, qns string) (string, string) {
	var ns string
	typ := "A"

	if strings.Contains(rrs[4], rrs[0]) || strings.Compare(qns, root) == 0 {
		for _, rr := range r.Extra {
			rrss := splitRR(rr)
			if strings.Compare(rrss[0], rrs[4]) == 0 {
				ns = rrss[4]
				typ = rrss[3]
			}
		}
	} else {
		ns = rrs[4]
	}

	return ns, typ
}

func noRecRsolve(dst, ns string) string {
	rand.Seed(time.Now().UnixNano())
	for {
		r := rrsearch(ns, dst, dns.TypeA)
		if len(r.Answer) == 0 {
			rrs := splitRR(r.Ns[rand.Intn(len(r.Ns))])
			nss, typ := setNS(rrs, r, ns)
			for strings.Compare(typ, "AAAA") == 0 {
				rrs := splitRR(r.Ns[rand.Intn(len(r.Ns))])
				nss, typ = setNS(rrs, r, ns)
			}
			fmt.Println(ns + "=>")
			fmt.Printf("\t%s -> %s\n",
				rrs[0],
				rrs[4],
			)
			if strings.Compare(rrs[0], ".") == 0 {
				fmt.Println("Domain Name Not found")
				os.Exit(1)
			}
			ns = nss
		} else {
			fmt.Println("Answer : " + ns + "=>")
			var ip string
			for _, ans := range r.Answer {
				anss := splitRR(ans)
				fmt.Printf("\t%s -> %s\n",
					dst,
					anss[4],
				)
				ip = anss[4]
			}
			return ip
		}
	}
}

func recRsolve(dst, ns string, n int) string {
	var tab string
	for i := 0; i < n; i++ {
		tab = tab + "\t"
	}
	rand.Seed(time.Now().UnixNano())
	for {
		checkNS, _ := regexp.MatchString("^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}$", ns)
		if checkNS == false {
			ns = recRsolve(ns, root, n+1)
		}
		r := rrsearch(ns, dst, dns.TypeA)
		if len(r.Answer) == 0 {
			rrs := splitRR(r.Ns[rand.Intn(len(r.Ns))])
			nss, typ := setNS(rrs, r, ns)
			for strings.Compare(typ, "AAAA") == 0 {
				rrs := splitRR(r.Ns[rand.Intn(len(r.Ns))])
				nss, typ = setNS(rrs, r, ns)
			}
			fmt.Println(tab + ns + "=>")
			fmt.Printf(tab+"\t%s -> %s\n",
				rrs[0],
				rrs[4],
			)
			if strings.Compare(rrs[0], ".") == 0 {
				fmt.Println("Domain Name Not found")
				os.Exit(1)
			}

			ns = nss
		} else {
			fmt.Println(tab + "Answer : " + ns + "=>")
			var ip string
			for _, ans := range r.Answer {
				anss := splitRR(ans)
				fmt.Printf(tab+"\t%s -> %s\n",
					dst,
					anss[4],
				)
				ip = anss[4]
			}
			return ip
		}
	}
}

func main() {
	app := cli.NewApp()
	app.Usage = "It is a tool to see recursive search of the DNS is how to."
	app.Version = Version

	app.Commands = []cli.Command{
		{
			Name:    "rec",
			Aliases: []string{"r"},
			Usage:   "Iterate Search",
			Action: func(c *cli.Context) {
				recRsolve(c.Args().First(), root, 0)
			},
		},
		{
			Name:    "norec",
			Aliases: []string{"n"},
			Usage:   "No Iterate Search",
			Action: func(c *cli.Context) {
				noRecRsolve(c.Args().First(), root)
			},
		},
	}

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:        "root",
			Value:       "202.12.27.33",
			Usage:       "Root DNS Server's IP address",
			Destination: &root,
		},
	}
	app.Run(os.Args)
}
<commit_msg>バージョン変更<commit_after>package main

import (
	"fmt"
	"math/rand"
	"net"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/codegangsta/cli"
	"github.com/miekg/dns"
)

const (
	Version = "0.0.2"
)

var (
	root string
)

func rrsearch(ns string, q string, t uint16) *dns.Msg {
	c := new(dns.Client)
	m := new(dns.Msg)
	m.SetQuestion(dns.Fqdn(q), t)

	r, _, err := c.Exchange(m, net.JoinHostPort(ns, "53"))

	if err != nil {
		fmt.Println(err.Error())
		panic("rrsearch Error")
	}
	return r
}

func splitRR(rr dns.RR) []string {
	rrary := strings.SplitN(rr.String(), "\t", 5)
	return rrary
}

func setNS(rrs []string, r *dns.Msg, qns string) (string, string) {
	var ns string
	typ := "A"

	if strings.Contains(rrs[4], rrs[0]) || strings.Compare(qns, root) == 0 {
		for _, rr := range r.Extra {
			rrss := splitRR(rr)
			if strings.Compare(rrss[0], rrs[4]) == 0 {
				ns = rrss[4]
				typ = rrss[3]
			}
		}
	} else {
		ns = rrs[4]
	}

	return ns, typ
}

func noRecRsolve(dst, ns string) string {
	rand.Seed(time.Now().UnixNano())
	for {
		r := rrsearch(ns, dst, dns.TypeA)
		if len(r.Answer) == 0 {
			rrs := splitRR(r.Ns[rand.Intn(len(r.Ns))])
			nss, typ := setNS(rrs, r, ns)
			for strings.Compare(typ, "AAAA") == 0 {
				rrs := splitRR(r.Ns[rand.Intn(len(r.Ns))])
				nss, typ = setNS(rrs, r, ns)
			}
			fmt.Println(ns + "=>")
			fmt.Printf("\t%s -> %s\n",
				rrs[0],
				rrs[4],
			)
			if strings.Compare(rrs[0], ".") == 0 {
				fmt.Println("Domain Name Not found")
				os.Exit(1)
			}
			ns = nss
		} else {
			fmt.Println("Answer : " + ns + "=>")
			var ip string
			for _, ans := range r.Answer {
				anss := splitRR(ans)
				fmt.Printf("\t%s -> %s\n",
					dst,
					anss[4],
				)
				ip = anss[4]
			}
			return ip
		}
	}
}

func recRsolve(dst, ns string, n int) string {
	var tab string
	for i := 0; i < n; i++ {
		tab = tab + "\t"
	}
	rand.Seed(time.Now().UnixNano())
	for {
		checkNS, _ := regexp.MatchString("^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}$", ns)
		if checkNS == false {
			ns = recRsolve(ns, root, n+1)
		}
		r := rrsearch(ns, dst, dns.TypeA)
		if len(r.Answer) == 0 {
			rrs := splitRR(r.Ns[rand.Intn(len(r.Ns))])
			nss, typ := setNS(rrs, r, ns)
			for strings.Compare(typ, "AAAA") == 0 {
				rrs := splitRR(r.Ns[rand.Intn(len(r.Ns))])
				nss, typ = setNS(rrs, r, ns)
			}
			fmt.Println(tab + ns + "=>")
			fmt.Printf(tab+"\t%s -> %s\n",
				rrs[0],
				rrs[4],
			)
			if strings.Compare(rrs[0], ".") == 0 {
				fmt.Println("Domain Name Not found")
				os.Exit(1)
			}

			ns = nss
		} else {
			fmt.Println(tab + "Answer : " + ns + "=>")
			var ip string
			for _, ans := range r.Answer {
				anss := splitRR(ans)
				fmt.Printf(tab+"\t%s -> %s\n",
					dst,
					anss[4],
				)
				ip = anss[4]
			}
			return ip
		}
	}
}

func main() {
	app := cli.NewApp()
	app.Usage = "It is a tool to see recursive search of the DNS is how to."
	app.Version = Version

	app.Commands = []cli.Command{
		{
			Name:    "rec",
			Aliases: []string{"r"},
			Usage:   "Iterate Search",
			Action: func(c *cli.Context) {
				recRsolve(c.Args().First(), root, 0)
			},
		},
		{
			Name:    "norec",
			Aliases: []string{"n"},
			Usage:   "No Iterate Search",
			Action: func(c *cli.Context) {
				noRecRsolve(c.Args().First(), root)
			},
		},
	}

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:        "root",
			Value:       "202.12.27.33",
			Usage:       "Root DNS Server's IP address",
			Destination: &root,
		},
	}
	app.Run(os.Args)
}
<|endoftext|>