<commit_before>package flags

import (
	"os"
	"time"

	log "github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

// DockerAPIMinVersion is the minimum version of the docker api required to
// use watchtower
const DockerAPIMinVersion string = "1.24"

// RegisterDockerFlags that are used directly by the docker api client
func RegisterDockerFlags(rootCmd *cobra.Command) {
	flags := rootCmd.PersistentFlags()
	flags.StringP("host", "H", viper.GetString("DOCKER_HOST"), "daemon socket to connect to")
	flags.BoolP("tlsverify", "v", viper.GetBool("DOCKER_TLS_VERIFY"), "use TLS and verify the remote")
	flags.StringP("api-version", "a", viper.GetString("DOCKER_API_VERSION"), "api version to use by docker client")
}

// RegisterSystemFlags that are used by watchtower to modify the program flow
func RegisterSystemFlags(rootCmd *cobra.Command) {
	flags := rootCmd.PersistentFlags()
	flags.IntP(
		"interval",
		"i",
		viper.GetInt("WATCHTOWER_POLL_INTERVAL"),
		"poll interval (in seconds)")

	flags.StringP("schedule",
		"s",
		viper.GetString("WATCHTOWER_SCHEDULE"),
		"the cron expression which defines when to update")

	flags.DurationP("stop-timeout",
		"t",
		viper.GetDuration("WATCHTOWER_TIMEOUT"),
		"timeout before a container is forcefully stopped")

	flags.BoolP(
		"no-pull",
		"",
		viper.GetBool("WATCHTOWER_NO_PULL"),
		"do not pull any new images")

	flags.BoolP(
		"no-restart",
		"",
		viper.GetBool("WATCHTOWER_NO_RESTART"),
		"do not restart any containers")

	flags.BoolP(
		"cleanup",
		"c",
		viper.GetBool("WATCHTOWER_CLEANUP"),
		"remove previously used images after updating")

	flags.BoolP(
		"remove-volumes",
		"",
		viper.GetBool("WATCHTOWER_REMOVE_VOLUMES"),
		"remove attached volumes before updating")

	flags.BoolP(
		"label-enable",
		"e",
		viper.GetBool("WATCHTOWER_LABEL_ENABLE"),
		"watch containers where the com.centurylinklabs.watchtower.enable label is true")

	flags.BoolP(
		"debug",
		"d",
		viper.GetBool("WATCHTOWER_DEBUG"),
		"enable debug mode with verbose logging")

	flags.BoolP(
		"monitor-only",
		"m",
		viper.GetBool("WATCHTOWER_MONITOR_ONLY"),
		"Will only monitor for new images, not update the containers")

	flags.BoolP(
		"run-once",
		"R",
		viper.GetBool("WATCHTOWER_RUN_ONCE"),
		"Run once now and exit")

	flags.BoolP(
		"include-stopped",
		"S",
		viper.GetBool("WATCHTOWER_INCLUDE_STOPPED"),
		"Will also include created and exited containers")

	flags.BoolP(
		"enable-lifecycle-hooks",
		"",
		viper.GetBool("WATCHTOWER_LIFECYCLE_HOOKS"),
		"Enable the execution of commands triggered by pre- and post-update lifecycle hooks")
}

// RegisterNotificationFlags that are used by watchtower to send notifications
func RegisterNotificationFlags(rootCmd *cobra.Command) {
	flags := rootCmd.PersistentFlags()

	flags.StringSliceP(
		"notifications",
		"n",
		viper.GetStringSlice("WATCHTOWER_NOTIFICATIONS"),
		" notification types to send (valid: email, slack, msteams, gotify)")

	flags.StringP(
		"notifications-level",
		"",
		viper.GetString("WATCHTOWER_NOTIFICATIONS_LEVEL"),
		"The log level used for sending notifications. Possible values: panic, fatal, error, warn, info or debug")

	flags.StringP(
		"notification-email-from",
		"",
		viper.GetString("WATCHTOWER_NOTIFICATION_EMAIL_FROM"),
		"Address to send notification emails from")

	flags.StringP(
		"notification-email-to",
		"",
		viper.GetString("WATCHTOWER_NOTIFICATION_EMAIL_TO"),
		"Address to send notification emails to")

	flags.IntP(
		"notification-email-delay",
		"",
		viper.GetInt("WATCHTOWER_NOTIFICATION_EMAIL_DELAY"),
		"Delay before sending notifications, expressed in seconds")

	flags.StringP(
		"notification-email-server",
		"",
		viper.GetString("WATCHTOWER_NOTIFICATION_EMAIL_SERVER"),
		"SMTP server to send notification emails through")

	flags.IntP(
		"notification-email-server-port",
		"",
		viper.GetInt("WATCHTOWER_NOTIFICATION_EMAIL_SERVER_PORT"),
		"SMTP server port to send notification emails through")

	flags.BoolP(
		"notification-email-server-tls-skip-verify",
		"",
		viper.GetBool("WATCHTOWER_NOTIFICATION_EMAIL_SERVER_TLS_SKIP_VERIFY"),
		`
Controls whether watchtower verifies the SMTP server's certificate chain and host name.
Should only be used for testing.
`)

	flags.StringP(
		"notification-email-server-user",
		"",
		viper.GetString("WATCHTOWER_NOTIFICATION_EMAIL_SERVER_USER"),
		"SMTP server user for sending notifications")

	flags.StringP(
		"notification-email-server-password",
		"",
		viper.GetString("WATCHTOWER_NOTIFICATION_EMAIL_SERVER_PASSWORD"),
		"SMTP server password for sending notifications")

	flags.StringP(
		"notification-slack-hook-url",
		"",
		viper.GetString("WATCHTOWER_NOTIFICATION_SLACK_HOOK_URL"),
		"The Slack Hook URL to send notifications to")

	flags.StringP(
		"notification-slack-identifier",
		"",
		viper.GetString("WATCHTOWER_NOTIFICATION_SLACK_IDENTIFIER"),
		"A string which will be used to identify the messages coming from this watchtower instance")

	flags.StringP(
		"notification-slack-channel",
		"",
		viper.GetString("WATCHTOWER_NOTIFICATION_SLACK_CHANNEL"),
		"A string which overrides the webhook's default channel. Example: #my-custom-channel")

	flags.StringP(
		"notification-slack-icon-emoji",
		"",
		viper.GetString("WATCHTOWER_NOTIFICATION_SLACK_ICON_EMOJI"),
		"An emoji code string to use in place of the default icon")

	flags.StringP(
		"notification-slack-icon-url",
		"",
		viper.GetString("WATCHTOWER_NOTIFICATION_SLACK_ICON_URL"),
		"An icon image URL string to use in place of the default icon")

	flags.StringP(
		"notification-msteams-hook",
		"",
		viper.GetString("WATCHTOWER_NOTIFICATION_MSTEAMS_HOOK_URL"),
		"The MSTeams WebHook URL to send notifications to")

	flags.BoolP(
		"notification-msteams-data",
		"",
		viper.GetBool("WATCHTOWER_NOTIFICATION_MSTEAMS_USE_LOG_DATA"),
		"The MSTeams notifier will try to extract log entry fields as MSTeams message facts")

	flags.StringP(
		"notification-gotify-url",
		"",
		viper.GetString("WATCHTOWER_NOTIFICATION_GOTIFY_URL"),
		"The Gotify URL to send notifications to")
	flags.StringP(
		"notification-gotify-token",
		"",
		viper.GetString("WATCHTOWER_NOTIFICATION_GOTIFY_TOKEN"),
		"The Gotify Application required to query the Gotify API")
}

// SetDefaults provides default values for environment variables
func SetDefaults() {
	viper.AutomaticEnv()
	viper.SetDefault("DOCKER_HOST", "unix:///var/run/docker.sock")
	viper.SetDefault("DOCKER_API_VERSION", DockerAPIMinVersion)
	viper.SetDefault("WATCHTOWER_POLL_INTERVAL", 300)
	viper.SetDefault("WATCHTOWER_TIMEOUT", time.Second*10)
	viper.SetDefault("WATCHTOWER_NOTIFICATIONS", []string{})
	viper.SetDefault("WATCHTOWER_NOTIFICATIONS_LEVEL", "info")
	viper.SetDefault("WATCHTOWER_NOTIFICATION_EMAIL_SERVER_PORT", 25)
	viper.SetDefault("WATCHTOWER_NOTIFICATION_SLACK_IDENTIFIER", "watchtower")
}

// EnvConfig translates the command-line options into environment variables
// that will initialize the api client
func EnvConfig(cmd *cobra.Command) error {
	var err error
	var host string
	var tls bool
	var version string

	flags := cmd.PersistentFlags()

	if host, err = flags.GetString("host"); err != nil {
		return err
	}
	if tls, err = flags.GetBool("tlsverify"); err != nil {
		return err
	}
	if version, err = flags.GetString("api-version"); err != nil {
		return err
	}
	if err = setEnvOptStr("DOCKER_HOST", host); err != nil {
		return err
	}
	if err = setEnvOptBool("DOCKER_TLS_VERIFY", tls); err != nil {
		return err
	}
	if err = setEnvOptStr("DOCKER_API_VERSION", version); err != nil {
		return err
	}
	return nil
}

// ReadFlags reads common flags used in the main program flow of watchtower
func ReadFlags(cmd *cobra.Command) (bool, bool, bool, time.Duration) {
	flags := cmd.PersistentFlags()

	var err error
	var cleanup bool
	var noRestart bool
	var monitorOnly bool
	var timeout time.Duration

	if cleanup, err = flags.GetBool("cleanup"); err != nil {
		log.Fatal(err)
	}
	if noRestart, err = flags.GetBool("no-restart"); err != nil {
		log.Fatal(err)
	}
	if monitorOnly, err = flags.GetBool("monitor-only"); err != nil {
		log.Fatal(err)
	}
	if timeout, err = flags.GetDuration("stop-timeout"); err != nil {
		log.Fatal(err)
	}

	return cleanup, noRestart, monitorOnly, timeout
}

func setEnvOptStr(env string, opt string) error {
	if opt == "" || opt == os.Getenv(env) {
		return nil
	}
	err := os.Setenv(env, opt)
	if err != nil {
		return err
	}
	return nil
}

func setEnvOptBool(env string, opt bool) error {
	if opt {
		return setEnvOptStr(env, "1")
	}
	return nil
}
<commit_msg>Update flags.go<commit_after>package flags

import (
	"os"
	"time"

	log "github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

// DockerAPIMinVersion is the minimum version of the docker api required to
// use watchtower
const DockerAPIMinVersion string = "1.24"

// RegisterDockerFlags that are used directly by the docker api client
func RegisterDockerFlags(rootCmd *cobra.Command) {
	flags := rootCmd.PersistentFlags()
	flags.StringP("host", "H", viper.GetString("DOCKER_HOST"), "daemon socket to connect to")
	flags.BoolP("tlsverify", "v", viper.GetBool("DOCKER_TLS_VERIFY"), "use TLS and verify the remote")
	flags.StringP("api-version", "a", viper.GetString("DOCKER_API_VERSION"), "api version to use by docker client")
}

// RegisterSystemFlags that are used by watchtower to modify the program flow
func RegisterSystemFlags(rootCmd *cobra.Command) {
	flags := rootCmd.PersistentFlags()
	flags.IntP(
		"interval",
		"i",
		viper.GetInt("WATCHTOWER_POLL_INTERVAL"),
		"poll interval (in seconds)")

	flags.StringP("schedule",
		"s",
		viper.GetString("WATCHTOWER_SCHEDULE"),
		"the cron expression which defines when to update")

	flags.DurationP("stop-timeout",
		"t",
		viper.GetDuration("WATCHTOWER_TIMEOUT"),
		"timeout before a container is forcefully stopped")

	flags.BoolP(
		"no-pull",
		"",
		viper.GetBool("WATCHTOWER_NO_PULL"),
		"do not pull any new images")

	flags.BoolP(
		"no-restart",
		"",
		viper.GetBool("WATCHTOWER_NO_RESTART"),
		"do not restart any containers")

	flags.BoolP(
		"cleanup",
		"c",
		viper.GetBool("WATCHTOWER_CLEANUP"),
		"remove previously used images after updating")

	flags.BoolP(
		"remove-volumes",
		"",
		viper.GetBool("WATCHTOWER_REMOVE_VOLUMES"),
		"remove attached volumes before updating")

	flags.BoolP(
		"label-enable",
		"e",
		viper.GetBool("WATCHTOWER_LABEL_ENABLE"),
		"watch containers where the com.centurylinklabs.watchtower.enable label is true")

	flags.BoolP(
		"debug",
		"d",
		viper.GetBool("WATCHTOWER_DEBUG"),
		"enable debug mode with verbose logging")

	flags.BoolP(
		"monitor-only",
		"m",
		viper.GetBool("WATCHTOWER_MONITOR_ONLY"),
		"Will only monitor for new images, not update the containers")

	flags.BoolP(
		"run-once",
		"R",
		viper.GetBool("WATCHTOWER_RUN_ONCE"),
		"Run once now and exit")

	flags.BoolP(
		"include-stopped",
		"S",
		viper.GetBool("WATCHTOWER_INCLUDE_STOPPED"),
		"Will also include created and exited containers")

	flags.BoolP(
		"enable-lifecycle-hooks",
		"",
		viper.GetBool("WATCHTOWER_LIFECYCLE_HOOKS"),
		"Enable the execution of commands triggered by pre- and post-update lifecycle hooks")
}

// RegisterNotificationFlags that are used by watchtower to send notifications
func RegisterNotificationFlags(rootCmd *cobra.Command) {
	flags := rootCmd.PersistentFlags()

	flags.StringSliceP(
		"notifications",
		"n",
		viper.GetStringSlice("WATCHTOWER_NOTIFICATIONS"),
		" notification types to send (valid: email, slack, msteams, gotify)")

	flags.StringP(
		"notifications-level",
		"",
		viper.GetString("WATCHTOWER_NOTIFICATIONS_LEVEL"),
		"The log level used for sending notifications. Possible values: panic, fatal, error, warn, info or debug")

	flags.StringP(
		"notification-email-from",
		"",
		viper.GetString("WATCHTOWER_NOTIFICATION_EMAIL_FROM"),
		"Address to send notification emails from")

	flags.StringP(
		"notification-email-to",
		"",
		viper.GetString("WATCHTOWER_NOTIFICATION_EMAIL_TO"),
		"Address to send notification emails to")

	flags.IntP(
		"notification-email-delay",
		"",
		viper.GetInt("WATCHTOWER_NOTIFICATION_EMAIL_DELAY"),
		"Delay before sending notifications, expressed in seconds")

	flags.StringP(
		"notification-email-server",
		"",
		viper.GetString("WATCHTOWER_NOTIFICATION_EMAIL_SERVER"),
		"SMTP server to send notification emails through")

	flags.IntP(
		"notification-email-server-port",
		"",
		viper.GetInt("WATCHTOWER_NOTIFICATION_EMAIL_SERVER_PORT"),
		"SMTP server port to send notification emails through")

	flags.BoolP(
		"notification-email-server-tls-skip-verify",
		"",
		viper.GetBool("WATCHTOWER_NOTIFICATION_EMAIL_SERVER_TLS_SKIP_VERIFY"),
		`
Controls whether watchtower verifies the SMTP server's certificate chain and host name.
Should only be used for testing.
`)

	flags.StringP(
		"notification-email-server-user",
		"",
		viper.GetString("WATCHTOWER_NOTIFICATION_EMAIL_SERVER_USER"),
		"SMTP server user for sending notifications")

	flags.StringP(
		"notification-email-server-password",
		"",
		viper.GetString("WATCHTOWER_NOTIFICATION_EMAIL_SERVER_PASSWORD"),
		"SMTP server password for sending notifications")

	flags.StringP(
		"notification-email-subjecttag",
		"",
		viper.GetString("WATCHTOWER_NOTIFICATION_EMAIL_SUBJECTTAG"),
		"Subject prefix tag for notifications via mail")

	flags.StringP(
		"notification-slack-hook-url",
		"",
		viper.GetString("WATCHTOWER_NOTIFICATION_SLACK_HOOK_URL"),
		"The Slack Hook URL to send notifications to")

	flags.StringP(
		"notification-slack-identifier",
		"",
		viper.GetString("WATCHTOWER_NOTIFICATION_SLACK_IDENTIFIER"),
		"A string which will be used to identify the messages coming from this watchtower instance")

	flags.StringP(
		"notification-slack-channel",
		"",
		viper.GetString("WATCHTOWER_NOTIFICATION_SLACK_CHANNEL"),
		"A string which overrides the webhook's default channel. Example: #my-custom-channel")

	flags.StringP(
		"notification-slack-icon-emoji",
		"",
		viper.GetString("WATCHTOWER_NOTIFICATION_SLACK_ICON_EMOJI"),
		"An emoji code string to use in place of the default icon")

	flags.StringP(
		"notification-slack-icon-url",
		"",
		viper.GetString("WATCHTOWER_NOTIFICATION_SLACK_ICON_URL"),
		"An icon image URL string to use in place of the default icon")

	flags.StringP(
		"notification-msteams-hook",
		"",
		viper.GetString("WATCHTOWER_NOTIFICATION_MSTEAMS_HOOK_URL"),
		"The MSTeams WebHook URL to send notifications to")

	flags.BoolP(
		"notification-msteams-data",
		"",
		viper.GetBool("WATCHTOWER_NOTIFICATION_MSTEAMS_USE_LOG_DATA"),
		"The MSTeams notifier will try to extract log entry fields as MSTeams message facts")

	flags.StringP(
		"notification-gotify-url",
		"",
		viper.GetString("WATCHTOWER_NOTIFICATION_GOTIFY_URL"),
		"The Gotify URL to send notifications to")
	flags.StringP(
		"notification-gotify-token",
		"",
		viper.GetString("WATCHTOWER_NOTIFICATION_GOTIFY_TOKEN"),
		"The Gotify Application required to query the Gotify API")
}

// SetDefaults provides default values for environment variables
func SetDefaults() {
	viper.AutomaticEnv()
	viper.SetDefault("DOCKER_HOST", "unix:///var/run/docker.sock")
	viper.SetDefault("DOCKER_API_VERSION", DockerAPIMinVersion)
	viper.SetDefault("WATCHTOWER_POLL_INTERVAL", 300)
	viper.SetDefault("WATCHTOWER_TIMEOUT", time.Second*10)
	viper.SetDefault("WATCHTOWER_NOTIFICATIONS", []string{})
	viper.SetDefault("WATCHTOWER_NOTIFICATIONS_LEVEL", "info")
	viper.SetDefault("WATCHTOWER_NOTIFICATION_EMAIL_SERVER_PORT", 25)
	viper.SetDefault("WATCHTOWER_NOTIFICATION_EMAIL_SUBJECTTAG", "")
	viper.SetDefault("WATCHTOWER_NOTIFICATION_SLACK_IDENTIFIER", "watchtower")
}

// EnvConfig translates the command-line options into environment variables
// that will initialize the api client
func EnvConfig(cmd *cobra.Command) error {
	var err error
	var host string
	var tls bool
	var version string

	flags := cmd.PersistentFlags()

	if host, err = flags.GetString("host"); err != nil {
		return err
	}
	if tls, err = flags.GetBool("tlsverify"); err != nil {
		return err
	}
	if version, err = flags.GetString("api-version"); err != nil {
		return err
	}
	if err = setEnvOptStr("DOCKER_HOST", host); err != nil {
		return err
	}
	if err = setEnvOptBool("DOCKER_TLS_VERIFY", tls); err != nil {
		return err
	}
	if err = setEnvOptStr("DOCKER_API_VERSION", version); err != nil {
		return err
	}
	return nil
}

// ReadFlags reads common flags used in the main program flow of watchtower
func ReadFlags(cmd *cobra.Command) (bool, bool, bool, time.Duration) {
	flags := cmd.PersistentFlags()

	var err error
	var cleanup bool
	var noRestart bool
	var monitorOnly bool
	var timeout time.Duration

	if cleanup, err = flags.GetBool("cleanup"); err != nil {
		log.Fatal(err)
	}
	if noRestart, err = flags.GetBool("no-restart"); err != nil {
		log.Fatal(err)
	}
	if monitorOnly, err = flags.GetBool("monitor-only"); err != nil {
		log.Fatal(err)
	}
	if timeout, err = flags.GetDuration("stop-timeout"); err != nil {
		log.Fatal(err)
	}

	return cleanup, noRestart, monitorOnly, timeout
}

func setEnvOptStr(env string, opt string) error {
	if opt == "" || opt == os.Getenv(env) {
		return nil
	}
	err := os.Setenv(env, opt)
	if err != nil {
		return err
	}
	return nil
}

func setEnvOptBool(env string, opt bool) error {
	if opt {
		return setEnvOptStr(env, "1")
	}
	return nil
}
<|endoftext|>


<commit_before>package util

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	log "github.com/sirupsen/logrus"
	"github.com/skeema/mybase"
	terminal "golang.org/x/term"
)

// AddGlobalOptions adds Skeema global options to the supplied mybase.Command.
// Typically cmd should be the top-level Command / Command Suite.
func AddGlobalOptions(cmd *mybase.Command) {
	// Options typically only found in .skeema files -- all hidden by default
	cmd.AddOption(mybase.StringOption("host", 0, "", "Database hostname or IP address").Hidden())
	cmd.AddOption(mybase.StringOption("port", 0, "3306", "Port to use for database host").Hidden())
	cmd.AddOption(mybase.StringOption("socket", 'S', "/tmp/mysql.sock", "Absolute path to Unix socket file used if host is localhost").Hidden())
	cmd.AddOption(mybase.StringOption("schema", 0, "", "Database schema name").Hidden())
	cmd.AddOption(mybase.StringOption("default-character-set", 0, "", "Schema-level default character set").Hidden())
	cmd.AddOption(mybase.StringOption("default-collation", 0, "", "Schema-level default collation").Hidden())
	cmd.AddOption(mybase.StringOption("flavor", 0, "", "Database server expressed in format vendor:major.minor, for use in vendor/version specific syntax").Hidden())
	cmd.AddOption(mybase.StringOption("generator", 0, "", "Version of Skeema used for `skeema init` or most recent `skeema pull`").Hidden())

	// Visible global options
	cmd.AddOptions("global",
		mybase.StringOption("user", 'u', "root", "Username to connect to database host"),
		mybase.StringOption("password", 'p', "", "Password for database user; omit value to prompt from TTY (default no password)").ValueOptional(),
		mybase.StringOption("host-wrapper", 'H', "", "External bin to shell out to for host lookup; see manual for template vars"),
		mybase.StringOption("connect-options", 'o', "", "Comma-separated session options to set upon connecting to each database instance"),
		mybase.StringOption("ignore-schema", 0, "", "Ignore schemas that match regex"),
		mybase.StringOption("ignore-table", 0, "", "Ignore tables that match regex"),
		mybase.StringOption("ssl-mode", 0, "", `Specify desired connection security SSL/TLS usage (valid values: "disabled", "preferred", "required")`),
		mybase.BoolOption("debug", 0, false, "Enable debug logging"),
		mybase.BoolOption("my-cnf", 0, true, "Parse ~/.my.cnf for configuration"),
	)
}

// AddGlobalConfigFiles takes the mybase.Config generated from the CLI and adds
// global option files as sources.
func AddGlobalConfigFiles(cfg *mybase.Config) {
	globalFilePaths := make([]string, 0, 4)

	// Avoid using "real" global paths in test logic. Otherwise, if the user
	// running the test happens to have a ~/.my.cnf, ~/.skeema, /etc/skeema, it
	// it would affect the test logic.
	if cfg.IsTest {
		globalFilePaths = append(globalFilePaths, "fake-etc/skeema", "fake-home/.my.cnf")
	} else {
		if runtime.GOOS == "windows" {
			globalFilePaths = append(globalFilePaths, "C:\\Program Files\\Skeema\\skeema.cnf")
		} else {
			globalFilePaths = append(globalFilePaths, "/etc/skeema", "/usr/local/etc/skeema")
		}
		if home, err := os.UserHomeDir(); home != "" && err == nil {
			globalFilePaths = append(globalFilePaths, filepath.Join(home, ".my.cnf"), filepath.Join(home, ".skeema"))
		}
	}

	for _, path := range globalFilePaths {
		f := mybase.NewFile(path)
		if !f.Exists() {
			continue
		}
		if err := f.Read(); err != nil {
			log.Warnf("Ignoring global option file %s due to read error: %s", f.Path(), err)
			continue
		}
		if strings.HasSuffix(path, ".my.cnf") {
			f.IgnoreUnknownOptions = true
			f.IgnoreOptions("host")
			if !cfg.GetBool("my-cnf") {
				continue
			}
		}
		if err := f.Parse(cfg); err != nil {
			log.Warnf("Ignoring global option file %s due to parse error: %s", f.Path(), err)
			continue
		}
		if strings.HasSuffix(path, ".my.cnf") {
			_ = f.UseSection("skeema", "client", "mysql") // safe to ignore error (doesn't matter if section doesn't exist)
		} else if cfg.CLI.Command.HasArg("environment") { // avoid panic on command without environment arg, such as help command!
			_ = f.UseSection(cfg.Get("environment")) // safe to ignore error (doesn't matter if section doesn't exist)
		}

		cfg.AddSource(f)
	}
}

// ProcessSpecialGlobalOptions performs special handling of global options with
// unusual semantics -- handling restricted placement of host and schema;
// obtaining a password from MYSQL_PWD or STDIN; enable debug logging.
func ProcessSpecialGlobalOptions(cfg *mybase.Config) error {
	// The host and schema options are special -- most commands only expect
	// to find them when recursively crawling directory configs. So if these
	// options have been set globally (via CLI or a global config file), and
	// the current subcommand hasn't explicitly overridden these options (as
	// init and add-environment do), return an error.
	cmdSuite := cfg.CLI.Command.Root()
	for _, name := range []string{"host", "schema"} {
		if cfg.Changed(name) && cfg.FindOption(name) == cmdSuite.Options()[name] {
			return fmt.Errorf("Option %s cannot be set via %s for this command", name, cfg.Source(name))
		}
	}

	// Special handling for password option: if not supplied at all, check env
	// var instead. Or if supplied but with no equals sign or value, prompt on
	// STDIN like mysql client does.
	if !cfg.Supplied("password") {
		if val := os.Getenv("MYSQL_PWD"); val != "" {
			cfg.CLI.OptionValues["password"] = val
			cfg.MarkDirty()
		}
	} else if !cfg.SuppliedWithValue("password") {
		var err error
		cfg.CLI.OptionValues["password"], err = PromptPassword()
		cfg.MarkDirty()
		if err != nil {
			return err
		}
	}

	if cfg.GetBool("debug") {
		log.SetLevel(log.DebugLevel)
	}

	return nil
}

// PromptPassword reads a password from STDIN without echoing the typed
// characters. Requires that STDIN is a TTY. Optionally supply args to build
// a custom prompt string; first arg must be a string if so, with args behaving
// like those to fmt.Printf().
func PromptPassword(promptArgs ...interface{}) (string, error) {
	if len(promptArgs) == 0 {
		promptArgs = append(promptArgs, "Enter password: ")
	}
	stdin := int(os.Stdin.Fd())
	if !terminal.IsTerminal(stdin) {
		return "", errors.New("STDIN must be a TTY to read password")
	}
	fmt.Printf(promptArgs[0].(string), promptArgs[1:]...)
	bytePassword, err := terminal.ReadPassword(stdin)
	if err != nil {
		return "", err
	}
	fmt.Println() // since ReadPassword also won't echo the ENTER key as a newline!
	return string(bytePassword), nil
}

// SplitConnectOptions takes a string containing a comma-separated list of
// connection options (typically obtained from the "connect-options" option)
// and splits them into a map of individual key: value strings. This function
// understands single-quoted values may contain commas, and will properly
// treat them not as delimiters. Single-quoted values may also include escaped
// single quotes, and values in general may contain escaped commas; these are
// all also treated properly.
func SplitConnectOptions(connectOpts string) (map[string]string, error) {
	if len(connectOpts) == 0 {
		return map[string]string{}, nil
	}
	if connectOpts[len(connectOpts)-1] == '\\' {
		return nil, fmt.Errorf("Trailing backslash in connect-options \"%s\"", connectOpts)
	}
	return parseConnectOptions(connectOpts)
}

func parseConnectOptions(input string) (map[string]string, error) {
	result := make(map[string]string)
	var startToken int
	var name string
	var inQuote, escapeNext bool

	// Add a trailing comma to simplify handling of end-of-string
	for n, c := range input + "," {
		if escapeNext {
			escapeNext = false
			continue
		}
		switch c {
		case '\'':
			if name == "" {
				return result, fmt.Errorf("Invalid quote character in option name at byte offset %d in connect-options \"%s\"", n, input)
			}
			inQuote = !inQuote
		case '\\':
			escapeNext = true
		case '=':
			if inQuote {
				continue
			}
			if name == "" {
				name = input[startToken:n]
				startToken = n + 1
			} else {
				return result, fmt.Errorf("Invalid equals-sign character in option value at byte offset %d in connect-options \"%s\"", n, input)
			}
		case ',':
			if inQuote {
				continue
			}
			if startToken == n { // comma directly after equals sign, comma, or start of string
				return result, fmt.Errorf("Invalid comma placement in option value at byte offset %d in connect-options \"%s\"", n, input)
			}
			if name == "" {
				return result, fmt.Errorf("Option %s is missing a value at byte offset %d in connect-options \"%s\"", input[startToken:n], n, input)
			}
			if _, already := result[name]; already {
				// Disallow this since it's inherently ordering-dependent, and would
				// further complicate RealConnectOptions logic
				return result, fmt.Errorf("Option %s is set multiple times in connect-options \"%s\"", name, input)
			}
			result[name] = input[startToken:n]
			name = ""
			startToken = n + 1
		}
	}

	var err error
	if inQuote {
		err = fmt.Errorf("Unterminated quote in connect-options \"%s\"", input)
	}
	return result, err
}

// RealConnectOptions takes a comma-separated string of connection options,
// strips any Go driver-specific ones, and then returns the new string which
// is now suitable for passing to an external tool.
func RealConnectOptions(connectOpts string) (string, error) {
	// list of lowercased versions of all go-sql-driver/mysql special params
	ignored := map[string]bool{
		"allowallfiles":           true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
		"allowcleartextpasswords": true,
		"allownativepasswords":    true,
		"allowoldpasswords":       true,
		"charset":                 true,
		"checkconnliveness":       true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
		"clientfoundrows":         true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
		"collation":               true,
		"columnswithalias":        true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
		"interpolateparams":       true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
		"loc":                     true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
		"maxallowedpacket":        true,
		"multistatements":         true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
		"parsetime":               true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
		"readtimeout":             true,
		"rejectreadonly":          true,
		"serverpubkey":            true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
		"timeout":                 true,
		"tls":                     true,
		"writetimeout":            true,
	}

	options, err := SplitConnectOptions(connectOpts)
	if err != nil {
		return "", err
	}

	// Iterate through the returned map, and remove any driver-specific options.
	// This is done via regular expressions substitution in order to keep the
	// string in its original order.
	for name, value := range options {
		if ignored[strings.ToLower(name)] {
			re, err := regexp.Compile(fmt.Sprintf(`%s=%s(,|$)`, regexp.QuoteMeta(name), regexp.QuoteMeta(value)))
			if err != nil {
				return "", err
			}
			connectOpts = re.ReplaceAllString(connectOpts, "")
		}
	}
	if len(connectOpts) > 0 && connectOpts[len(connectOpts)-1] == ',' {
		connectOpts = connectOpts[0 : len(connectOpts)-1]
	}
	return connectOpts, nil
}
<commit_msg>password prompt: prefer STDERR for the interactive prompt text<commit_after>package util

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	log "github.com/sirupsen/logrus"
	"github.com/skeema/mybase"
	terminal "golang.org/x/term"
)

// AddGlobalOptions adds Skeema global options to the supplied mybase.Command.
// Typically cmd should be the top-level Command / Command Suite.
func AddGlobalOptions(cmd *mybase.Command) {
	// Options typically only found in .skeema files -- all hidden by default
	cmd.AddOption(mybase.StringOption("host", 0, "", "Database hostname or IP address").Hidden())
	cmd.AddOption(mybase.StringOption("port", 0, "3306", "Port to use for database host").Hidden())
	cmd.AddOption(mybase.StringOption("socket", 'S', "/tmp/mysql.sock", "Absolute path to Unix socket file used if host is localhost").Hidden())
	cmd.AddOption(mybase.StringOption("schema", 0, "", "Database schema name").Hidden())
	cmd.AddOption(mybase.StringOption("default-character-set", 0, "", "Schema-level default character set").Hidden())
	cmd.AddOption(mybase.StringOption("default-collation", 0, "", "Schema-level default collation").Hidden())
	cmd.AddOption(mybase.StringOption("flavor", 0, "", "Database server expressed in format vendor:major.minor, for use in vendor/version specific syntax").Hidden())
	cmd.AddOption(mybase.StringOption("generator", 0, "", "Version of Skeema used for `skeema init` or most recent `skeema pull`").Hidden())

	// Visible global options
	cmd.AddOptions("global",
		mybase.StringOption("user", 'u', "root", "Username to connect to database host"),
		mybase.StringOption("password", 'p', "", "Password for database user; omit value to prompt from TTY (default no password)").ValueOptional(),
		mybase.StringOption("host-wrapper", 'H', "", "External bin to shell out to for host lookup; see manual for template vars"),
		mybase.StringOption("connect-options", 'o', "", "Comma-separated session options to set upon connecting to each database instance"),
		mybase.StringOption("ignore-schema", 0, "", "Ignore schemas that match regex"),
		mybase.StringOption("ignore-table", 0, "", "Ignore tables that match regex"),
		mybase.StringOption("ssl-mode", 0, "", `Specify desired connection security SSL/TLS usage (valid values: "disabled", "preferred", "required")`),
		mybase.BoolOption("debug", 0, false, "Enable debug logging"),
		mybase.BoolOption("my-cnf", 0, true, "Parse ~/.my.cnf for configuration"),
	)
}

// AddGlobalConfigFiles takes the mybase.Config generated from the CLI and adds
// global option files as sources.
func AddGlobalConfigFiles(cfg *mybase.Config) {
	globalFilePaths := make([]string, 0, 4)

	// Avoid using "real" global paths in test logic. Otherwise, if the user
	// running the test happens to have a ~/.my.cnf, ~/.skeema, /etc/skeema, it
	// it would affect the test logic.
	if cfg.IsTest {
		globalFilePaths = append(globalFilePaths, "fake-etc/skeema", "fake-home/.my.cnf")
	} else {
		if runtime.GOOS == "windows" {
			globalFilePaths = append(globalFilePaths, "C:\\Program Files\\Skeema\\skeema.cnf")
		} else {
			globalFilePaths = append(globalFilePaths, "/etc/skeema", "/usr/local/etc/skeema")
		}
		if home, err := os.UserHomeDir(); home != "" && err == nil {
			globalFilePaths = append(globalFilePaths, filepath.Join(home, ".my.cnf"), filepath.Join(home, ".skeema"))
		}
	}

	for _, path := range globalFilePaths {
		f := mybase.NewFile(path)
		if !f.Exists() {
			continue
		}
		if err := f.Read(); err != nil {
			log.Warnf("Ignoring global option file %s due to read error: %s", f.Path(), err)
			continue
		}
		if strings.HasSuffix(path, ".my.cnf") {
			f.IgnoreUnknownOptions = true
			f.IgnoreOptions("host")
			if !cfg.GetBool("my-cnf") {
				continue
			}
		}
		if err := f.Parse(cfg); err != nil {
			log.Warnf("Ignoring global option file %s due to parse error: %s", f.Path(), err)
			continue
		}
		if strings.HasSuffix(path, ".my.cnf") {
			_ = f.UseSection("skeema", "client", "mysql") // safe to ignore error (doesn't matter if section doesn't exist)
		} else if cfg.CLI.Command.HasArg("environment") { // avoid panic on command without environment arg, such as help command!
			_ = f.UseSection(cfg.Get("environment")) // safe to ignore error (doesn't matter if section doesn't exist)
		}

		cfg.AddSource(f)
	}
}

// ProcessSpecialGlobalOptions performs special handling of global options with
// unusual semantics -- handling restricted placement of host and schema;
// obtaining a password from MYSQL_PWD or STDIN; enable debug logging.
func ProcessSpecialGlobalOptions(cfg *mybase.Config) error {
	// The host and schema options are special -- most commands only expect
	// to find them when recursively crawling directory configs. So if these
	// options have been set globally (via CLI or a global config file), and
	// the current subcommand hasn't explicitly overridden these options (as
	// init and add-environment do), return an error.
	cmdSuite := cfg.CLI.Command.Root()
	for _, name := range []string{"host", "schema"} {
		if cfg.Changed(name) && cfg.FindOption(name) == cmdSuite.Options()[name] {
			return fmt.Errorf("Option %s cannot be set via %s for this command", name, cfg.Source(name))
		}
	}

	// Special handling for password option: if not supplied at all, check env
	// var instead. Or if supplied but with no equals sign or value, prompt on
	// STDIN like mysql client does.
	if !cfg.Supplied("password") {
		if val := os.Getenv("MYSQL_PWD"); val != "" {
			cfg.CLI.OptionValues["password"] = val
			cfg.MarkDirty()
		}
	} else if !cfg.SuppliedWithValue("password") {
		var err error
		cfg.CLI.OptionValues["password"], err = PromptPassword()
		cfg.MarkDirty()
		if err != nil {
			return err
		}
	}

	if cfg.GetBool("debug") {
		log.SetLevel(log.DebugLevel)
	}

	return nil
}

// PromptPassword reads a password from STDIN without echoing the typed
// characters. Requires that STDIN is a TTY. Optionally supply args to build
// a custom prompt string; first arg must be a string if so, with args behaving
// like those to fmt.Printf(). The prompt will be written to STDERR, unless
// STDERR is a non-terminal and STDOUT is a terminal, in which case STDOUT is
// used.
func PromptPassword(promptArgs ...interface{}) (string, error) {
	if len(promptArgs) == 0 {
		promptArgs = append(promptArgs, "Enter password: ")
	}
	stdin := int(os.Stdin.Fd())
	if !terminal.IsTerminal(stdin) {
		return "", errors.New("STDIN must be a TTY to read password")
	}

	w := os.Stderr
	if !terminal.IsTerminal(int(w.Fd())) && terminal.IsTerminal(int(os.Stdout.Fd())) {
		w = os.Stdout
	}
	fmt.Fprintf(w, promptArgs[0].(string), promptArgs[1:]...)

	bytePassword, err := terminal.ReadPassword(stdin)
	if err != nil {
		return "", err
	}

	fmt.Fprintln(w) // since ReadPassword also won't echo the ENTER key as a newline!
	return string(bytePassword), nil
}

// SplitConnectOptions takes a string containing a comma-separated list of
// connection options (typically obtained from the "connect-options" option)
// and splits them into a map of individual key: value strings. This function
// understands single-quoted values may contain commas, and will properly
// treat them not as delimiters. Single-quoted values may also include escaped
// single quotes, and values in general may contain escaped commas; these are
// all also treated properly.
func SplitConnectOptions(connectOpts string) (map[string]string, error) {
	if len(connectOpts) == 0 {
		return map[string]string{}, nil
	}
	if connectOpts[len(connectOpts)-1] == '\\' {
		return nil, fmt.Errorf("Trailing backslash in connect-options \"%s\"", connectOpts)
	}
	return parseConnectOptions(connectOpts)
}

func parseConnectOptions(input string) (map[string]string, error) {
	result := make(map[string]string)
	var startToken int
	var name string
	var inQuote, escapeNext bool

	// Add a trailing comma to simplify handling of end-of-string
	for n, c := range input + "," {
		if escapeNext {
			escapeNext = false
			continue
		}
		switch c {
		case '\'':
			if name == "" {
				return result, fmt.Errorf("Invalid quote character in option name at byte offset %d in connect-options \"%s\"", n, input)
			}
			inQuote = !inQuote
		case '\\':
			escapeNext = true
		case '=':
			if inQuote {
				continue
			}
			if name == "" {
				name = input[startToken:n]
				startToken = n + 1
			} else {
				return result, fmt.Errorf("Invalid equals-sign character in option value at byte offset %d in connect-options \"%s\"", n, input)
			}
		case ',':
			if inQuote {
				continue
			}
			if startToken == n { // comma directly after equals sign, comma, or start of string
				return result, fmt.Errorf("Invalid comma placement in option value at byte offset %d in connect-options \"%s\"", n, input)
			}
			if name == "" {
				return result, fmt.Errorf("Option %s is missing a value at byte offset %d in connect-options \"%s\"", input[startToken:n], n, input)
			}
			if _, already := result[name]; already {
				// Disallow this since it's inherently ordering-dependent, and would
				// further complicate RealConnectOptions logic
				return result, fmt.Errorf("Option %s is set multiple times in connect-options \"%s\"", name, input)
			}
			result[name] = input[startToken:n]
			name = ""
			startToken = n + 1
		}
	}

	var err error
	if inQuote {
		err = fmt.Errorf("Unterminated quote in connect-options \"%s\"", input)
	}
	return result, err
}

// RealConnectOptions takes a comma-separated string of connection options,
// strips any Go driver-specific ones, and then returns the new string which
// is now suitable for passing to an external tool.
func RealConnectOptions(connectOpts string) (string, error) {
	// list of lowercased versions of all go-sql-driver/mysql special params
	ignored := map[string]bool{
		"allowallfiles":           true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
		"allowcleartextpasswords": true,
		"allownativepasswords":    true,
		"allowoldpasswords":       true,
		"charset":                 true,
		"checkconnliveness":       true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
		"clientfoundrows":         true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
		"collation":               true,
		"columnswithalias":        true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
		"interpolateparams":       true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
		"loc":                     true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
		"maxallowedpacket":        true,
		"multistatements":         true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
		"parsetime":               true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
		"readtimeout":             true,
		"rejectreadonly":          true,
		"serverpubkey":            true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
		"timeout":                 true,
		"tls":                     true,
		"writetimeout":            true,
	}

	options, err := SplitConnectOptions(connectOpts)
	if err != nil {
		return "", err
	}

	// Iterate through the returned map, and remove any driver-specific options.
	// This is done via regular expressions substitution in order to keep the
	// string in its original order.
	for name, value := range options {
		if ignored[strings.ToLower(name)] {
			re, err := regexp.Compile(fmt.Sprintf(`%s=%s(,|$)`, regexp.QuoteMeta(name), regexp.QuoteMeta(value)))
			if err != nil {
				return "", err
			}
			connectOpts = re.ReplaceAllString(connectOpts, "")
		}
	}
	if len(connectOpts) > 0 && connectOpts[len(connectOpts)-1] == ',' {
		connectOpts = connectOpts[0 : len(connectOpts)-1]
	}
	return connectOpts, nil
}
<|endoftext|>


<commit_before>package jocko

import (
	"fmt"
	"sync"
)

type replicaLookup struct {
	lock sync.RWMutex
	// topic to partition id to replica id to replica
	replica map[string]map[int32]*Replica
}

func NewReplicaLookup() *replicaLookup {
	return &replicaLookup{
		replica: make(map[string]map[int32]*Replica),
	}
}

func (rl *replicaLookup) AddReplica(replica *Replica) {
	rl.lock.Lock()
	defer rl.lock.Unlock()
ADD:
	if t, ok := rl.replica[replica.Partition.Topic]; ok {
		if _, ok := t[replica.Partition.ID]; ok {
			// replica exists already -- leave it?
			return
		}
		t[replica.Partition.ID] = replica
	} else {
		rl.replica[replica.Partition.Topic] = make(map[int32]*Replica)
		goto ADD
	}
}

func (rl *replicaLookup) Replica(topic string, partition int32) (*Replica, error) {
	rl.lock.RLock()
	defer rl.lock.RUnlock()
	r, ok := rl.replica[topic][partition]
	if !ok {
		return nil, fmt.Errorf("no replica for topic %s partition %d", topic, partition)
	}
	return r, nil
}

func (rl *replicaLookup) RemoveReplica(replica *Replica) {
	rl.lock.Lock()
	defer rl.lock.Unlock()
	delete(rl.replica[replica.Partition.Topic], replica.Partition.ID)
}
<commit_msg>allow upserting replicas in the lookup<commit_after>package jocko

import (
	"fmt"
	"sync"
)

type replicaLookup struct {
	lock sync.RWMutex
	// topic to partition id to replica id to replica
	replica map[string]map[int32]*Replica
}

func NewReplicaLookup() *replicaLookup {
	return &replicaLookup{
		replica: make(map[string]map[int32]*Replica),
	}
}

func (rl *replicaLookup) AddReplica(replica *Replica) {
	rl.lock.Lock()
	defer rl.lock.Unlock()
ADD:
	if t, ok := rl.replica[replica.Partition.Topic]; ok {
		t[replica.Partition.ID] = replica
	} else {
		rl.replica[replica.Partition.Topic] = make(map[int32]*Replica)
		goto ADD
	}
}

func (rl *replicaLookup) Replica(topic string, partition int32) (*Replica, error) {
	rl.lock.RLock()
	defer rl.lock.RUnlock()
	r, ok := rl.replica[topic][partition]
	if !ok {
		return nil, fmt.Errorf("no replica for topic %s partition %d", topic, partition)
	}
	return r, nil
}

func (rl *replicaLookup) RemoveReplica(replica *Replica) {
	rl.lock.Lock()
	defer rl.lock.Unlock()
	delete(rl.replica[replica.Partition.Topic], replica.Partition.ID)
}
<|endoftext|>


<commit_before>package admin

import (
	"encoding/json"
	"fmt"
	"net/http"
	"path"
	"strings"

	"github.com/qor/qor/responder"
	"github.com/qor/qor/roles"
)

type controller struct {
	*Admin
}

func renderError(context *Context, err error) {
	responder.With("html", func() {
		context.Writer.WriteHeader(http.StatusNotAcceptable)
		if _, er := context.Writer.Write([]byte(err.Error())); er != nil {
			println("failed to write response", er.Error())
		}
	}).With("json", func() {
		data, er := json.Marshal(map[string]string{"error": err.Error()})
		if er != nil {
			println("failed to marshal error json")
		}
		context.Writer.WriteHeader(http.StatusNotAcceptable)
		if _, er := context.Writer.Write(data); er != nil {
			println("failed to write reponse", er.Error())
		}
	}).Respond(context.Writer, context.Request)
}

func (context *Context) checkResourcePermission(permission roles.PermissionMode) bool {
	if context.Resource == nil || context.Resource.HasPermission(permission, context.Context) {
		return true
	}
	context.Writer.Write([]byte("Permission denied"))
	return false
}

func (ac *controller) Dashboard(context *Context) {
	context.Execute("dashboard", nil)
}

func (ac *controller) Index(context *Context) {
	if context.checkResourcePermission(roles.Read) {
		if result, err := context.FindMany(); err == nil {
			responder.With("html", func() {
				context.Execute("index", result)
			}).With("json", func() {
				res := context.Resource
				js, _ := json.Marshal(res.convertObjectToMap(context, result, "index"))
				context.Writer.Write(js)
			}).Respond(context.Writer, context.Request)
		} else {
			http.NotFound(context.Writer, context.Request)
		}
	}
}

func (ac *controller) Show(context *Context) {
	if context.checkResourcePermission(roles.Read) {
		result, _ := context.FindOne()

		responder.With("html", func() {
			context.Execute("show", result)
		}).With("json", func() {
			res := context.Resource
			js, _ := json.Marshal(res.convertObjectToMap(context, result, "show"))
			context.Writer.Write(js)
		}).Respond(context.Writer, context.Request)
	}
}

func (ac *controller) New(context *Context) {
	if context.checkResourcePermission(roles.Create) {
		context.Execute("new", nil)
	}
}

func (ac *controller) Create(context *Context) {
	if context.checkResourcePermission(roles.Create) {
		res := context.Resource

		result := res.NewStruct()
		if errs := res.Decode(context.Context, result); len(errs) == 0 {
			res.CallSaver(result, context.Context)
			responder.With("html", func() {
				context.Flash(context.dt("resource_successfully_created", "{{.Name}} was successfully created", res), "success")
				primaryKey := fmt.Sprintf("%v", context.GetDB().NewScope(result).PrimaryKeyValue())
				http.Redirect(context.Writer, context.Request, path.Join(context.Request.URL.Path, primaryKey), http.StatusFound)
			}).With("json", func() {
				res := context.Resource
				js, _ := json.Marshal(res.convertObjectToMap(context, result, "show"))
				context.Writer.Write(js)
			}).Respond(context.Writer, context.Request)
		}
	}
}

func (ac *controller) Update(context *Context) {
	if context.checkResourcePermission(roles.Update) {
		res := context.Resource
		if result, err := context.FindOne(); err == nil {
			if errs := res.Decode(context.Context, result); len(errs) == 0 {
				if err := res.CallSaver(result, context.Context); err != nil {
					renderError(context, err)
					return
				}
				responder.With("html", func() {
					context.FlashNow(context.dt("resource_successfully_updated", "{{.Name}} was successfully updated", res), "success")
					context.Execute("show", result)
				}).With("json", func() {
					res := context.Resource
					js, _ := json.Marshal(res.convertObjectToMap(context, result, "show"))
					context.Writer.Write(js)
				}).Respond(context.Writer, context.Request)
			}
		} else {
			renderError(context, err)
		}
	}
}

func (ac *controller) Delete(context *Context) {
	if context.checkResourcePermission(roles.Delete) {
		res := context.Resource

		responder.With("html", func() {
			if res.CallDeleter(res.NewStruct(), context.Context) == nil {
				http.Redirect(context.Writer, context.Request, path.Join(ac.GetRouter().Prefix, res.ToParam()), http.StatusFound)
			} else {
				http.Redirect(context.Writer, context.Request, path.Join(ac.GetRouter().Prefix, res.ToParam()), http.StatusNotFound)
			}
		}).With("json", func() {
			if res.CallDeleter(res.NewStruct(), context.Context) == nil {
				context.Writer.WriteHeader(http.StatusOK)
			} else {
				context.Writer.WriteHeader(http.StatusNotFound)
			}
		}).Respond(context.Writer, context.Request)
	}
}

func (ac *controller) Action(context *Context) {
	var err error
	name := strings.Split(context.Request.URL.Path, "/")[4]

	for _, action := range context.Resource.actions {
		if action.Name == name {
			ids := context.Request.Form.Get("ids")
			scope := context.GetDB().Where(fmt.Sprintf("%v IN (?)", context.Resource.PrimaryField().DBName), ids)
			err = action.Handle(scope, context.Context)
		}
	}

	responder.With("html", func() {
		http.Redirect(context.Writer, context.Request, context.Request.Referer(), http.StatusFound)
	}).With("json", func() {
		if err == nil {
			context.Writer.Write([]byte("OK"))
		} else {
			context.Writer.Write([]byte(err.Error()))
		}
	}).Respond(context.Writer, context.Request)
}

func (ac *controller) Asset(context *Context) {
	file := strings.TrimPrefix(context.Request.URL.Path, ac.GetRouter().Prefix)
	if filename, err := context.findFile(file); err == nil {
		http.ServeFile(context.Writer, context.Request, filename)
	} else {
		http.NotFound(context.Writer, context.Request)
	}
}
<commit_msg>handle admin create error<commit_after>package admin

import (
	"encoding/json"
	"fmt"
	"net/http"
	"path"
	"strings"

	"github.com/qor/qor/responder"
	"github.com/qor/qor/roles"
)

type controller struct {
	*Admin
}

func renderError(context *Context, err error) {
	responder.With("html", func() {
		context.Writer.WriteHeader(http.StatusNotAcceptable)
		if _, er := context.Writer.Write([]byte(err.Error())); er != nil {
			println("failed to write response", er.Error())
		}
	}).With("json", func() {
		data, er := json.Marshal(map[string]string{"error": err.Error()})
		if er != nil {
			println("failed to marshal error json")
		}
		context.Writer.WriteHeader(http.StatusNotAcceptable)
		if _, er := context.Writer.Write(data); er != nil {
			println("failed to write reponse", er.Error())
		}
	}).Respond(context.Writer, context.Request)
}

func (context *Context) checkResourcePermission(permission roles.PermissionMode) bool {
	if context.Resource == nil || context.Resource.HasPermission(permission, context.Context) {
		return true
	}
	context.Writer.Write([]byte("Permission denied"))
	return false
}

func (ac *controller) Dashboard(context *Context) {
	context.Execute("dashboard", nil)
}

func (ac *controller) Index(context *Context) {
	if context.checkResourcePermission(roles.Read) {
		if result, err := context.FindMany(); err == nil {
			responder.With("html", func() {
				context.Execute("index", result)
			}).With("json", func() {
				res := context.Resource
				js, _ := json.Marshal(res.convertObjectToMap(context, result, "index"))
				context.Writer.Write(js)
			}).Respond(context.Writer, context.Request)
		} else {
			http.NotFound(context.Writer, context.Request)
		}
	}
}

func (ac *controller) Show(context *Context) {
	if context.checkResourcePermission(roles.Read) {
		result, _ := context.FindOne()

		responder.With("html", func() {
			context.Execute("show", result)
		}).With("json", func() {
			res := context.Resource
			js, _ := json.Marshal(res.convertObjectToMap(context, result, "show"))
			context.Writer.Write(js)
		}).Respond(context.Writer, context.Request)
	}
}

func (ac *controller) New(context *Context) {
	if context.checkResourcePermission(roles.Create) {
		context.Execute("new", nil)
	}
}

func (ac *controller) Create(context *Context) {
	if context.checkResourcePermission(roles.Create) {
		res := context.Resource

		result := res.NewStruct()
		if errs := res.Decode(context.Context, result); len(errs) == 0 {
			if err := res.CallSaver(result, context.Context); err != nil {
				renderError(context, err)
				return
			}
			responder.With("html", func() {
				context.Flash(context.dt("resource_successfully_created", "{{.Name}} was successfully created", res), "success")
				primaryKey := fmt.Sprintf("%v", context.GetDB().NewScope(result).PrimaryKeyValue())
				http.Redirect(context.Writer, context.Request, path.Join(context.Request.URL.Path, primaryKey), http.StatusFound)
			}).With("json", func() {
				res := context.Resource
				js, _ := json.Marshal(res.convertObjectToMap(context, result, "show"))
				context.Writer.Write(js)
			}).Respond(context.Writer, context.Request)
		}
	}
}

func (ac *controller) Update(context *Context) {
	if context.checkResourcePermission(roles.Update) {
		res := context.Resource
		if result, err := context.FindOne(); err == nil {
			if errs := res.Decode(context.Context, result); len(errs) == 0 {
				if err := res.CallSaver(result, context.Context); err != nil {
					renderError(context, err)
					return
				}
				responder.With("html", func() {
					context.FlashNow(context.dt("resource_successfully_updated", "{{.Name}} was successfully updated", res), "success")
					context.Execute("show", result)
				}).With("json", func() {
					res := context.Resource
					js, _ := json.Marshal(res.convertObjectToMap(context, result, "show"))
					context.Writer.Write(js)
				}).Respond(context.Writer, context.Request)
			}
		} else {
			renderError(context, err)
		}
	}
}

func (ac *controller) Delete(context *Context) {
	if context.checkResourcePermission(roles.Delete) {
		res := context.Resource

		responder.With("html", func() {
			if res.CallDeleter(res.NewStruct(), context.Context) == nil {
				http.Redirect(context.Writer, context.Request, path.Join(ac.GetRouter().Prefix, res.ToParam()), http.StatusFound)
			} else {
				http.Redirect(context.Writer, context.Request, path.Join(ac.GetRouter().Prefix, res.ToParam()), http.StatusNotFound)
			}
		}).With("json", func() {
			if res.CallDeleter(res.NewStruct(), context.Context) == nil {
				context.Writer.WriteHeader(http.StatusOK)
			} else {
				context.Writer.WriteHeader(http.StatusNotFound)
			}
		}).Respond(context.Writer, context.Request)
	}
}

func (ac *controller) Action(context *Context) {
	var err error
	name := strings.Split(context.Request.URL.Path, "/")[4]

	for _, action := range context.Resource.actions {
		if action.Name == name {
			ids := context.Request.Form.Get("ids")
			scope := context.GetDB().Where(fmt.Sprintf("%v IN (?)", context.Resource.PrimaryField().DBName), ids)
			err = action.Handle(scope, context.Context)
		}
	}

	responder.With("html", func() {
		http.Redirect(context.Writer, context.Request, context.Request.Referer(), http.StatusFound)
	}).With("json", func() {
		if err == nil {
			context.Writer.Write([]byte("OK"))
		} else {
			context.Writer.Write([]byte(err.Error()))
		}
	}).Respond(context.Writer, context.Request)
}

func (ac *controller) Asset(context *Context) {
	file := strings.TrimPrefix(context.Request.URL.Path, ac.GetRouter().Prefix)
	if filename, err := context.findFile(file); err == nil {
		http.ServeFile(context.Writer, context.Request, filename)
	} else {
		http.NotFound(context.Writer, context.Request)
	}
}
<|endoftext|>


<commit_before>package api_test

import (
	"testing"
	"net/http"
	"net/http/httptest"
)

func TestListPackage(t *testing.T) {
	request, _ := http.NewRequest("GET", "test/package", nil)
	response := httptest.NewRecorder()

	listPackage(response, request)
	if response.Code != http.StatusOK {
		t.Fatalf("Response code expected 200 got %v:\n", response.Code)
	}
}

func TestUploadPackage(t *testing.T) {
	request, _ := http.NewRequest("POST", "test/package", nil)
	response := httptest.NewRecorder()

	uploadPackage(response, request)
	if response.Code != http.StatusOK {
		t.Fatalf("Response code expected 200 got %v:\n", response.Code)
	}
}

func TestDownloadPackage(t *testing.T) {
	request, _ := http.NewRequest("GET", "test/package/pkg.tgz", nil)
	response := httptest.NewRecorder()

	downloadPackage(response, request)
	if response.Code != http.StatusOK {
		t.Fatalf("Response code expected 200 got %v:\n", response.Code)
	}
}
<commit_msg>Renamed func names<commit_after>package api

import (
	"testing"
	"net/http"
	"net/http/httptest"
)

func TestPackageList(t *testing.T) {
	request, _ := http.NewRequest("GET", "test/package", nil)
	response := httptest.NewRecorder()

	packageList(response, request)
	if response.Code != http.StatusOK {
		t.Fatalf("Response code expected %v got %v:\n",
		http.StatusOK, response.Code)
	}
}

func TestPackageUpload(t *testing.T) {
	request, _ := http.NewRequest("POST", "test/package", nil)
	response := httptest.NewRecorder()

	packageUpload(response, request)
	if response.Code != http.StatusCreated {
		t.Fatalf("Response code expected %v got %v:\n",
		http.StatusCreated, response.Code)
	}
}

func TestPackageDownload(t *testing.T) {
	request, _ := http.NewRequest("GET", "test/package/pkg.tgz", nil)
	response := httptest.NewRecorder()

	packageDownload(response, request)
	if response.Code != http.StatusOK {
		t.Fatalf("Response code expected %v got %v:\n",
		http.StatusOK, response.Code)
	}
}
<|endoftext|>


<commit_before>package commands

import (
	"bufio"
	"compress/gzip"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"regexp"

	"github.com/9seconds/ah/app/environments"
	"github.com/9seconds/ah/app/historyentries"
	"github.com/9seconds/ah/app/utils"
)

// Tee implements t (trace, tee) command.
func Tee(input string, interactive bool, pseudoTTY bool, env *environments.Environment) {
	output, err := ioutil.TempFile(os.TempDir(), "ah")
	if err != nil {
		utils.Logger.Panic("Cannot create temporary file")
	}

	bufferedOutput := bufio.NewWriter(output)
	gzippedWrapper := utils.NewSynchronizedWriter(gzip.NewWriter(bufferedOutput))
	combinedStdout := io.MultiWriter(os.Stdout, gzippedWrapper)
	combinedStderr := io.MultiWriter(os.Stderr, gzippedWrapper)

	commandError := utils.Exec(input,
		env.GetShell(), interactive, pseudoTTY,
		os.Stdin, combinedStdout, combinedStderr)

	defer func() {
		// defer here because command may cause a panic but we do not want to lose any output
		gzippedWrapper.Close()
		bufferedOutput.Flush()
		output.Close()

		if hash, err := getPreciseHash(input, env); err == nil {
			err = os.Rename(output.Name(), env.GetTraceFileName(hash))
			if err != nil {
				utils.Logger.Errorf("Cannot save trace: %v. Get it here: %s", err, output.Name())
			}
		} else {
			utils.Logger.Errorf("Error occured on fetching command number: %v", err)
		}
	}()

	if commandError != nil {
		os.Exit(utils.GetStatusCode(commandError))
	}
}

func getPreciseHash(cmd string, env *environments.Environment) (hash string, err error) {
	filter := utils.CreateRegexp(regexp.QuoteMeta(cmd))
	commands, err := historyentries.GetCommands(historyentries.GetCommandsAll, filter, env)
	if err != nil {
		err = fmt.Errorf("Cannot fetch commands list: %v", err)
		return
	}
	commandList := commands.Result().([]historyentries.HistoryEntry)

	if len(commandList) == 0 {
		err = errors.New("Command list is empty")
		return
	}

	found := len(commandList) - 1
	for idx := len(commandList) - 2; idx >= 0; idx-- {
		if commandList[idx].GetTimestamp() < environments.CreatedAt {
			break
		}
		found = idx
	}
	hash = commandList[found].GetTraceName()

	return
}
<commit_msg>Fixes for tracing output with exit code != 0<commit_after>package commands

import (
	"bufio"
	"compress/gzip"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"regexp"

	"github.com/9seconds/ah/app/environments"
	"github.com/9seconds/ah/app/historyentries"
	"github.com/9seconds/ah/app/utils"
)

// Tee implements t (trace, tee) command.
func Tee(input string, interactive bool, pseudoTTY bool, env *environments.Environment) {
	output, err := ioutil.TempFile(os.TempDir(), "ah")
	if err != nil {
		utils.Logger.Panic("Cannot create temporary file")
	}

	bufferedOutput := bufio.NewWriter(output)
	gzippedWrapper := utils.NewSynchronizedWriter(gzip.NewWriter(bufferedOutput))
	combinedStdout := io.MultiWriter(os.Stdout, gzippedWrapper)
	combinedStderr := io.MultiWriter(os.Stderr, gzippedWrapper)

	var commandError *exec.ExitError
	defer func() {
		// defer here because command may cause a panic but we do not want to lose any output
		gzippedWrapper.Close()
		bufferedOutput.Flush()
		output.Close()

		if hash, err := getPreciseHash(input, env); err == nil {
			err = os.Rename(output.Name(), env.GetTraceFileName(hash))
			if err != nil {
				utils.Logger.Errorf("Cannot save trace: %v. Get it here: %s", err, output.Name())
			}
		} else {
			utils.Logger.Errorf("Error occured on fetching command number: %v", err)
		}

		if commandError != nil {
			os.Exit(utils.GetStatusCode(commandError))
		}
	}()

	commandError = utils.Exec(input,
		env.GetShell(), interactive, pseudoTTY,
		os.Stdin, combinedStdout, combinedStderr)
}

func getPreciseHash(cmd string, env *environments.Environment) (hash string, err error) {
	filter := utils.CreateRegexp(regexp.QuoteMeta(cmd))
	commands, err := historyentries.GetCommands(historyentries.GetCommandsAll, filter, env)
	if err != nil {
		err = fmt.Errorf("Cannot fetch commands list: %v", err)
		return
	}
	commandList := commands.Result().([]historyentries.HistoryEntry)

	if len(commandList) == 0 {
		err = errors.New("Command list is empty")
		return
	}

	found := len(commandList) - 1
	for idx := len(commandList) - 2; idx >= 0; idx-- {
		if commandList[idx].GetTimestamp() < environments.CreatedAt {
			break
		}
		found = idx
	}
	hash = commandList[found].GetTraceName()

	return
}
<|endoftext|>


<commit_before>package appbits

import (
	"fmt"
	"net/http"
)

func AppHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Println("appbits.AppHandler not implemented")
}
<commit_msg>appbits resources demarshalling<commit_after>package appbits

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/cloudfoundry/cli/cf/api/resources"
	"github.com/glyn/bloblets/servutil"
)

func AppHandler(w http.ResponseWriter, r *http.Request) {
	r.ParseMultipartForm(100 * 1024 * 1024)
	mpForm := r.MultipartForm

	res := mpForm.Value["resources"]
	presentFiles := []resources.AppFileResource{}
	err := json.Unmarshal([]byte(res[0]), &presentFiles)
	if err != nil {
		servutil.Fail(w, "demarshalling resources failed: %s", err)
		return
	}

	application := mpForm.File["application"]

	fmt.Printf("appbits.AppHandler not implemented:\npresentFiles=%#v\napplication=%#v\n", presentFiles, application)
}
<|endoftext|>


<commit_before>// +build integration

// Copyright (c) 2020 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package integration

import (
	"testing"
	"time"

	"github.com/m3db/m3/src/dbnode/client"
	"github.com/m3db/m3/src/dbnode/generated/proto/annotation"
	"github.com/m3db/m3/src/dbnode/namespace"
	"github.com/m3db/m3/src/dbnode/persist/fs"
	"github.com/m3db/m3/src/dbnode/retention"
	"github.com/m3db/m3/src/dbnode/storage"
	"github.com/m3db/m3/src/dbnode/ts"
	xclock "github.com/m3db/m3/src/x/clock"
	"github.com/m3db/m3/src/x/ident"
	xtime "github.com/m3db/m3/src/x/time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"go.uber.org/zap"
)

var (
	blockSize  = 2 * time.Hour
	blockSizeT = 24 * time.Hour

	gaugePayload   = &annotation.Payload{MetricType: annotation.MetricType_GAUGE}
	counterPayload = &annotation.Payload{MetricType: annotation.MetricType_COUNTER, HandleValueResets: true}
)

func TestReadAggregateWrite(t *testing.T) {
	var (
		start                   = time.Now()
		testSetup, srcNs, trgNs = setupServer(t)
		storageOpts             = testSetup.StorageOpts()
		log                     = storageOpts.InstrumentOptions().Logger()
	)

	// Stop the server.
	defer func() {
		require.NoError(t, testSetup.StopServer())
		log.Debug("server is now down")
		testSetup.Close()
	}()

	session, err := testSetup.M3DBClient().DefaultSession()
	require.NoError(t, err)
	nowFn := testSetup.NowFn()

	// Write test data.
	dpTimeStart := nowFn().Truncate(blockSizeT).Add(-blockSizeT)
	dpTime := dpTimeStart

	// "aab" ID is stored to the same shard 0 same as "foo", this is important
	// for a test to store them to the same shard to test data consistency
	err = session.WriteTagged(srcNs.ID(), ident.StringID("aab"),
		ident.MustNewTagStringsIterator("__name__", "cpu", "job", "job1"),
		dpTime, 15, xtime.Second, annotationBytes(t, gaugePayload))

	testDataPointsCount := 60
	for a := 0; a < testDataPointsCount; a++ {
		if a < 10 {
			dpTime = dpTime.Add(10 * time.Minute)
			continue
		}
		err = session.WriteTagged(srcNs.ID(), ident.StringID("foo"),
			ident.MustNewTagStringsIterator("__name__", "cpu", "job", "job1"),
			dpTime, 42.1+float64(a), xtime.Nanosecond, annotationBytes(t, counterPayload))
		require.NoError(t, err)
		dpTime = dpTime.Add(10 * time.Minute)
	}
	log.Info("test data written", zap.Duration("took", time.Since(start)))

	log.Info("waiting till data is cold flushed")
	start = time.Now()
	expectedSourceBlocks := 5
	flushed := xclock.WaitUntil(func() bool {
		for i := 0; i < expectedSourceBlocks; i++ {
			blockStart := dpTimeStart.Add(time.Duration(i) * blockSize)
			_, ok, err := fs.FileSetAt(testSetup.FilesystemOpts().FilePathPrefix(), srcNs.ID(), 0, blockStart, 1)
			require.NoError(t, err)
			if !ok {
				return false
			}
		}
		return true
	}, time.Minute)
	require.True(t, flushed)
	log.Info("verified data has been cold flushed", zap.Duration("took", time.Since(start)))

	aggOpts, err := storage.NewAggregateTilesOptions(
		dpTimeStart, dpTimeStart.Add(blockSizeT), time.Hour,
		trgNs.ID(),
		storageOpts.InstrumentOptions(),
	)
	require.NoError(t, err)

	log.Info("Starting aggregation")
	start = time.Now()
	processedTileCount, err := testSetup.DB().AggregateTiles(
		storageOpts.ContextPool().Get(),
		srcNs.ID(), trgNs.ID(),
		aggOpts)
	log.Info("Finished aggregation", zap.Duration("took", time.Since(start)))
	require.NoError(t, err)
	assert.Equal(t, int64(10), processedTileCount)

	log.Info("validating aggregated data")

	// check shard 0 as we wrote both aab and foo to this shard.
	flushState, err := testSetup.DB().FlushState(trgNs.ID(), 0, dpTimeStart)
	require.NoError(t, err)
	require.Equal(t, 1, flushState.ColdVersionRetrievable)
	require.Equal(t, 1, flushState.ColdVersionFlushed)

	log.Info("waiting till aggregated data is readable")
	start = time.Now()
	readable := xclock.WaitUntil(func() bool {
		series, err := session.Fetch(trgNs.ID(), ident.StringID("foo"), dpTimeStart, nowFn())
		require.NoError(t, err)
		return series.Next()
	}, time.Minute)
	require.True(t, readable)
	log.Info("verified data is readable", zap.Duration("took", time.Since(start)))

	expectedDps := []ts.Datapoint{
		{Timestamp: dpTimeStart, Value: 15},
	}
	fetchAndValidate(t, session, trgNs.ID(),
		ident.StringID("aab"),
		dpTimeStart, nowFn(),
		expectedDps, xtime.Second, gaugePayload)

	expectedDps = []ts.Datapoint{
		{Timestamp: dpTimeStart.Add(100 * time.Minute), Value: 52.1},
		{Timestamp: dpTimeStart.Add(110 * time.Minute), Value: 53.1},
		{Timestamp: dpTimeStart.Add(170 * time.Minute), Value: 59.1},
		{Timestamp: dpTimeStart.Add(230 * time.Minute), Value: 65.1},
		{Timestamp: dpTimeStart.Add(290 * time.Minute), Value: 71.1},
		{Timestamp: dpTimeStart.Add(350 * time.Minute), Value: 77.1},
		{Timestamp: dpTimeStart.Add(410 * time.Minute), Value: 83.1},
		{Timestamp: dpTimeStart.Add(470 * time.Minute), Value: 89.1},
		{Timestamp: dpTimeStart.Add(530 * time.Minute), Value: 95.1},
		{Timestamp: dpTimeStart.Add(590 * time.Minute), Value: 101.1},
	}

	fetchAndValidate(t, session, trgNs.ID(),
		ident.StringID("foo"),
		dpTimeStart, nowFn(),
		expectedDps, xtime.Nanosecond, counterPayload)
}

func fetchAndValidate(
	t *testing.T,
	session client.Session,
	nsID ident.ID,
	id ident.ID,
	startInclusive, endExclusive time.Time,
	expectedDP []ts.Datapoint,
	expectedUnit xtime.Unit,
	expectedAnnotation *annotation.Payload,
) {
	series, err := session.Fetch(nsID, id, startInclusive, endExclusive)
	require.NoError(t, err)

	actual := make([]ts.Datapoint, 0, len(expectedDP))
	first := true
	for series.Next() {
		dp, unit, annotation := series.Current()
		if first {
			assert.Equal(t, expectedAnnotation, annotationPayload(t, annotation))
			first = false
		}
		assert.Equal(t, expectedUnit, unit)
		dp.TimestampNanos = 0
		actual = append(actual, dp)
	}

	assert.Equal(t, expectedDP, actual)
}

func setupServer(t *testing.T) (TestSetup, namespace.Metadata, namespace.Metadata) {
	var (
		rOpts    = retention.NewOptions().SetRetentionPeriod(500 * blockSize).SetBlockSize(blockSize)
		rOptsT   = retention.NewOptions().SetRetentionPeriod(100 * blockSize).SetBlockSize(blockSizeT).SetBufferPast(0)
		idxOpts  = namespace.NewIndexOptions().SetEnabled(true).SetBlockSize(blockSize)
		idxOptsT = namespace.NewIndexOptions().SetEnabled(true).SetBlockSize(blockSizeT)
		nsOpts   = namespace.NewOptions().
				SetRetentionOptions(rOpts).
				SetIndexOptions(idxOpts).
				SetColdWritesEnabled(true)
		nsOptsT = namespace.NewOptions().
			SetRetentionOptions(rOptsT).
			SetIndexOptions(idxOptsT)

		fixedNow = time.Now().Truncate(blockSizeT)
	)

	srcNs, err := namespace.NewMetadata(testNamespaces[0], nsOpts)
	require.NoError(t, err)
	trgNs, err := namespace.NewMetadata(testNamespaces[1], nsOptsT)
	require.NoError(t, err)

	testOpts := NewTestOptions(t).
		SetNamespaces([]namespace.Metadata{srcNs, trgNs}).
		SetWriteNewSeriesAsync(true).
		SetNumShards(1).
		SetFetchRequestTimeout(time.Second * 30).
		SetNowFn(func() time.Time {
			return fixedNow
		})

	testSetup := newTestSetupWithCommitLogAndFilesystemBootstrapper(t, testOpts)

	// Start the server.
	require.NoError(t, testSetup.StartServer())

	return testSetup, srcNs, trgNs
}

func annotationBytes(t *testing.T, payload *annotation.Payload) ts.Annotation {
	if payload != nil {
		annotationBytes, err := payload.Marshal()
		require.NoError(t, err)
		return annotationBytes
	}
	return nil
}

func annotationPayload(t *testing.T, annotationBytes ts.Annotation) *annotation.Payload {
	if annotationBytes != nil {
		payload := &annotation.Payload{}
		require.NoError(t, payload.Unmarshal(annotationBytes))
		return payload
	}
	return nil
}
<commit_msg>[dbnode] Skip flaky test (#2847)<commit_after>// +build integration

// Copyright (c) 2020 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package integration

import (
	"testing"
	"time"

	"github.com/m3db/m3/src/dbnode/client"
	"github.com/m3db/m3/src/dbnode/generated/proto/annotation"
	"github.com/m3db/m3/src/dbnode/namespace"
	"github.com/m3db/m3/src/dbnode/persist/fs"
	"github.com/m3db/m3/src/dbnode/retention"
	"github.com/m3db/m3/src/dbnode/storage"
	"github.com/m3db/m3/src/dbnode/ts"
	xclock "github.com/m3db/m3/src/x/clock"
	"github.com/m3db/m3/src/x/ident"
	xtime "github.com/m3db/m3/src/x/time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"go.uber.org/zap"
)

var (
	blockSize  = 2 * time.Hour
	blockSizeT = 24 * time.Hour

	gaugePayload   = &annotation.Payload{MetricType: annotation.MetricType_GAUGE}
	counterPayload = &annotation.Payload{MetricType: annotation.MetricType_COUNTER, HandleValueResets: true}
)

func TestReadAggregateWrite(t *testing.T) {
	t.Skip("flaky")
	var (
		start                   = time.Now()
		testSetup, srcNs, trgNs = setupServer(t)
		storageOpts             = testSetup.StorageOpts()
		log                     = storageOpts.InstrumentOptions().Logger()
	)

	// Stop the server.
	defer func() {
		require.NoError(t, testSetup.StopServer())
		log.Debug("server is now down")
		testSetup.Close()
	}()

	session, err := testSetup.M3DBClient().DefaultSession()
	require.NoError(t, err)
	nowFn := testSetup.NowFn()

	// Write test data.
	dpTimeStart := nowFn().Truncate(blockSizeT).Add(-blockSizeT)
	dpTime := dpTimeStart

	// "aab" ID is stored to the same shard 0 same as "foo", this is important
	// for a test to store them to the same shard to test data consistency
	err = session.WriteTagged(srcNs.ID(), ident.StringID("aab"),
		ident.MustNewTagStringsIterator("__name__", "cpu", "job", "job1"),
		dpTime, 15, xtime.Second, annotationBytes(t, gaugePayload))

	testDataPointsCount := 60
	for a := 0; a < testDataPointsCount; a++ {
		if a < 10 {
			dpTime = dpTime.Add(10 * time.Minute)
			continue
		}
		err = session.WriteTagged(srcNs.ID(), ident.StringID("foo"),
			ident.MustNewTagStringsIterator("__name__", "cpu", "job", "job1"),
			dpTime, 42.1+float64(a), xtime.Nanosecond, annotationBytes(t, counterPayload))
		require.NoError(t, err)
		dpTime = dpTime.Add(10 * time.Minute)
	}
	log.Info("test data written", zap.Duration("took", time.Since(start)))

	log.Info("waiting till data is cold flushed")
	start = time.Now()
	expectedSourceBlocks := 5
	flushed := xclock.WaitUntil(func() bool {
		for i := 0; i < expectedSourceBlocks; i++ {
			blockStart := dpTimeStart.Add(time.Duration(i) * blockSize)
			_, ok, err := fs.FileSetAt(testSetup.FilesystemOpts().FilePathPrefix(), srcNs.ID(), 0, blockStart, 1)
			require.NoError(t, err)
			if !ok {
				return false
			}
		}
		return true
	}, time.Minute)
	require.True(t, flushed)
	log.Info("verified data has been cold flushed", zap.Duration("took", time.Since(start)))

	aggOpts, err := storage.NewAggregateTilesOptions(
		dpTimeStart, dpTimeStart.Add(blockSizeT), time.Hour,
		trgNs.ID(),
		storageOpts.InstrumentOptions(),
	)
	require.NoError(t, err)

	log.Info("Starting aggregation")
	start = time.Now()
	processedTileCount, err := testSetup.DB().AggregateTiles(
		storageOpts.ContextPool().Get(),
		srcNs.ID(), trgNs.ID(),
		aggOpts)
	log.Info("Finished aggregation", zap.Duration("took", time.Since(start)))
	require.NoError(t, err)
	assert.Equal(t, int64(10), processedTileCount)

	log.Info("validating aggregated data")

	// check shard 0 as we wrote both aab and foo to this shard.
	flushState, err := testSetup.DB().FlushState(trgNs.ID(), 0, dpTimeStart)
	require.NoError(t, err)
	require.Equal(t, 1, flushState.ColdVersionRetrievable)
	require.Equal(t, 1, flushState.ColdVersionFlushed)

	log.Info("waiting till aggregated data is readable")
	start = time.Now()
	readable := xclock.WaitUntil(func() bool {
		series, err := session.Fetch(trgNs.ID(), ident.StringID("foo"), dpTimeStart, nowFn())
		require.NoError(t, err)
		return series.Next()
	}, time.Minute)
	require.True(t, readable)
	log.Info("verified data is readable", zap.Duration("took", time.Since(start)))

	expectedDps := []ts.Datapoint{
		{Timestamp: dpTimeStart, Value: 15},
	}
	fetchAndValidate(t, session, trgNs.ID(),
		ident.StringID("aab"),
		dpTimeStart, nowFn(),
		expectedDps, xtime.Second, gaugePayload)

	expectedDps = []ts.Datapoint{
		{Timestamp: dpTimeStart.Add(100 * time.Minute), Value: 52.1},
		{Timestamp: dpTimeStart.Add(110 * time.Minute), Value: 53.1},
		{Timestamp: dpTimeStart.Add(170 * time.Minute), Value: 59.1},
		{Timestamp: dpTimeStart.Add(230 * time.Minute), Value: 65.1},
		{Timestamp: dpTimeStart.Add(290 * time.Minute), Value: 71.1},
		{Timestamp: dpTimeStart.Add(350 * time.Minute), Value: 77.1},
		{Timestamp: dpTimeStart.Add(410 * time.Minute), Value: 83.1},
		{Timestamp: dpTimeStart.Add(470 * time.Minute), Value: 89.1},
		{Timestamp: dpTimeStart.Add(530 * time.Minute), Value: 95.1},
		{Timestamp: dpTimeStart.Add(590 * time.Minute), Value: 101.1},
	}

	fetchAndValidate(t, session, trgNs.ID(),
		ident.StringID("foo"),
		dpTimeStart, nowFn(),
		expectedDps, xtime.Nanosecond, counterPayload)
}

func fetchAndValidate(
	t *testing.T,
	session client.Session,
	nsID ident.ID,
	id ident.ID,
	startInclusive, endExclusive time.Time,
	expectedDP []ts.Datapoint,
	expectedUnit xtime.Unit,
	expectedAnnotation *annotation.Payload,
) {
	series, err := session.Fetch(nsID, id, startInclusive, endExclusive)
	require.NoError(t, err)

	actual := make([]ts.Datapoint, 0, len(expectedDP))
	first := true
	for series.Next() {
		dp, unit, annotation := series.Current()
		if first {
			assert.Equal(t, expectedAnnotation, annotationPayload(t, annotation))
			first = false
		}
		assert.Equal(t, expectedUnit, unit)
		dp.TimestampNanos = 0
		actual = append(actual, dp)
	}

	assert.Equal(t, expectedDP, actual)
}

func setupServer(t *testing.T) (TestSetup, namespace.Metadata, namespace.Metadata) {
	var (
		rOpts    = retention.NewOptions().SetRetentionPeriod(500 * blockSize).SetBlockSize(blockSize)
		rOptsT   = retention.NewOptions().SetRetentionPeriod(100 * blockSize).SetBlockSize(blockSizeT).SetBufferPast(0)
		idxOpts  = namespace.NewIndexOptions().SetEnabled(true).SetBlockSize(blockSize)
		idxOptsT = namespace.NewIndexOptions().SetEnabled(true).SetBlockSize(blockSizeT)
		nsOpts   = namespace.NewOptions().
				SetRetentionOptions(rOpts).
				SetIndexOptions(idxOpts).
				SetColdWritesEnabled(true)
		nsOptsT = namespace.NewOptions().
			SetRetentionOptions(rOptsT).
			SetIndexOptions(idxOptsT)

		fixedNow = time.Now().Truncate(blockSizeT)
	)

	srcNs, err := namespace.NewMetadata(testNamespaces[0], nsOpts)
	require.NoError(t, err)
	trgNs, err := namespace.NewMetadata(testNamespaces[1], nsOptsT)
	require.NoError(t, err)

	testOpts := NewTestOptions(t).
		SetNamespaces([]namespace.Metadata{srcNs, trgNs}).
		SetWriteNewSeriesAsync(true).
		SetNumShards(1).
		SetFetchRequestTimeout(time.Second * 30).
		SetNowFn(func() time.Time {
			return fixedNow
		})

	testSetup := newTestSetupWithCommitLogAndFilesystemBootstrapper(t, testOpts)

	// Start the server.
	require.NoError(t, testSetup.StartServer())

	return testSetup, srcNs, trgNs
}

func annotationBytes(t *testing.T, payload *annotation.Payload) ts.Annotation {
	if payload != nil {
		annotationBytes, err := payload.Marshal()
		require.NoError(t, err)
		return annotationBytes
	}
	return nil
}

func annotationPayload(t *testing.T, annotationBytes ts.Annotation) *annotation.Payload {
	if annotationBytes != nil {
		payload := &annotation.Payload{}
		require.NoError(t, payload.Unmarshal(annotationBytes))
		return payload
	}
	return nil
}
<|endoftext|>


<commit_before>/*
    User : https://api.github.com/users/Omie
        returns a dict
        has repos_url : https://api.github.com/users/Omie/repos
    Repos : https://api.github.com/users/Omie/repos
        returns a list of dict
        has contributors_url : https://api.github.com/repos/Omie/configfiles/contributors
    Contributors : https://api.github.com/repos/Omie/configfiles/contributors
        returns a list of dict
        has repos_url for each user
*/

package main

import (
        "os"
        "fmt"
        "encoding/json"
        "io/ioutil"
        "net/http"
        "log"
        "errors"
        "github.com/omie/ghlib"
)

/*  types used for marhsalling data to json */
type node struct {
    Name string `json:"name"`
    Group int `json:"group"`
    Image string `json:"image"`
}

type connection struct {
    Source int `json:"source"`
    Target int `json:"target"`
    Value int `json:"value"`
}

type graphdata struct {
    Nodes []node `json:"nodes"`
    Connections []connection `json:"links"`
}

// keep track of visited URLs and previously known users
var visited = make(map[string]string)
var knownUsers = make(map[string]int)

// gets set to current limit in runtime
var requestsLeft = 60

// github a/c credentials
var username, password string

// determines how deep to crawl
var maxDepth int

// holds list of users found as nodes
// node-to-node connections using 0 based index
// as required for d3
var nodes []node
var connections []connection

// get data from remote url and return unparsed output
func getData(url string) ([]byte, error) {
        log.Println("--- reached getData for ", url)

        requestsLeft--
        if requestsLeft < 0 {
            log.Println("--- LIMIT REACHED ")
            return nil, errors.New("limit reached")
        }

        client := &http.Client{}

        /* Authenticate */
        req, err := http.NewRequest("GET", url, nil)
        req.SetBasicAuth(username, password)
        resp, err := client.Do(req)
        if err != nil {
            log.Println("error in http request: ", err)
            return nil, err
        }
        defer resp.Body.Close()

        body, err := ioutil.ReadAll(resp.Body)
        if err != nil {
            log.Println("error reading request body: ", err)
            return nil, err
        }

        return body, nil
}

// determine current API limit
func getAPILimit() (int, error) {
    jsonData, err := getData("https://api.github.com/rate_limit")
    if err != nil {
        return 0, err
    }

    var limitData ghlib.GhLimit
    if err := json.Unmarshal(jsonData, &limitData); err != nil {
        return 0, err
    }

    limit := limitData.Rate.Remaining
    if limit <= 10 {
        return 0, errors.New("Too few of API calls left. Not worth it.")
    }

    return limit, nil
}

// get User details from API, retrieve repos_url for the user and
// return the same
func getReposURL(username string) (string, error) {
        log.Println("--- reached getReposURL for ", username)

        userJSONData, err := getData("https://api.github.com/users/" + username)
        if err != nil {
            return "", err
        }

        var user ghlib.GhUser
        if err := json.Unmarshal(userJSONData, &user); err != nil {
            return "", err
        }
        return user.ReposUrl, nil
}

// repo contributors is a list of maps [ {}, {}]
// retrieve the list, parse json, for each user:
//      process user's repositories to move to further depth
func processContributors(contribURL string, currentDepth int, parent int) {
        log.Println("--- reached processContributors for ", contribURL)
        if _, exists := visited[contribURL]; exists {
            log.Println("--- skipped ", contribURL)
            return
        }
        visited[contribURL] = contribURL

        jsonData, err := getData(contribURL)
        if err != nil {
            log.Println("error while getting contributors data: ", err)
            return
        }

        var contributors []*ghlib.GhUser
        err = json.Unmarshal(jsonData, &contributors)
        if err != nil {
            log.Println("error while unmarshalling contributors: ", err)
            return
        }
        //for each contributor
        for _, contributor := range contributors {
            //if user is already visited, just mark a connection and move to next
            tempUser := contributor.Login
            if nodeIdx, exists := knownUsers[tempUser]; exists {
                connections = append(connections, connection{parent, nodeIdx, 1})
                continue
            }
            //We found new user in network
            /* this might slow down
            for t:=0; t<=currentDepth; t++ {
                fmt.Print("\t|")
            }
            */
            fmt.Println(tempUser)

            //push to nodes list and connection list
            nodes = append(nodes, node{tempUser, 1, contributor.AvatarUrl})
            nodeIdx := len(nodes)-1
            connections = append(connections, connection{parent, nodeIdx, 1})

            knownUsers[tempUser] = nodeIdx
            tempRepoURL := contributor.ReposUrl

            //get repositories of this new user
            processRepos(tempRepoURL, currentDepth+1, nodeIdx)

        } //end for
}

// process a list of repositories
// for each repository, find and process collaborators
func processRepos(repoURL string, currentDepth int, parent int) {
        log.Println("--- reached processRepos for ", repoURL)
        if currentDepth > maxDepth {
            log.Println("maxDepth reached")
            return
        }

        if _, exists := visited[repoURL]; exists {
            return
        }
        visited[repoURL] = repoURL

        repoData, err := getData(repoURL) //get a list of repositories
        if err != nil {
            log.Println("error while getting repo list data: ", err)
            return
        }

        var repoList []*ghlib.GhRepository
        err = json.Unmarshal(repoData, &repoList)
        if err != nil {
            log.Println("error while unmarshalling repo list: ", err)
            return
        }

        for _, repo := range repoList {
            contribURL := repo.ContributorsUrl
            processContributors(contribURL, currentDepth, parent)
        }

} //end processRepos

func handleUserInput() {
    fmt.Println("Enter Github credentials -")
    fmt.Print("username(Email address): ")
    fmt.Scanln(&username)

    fmt.Print("password: ")
    fmt.Scanln(&password)

    fmt.Print("Max depth: ")
    fmt.Scanln(&maxDepth)
}

func dumpD3Json() {
    fw, err := os.OpenFile("graph.json", os.O_WRONLY | os.O_CREATE | os.O_TRUNC, 0666)
    if err != nil {
        fmt.Println("Could not open file for writing json")
        return
    }
    defer fw.Close()

    var gdata = &graphdata {
        Nodes: nodes,
        Connections: connections,
    }
    var toWrite []byte
    toWrite, err = json.Marshal(gdata)
    if err != nil {
        fmt.Println("Error marshalling data: ", err)
    }
    fw.Write(toWrite)
}

func main() {
    f, err := os.OpenFile("/tmp/linkedhub.log", os.O_WRONLY | os.O_CREATE | os.O_APPEND, 0666)
    if err != nil {
        fmt.Println("Could not open file for logging")
        return
    }
    defer f.Close()

    //log.SetOutput(f)
    log.SetOutput(ioutil.Discard)

    handleUserInput()

    //find out current API limit
    requestsLeft, err := getAPILimit()
    if err != nil {
        fmt.Println("error while getting api limit: ", err)
        return
    }
    fmt.Println("requests left for this hour: ", requestsLeft)

    //get username from command line
    var u string
    fmt.Println("Enter github username to start with: ")
    fmt.Scanln(&u)

    repoURL, err := getReposURL(u)
    if err != nil {
        log.Println("error while getting repo url for: ", u)
        return
    }

    processRepos(repoURL, 0, 0)
    dumpD3Json()
}

<commit_msg>minor fix. requestLeft regression<commit_after>/*
    User : https://api.github.com/users/Omie
        returns a dict
        has repos_url : https://api.github.com/users/Omie/repos
    Repos : https://api.github.com/users/Omie/repos
        returns a list of dict
        has contributors_url : https://api.github.com/repos/Omie/configfiles/contributors
    Contributors : https://api.github.com/repos/Omie/configfiles/contributors
        returns a list of dict
        has repos_url for each user
*/

package main

import (
        "os"
        "fmt"
        "encoding/json"
        "io/ioutil"
        "net/http"
        "log"
        "errors"
        "github.com/omie/ghlib"
)

/*  types used for marhsalling data to json */
type node struct {
    Name string `json:"name"`
    Group int `json:"group"`
    Image string `json:"image"`
}

type connection struct {
    Source int `json:"source"`
    Target int `json:"target"`
    Value int `json:"value"`
}

type graphdata struct {
    Nodes []node `json:"nodes"`
    Connections []connection `json:"links"`
}

// keep track of visited URLs and previously known users
var visited = make(map[string]string)
var knownUsers = make(map[string]int)

// gets set to current limit in runtime
var requestsLeft = 60

// github a/c credentials
var username, password string

// determines how deep to crawl
var maxDepth int

// holds list of users found as nodes
// node-to-node connections using 0 based index
// as required for d3
var nodes []node
var connections []connection

// get data from remote url and return unparsed output
func getData(url string) ([]byte, error) {
        log.Println("--- reached getData for ", url)

        requestsLeft--
        if requestsLeft < 0 {
            log.Println("--- LIMIT REACHED ")
            return nil, errors.New("limit reached")
        }

        client := &http.Client{}

        /* Authenticate */
        req, err := http.NewRequest("GET", url, nil)
        req.SetBasicAuth(username, password)
        resp, err := client.Do(req)
        if err != nil {
            log.Println("error in http request: ", err)
            return nil, err
        }
        defer resp.Body.Close()

        body, err := ioutil.ReadAll(resp.Body)
        if err != nil {
            log.Println("error reading request body: ", err)
            return nil, err
        }

        return body, nil
}

// determine current API limit
func getAPILimit() (int, error) {
    jsonData, err := getData("https://api.github.com/rate_limit")
    if err != nil {
        return 0, err
    }

    var limitData ghlib.GhLimit
    if err := json.Unmarshal(jsonData, &limitData); err != nil {
        return 0, err
    }

    limit := limitData.Rate.Remaining
    if limit <= 10 {
        return 0, errors.New("Too few of API calls left. Not worth it.")
    }

    return limit, nil
}

// get User details from API, retrieve repos_url for the user and
// return the same
func getReposURL(username string) (string, error) {
        log.Println("--- reached getReposURL for ", username)

        userJSONData, err := getData("https://api.github.com/users/" + username)
        if err != nil {
            return "", err
        }

        var user ghlib.GhUser
        if err := json.Unmarshal(userJSONData, &user); err != nil {
            return "", err
        }
        return user.ReposUrl, nil
}

// repo contributors is a list of maps [ {}, {}]
// retrieve the list, parse json, for each user:
//      process user's repositories to move to further depth
func processContributors(contribURL string, currentDepth int, parent int) {
        log.Println("--- reached processContributors for ", contribURL)
        if _, exists := visited[contribURL]; exists {
            log.Println("--- skipped ", contribURL)
            return
        }
        visited[contribURL] = contribURL

        jsonData, err := getData(contribURL)
        if err != nil {
            log.Println("error while getting contributors data: ", err)
            return
        }

        var contributors []*ghlib.GhUser
        err = json.Unmarshal(jsonData, &contributors)
        if err != nil {
            log.Println("error while unmarshalling contributors: ", err)
            return
        }
        //for each contributor
        for _, contributor := range contributors {
            //if user is already visited, just mark a connection and move to next
            tempUser := contributor.Login
            if nodeIdx, exists := knownUsers[tempUser]; exists {
                connections = append(connections, connection{parent, nodeIdx, 1})
                continue
            }
            //We found new user in network
            /* this might slow down
            for t:=0; t<=currentDepth; t++ {
                fmt.Print("\t|")
            }
            */
            fmt.Println(tempUser)

            //push to nodes list and connection list
            nodes = append(nodes, node{tempUser, 1, contributor.AvatarUrl})
            nodeIdx := len(nodes)-1
            connections = append(connections, connection{parent, nodeIdx, 1})

            knownUsers[tempUser] = nodeIdx
            tempRepoURL := contributor.ReposUrl

            //get repositories of this new user
            processRepos(tempRepoURL, currentDepth+1, nodeIdx)

        } //end for
}

// process a list of repositories
// for each repository, find and process collaborators
func processRepos(repoURL string, currentDepth int, parent int) {
        log.Println("--- reached processRepos for ", repoURL)
        if currentDepth > maxDepth {
            log.Println("maxDepth reached")
            return
        }

        if _, exists := visited[repoURL]; exists {
            return
        }
        visited[repoURL] = repoURL

        repoData, err := getData(repoURL) //get a list of repositories
        if err != nil {
            log.Println("error while getting repo list data: ", err)
            return
        }

        var repoList []*ghlib.GhRepository
        err = json.Unmarshal(repoData, &repoList)
        if err != nil {
            log.Println("error while unmarshalling repo list: ", err)
            return
        }

        for _, repo := range repoList {
            contribURL := repo.ContributorsUrl
            processContributors(contribURL, currentDepth, parent)
        }

} //end processRepos

func handleUserInput() {
    fmt.Println("Enter Github credentials -")
    fmt.Print("username(Email address): ")
    fmt.Scanln(&username)

    fmt.Print("password: ")
    fmt.Scanln(&password)

    fmt.Print("Max depth: ")
    fmt.Scanln(&maxDepth)
}

func dumpD3Json() {
    fw, err := os.OpenFile("graph.json", os.O_WRONLY | os.O_CREATE | os.O_TRUNC, 0666)
    if err != nil {
        fmt.Println("Could not open file for writing json")
        return
    }
    defer fw.Close()

    var gdata = &graphdata {
        Nodes: nodes,
        Connections: connections,
    }
    var toWrite []byte
    toWrite, err = json.Marshal(gdata)
    if err != nil {
        fmt.Println("Error marshalling data: ", err)
    }
    fw.Write(toWrite)
}

func main() {
    f, err := os.OpenFile("/tmp/linkedhub.log", os.O_WRONLY | os.O_CREATE | os.O_APPEND, 0666)
    if err != nil {
        fmt.Println("Could not open file for logging")
        return
    }
    defer f.Close()

    log.SetOutput(f)
    //log.SetOutput(ioutil.Discard)

    handleUserInput()

    //find out current API limit
    limit, err := getAPILimit()
    if err != nil {
        fmt.Println("error while getting api limit: ", err)
        return
    }
    requestsLeft = limit
    fmt.Println("requests left for this hour: ", requestsLeft)

    //get username from command line
    var u string
    fmt.Println("Enter github username to start with: ")
    fmt.Scanln(&u)

    repoURL, err := getReposURL(u)
    if err != nil {
        log.Println("error while getting repo url for: ", u)
        return
    }

    processRepos(repoURL, 0, 0)
    fmt.Println("dumping ", string(len(nodes)), " nodes to json")
    dumpD3Json()
}

<|endoftext|>


<commit_before>package backends

import (
	"github.com/3onyc/hipdate"
	"github.com/garyburd/redigo/redis"
	"log"
)

type HipacheBackend struct {
	r redis.Conn
}

func (hb *HipacheBackend) AddUpstream(
	h hipdate.Host,
	u hipdate.Upstream,
) error {
	exists, err := hb.HostExists(h)
	if err != nil {
		log.Println(err)
	}

	if !exists {
		if err := hb.HostCreate(h); err != nil {
			log.Println(err)
		}
	}

	if _, err := hb.r.Do("RPUSH", h.Key(), u); err != nil {
		return err
	}
	log.Println("Registered", h, u)

	return nil
}
func (hb *HipacheBackend) RemoveUpstream(
	h hipdate.Host,
	u hipdate.Upstream,
) error {
	if _, err := hb.r.Do("LREM", h.Key(), 0, u); err != nil {
		return err
	}

	log.Println("Unregistered", h, u)
	return nil
}
func (hb *HipacheBackend) HostExists(h hipdate.Host) (bool, error) {
	return redis.Bool(hb.r.Do("EXISTS", h.Key()))
}

func (hb *HipacheBackend) HostDelete(h hipdate.Host) error {
	if _, err := hb.r.Do("DEL", h.Key()); err != nil {
		return err
	}
	log.Println("Deleted", h)

	return nil
}

func (hb *HipacheBackend) HostCreate(h hipdate.Host) error {
	if _, err := hb.r.Do("RPUSH", h.Key(), h); err != nil {
		return err
	}
	log.Println("Created", h)

	return nil
}

func (hb *HipacheBackend) HostInitialise(h hipdate.Host) error {
	if err := hb.HostDelete(h); err != nil {
		return err
	}

	if err := hb.HostCreate(h); err != nil {
		return err
	}

	log.Println("Initialised", h)
	return nil
}

func NewHipacheBackend(r redis.Conn) *HipacheBackend {
	return &HipacheBackend{
		r: r,
	}
}
<commit_msg>Proper visibility on Hipache backend functions<commit_after>package backends

import (
	"github.com/3onyc/hipdate"
	"github.com/garyburd/redigo/redis"
	"log"
)

type HipacheBackend struct {
	r redis.Conn
}

func (hb *HipacheBackend) AddUpstream(
	h hipdate.Host,
	u hipdate.Upstream,
) error {
	exists, err := hb.hostExists(h)
	if err != nil {
		log.Println(err)
	}

	if !exists {
		if err := hb.hostCreate(h); err != nil {
			log.Println(err)
		}
	}

	if _, err := hb.r.Do("RPUSH", h.Key(), u); err != nil {
		return err
	}
	log.Println("Registered", h, u)

	return nil
}
func (hb *HipacheBackend) RemoveUpstream(
	h hipdate.Host,
	u hipdate.Upstream,
) error {
	if _, err := hb.r.Do("LREM", h.Key(), 0, u); err != nil {
		return err
	}

	log.Println("Unregistered", h, u)
	return nil
}
func (hb *HipacheBackend) hostExists(h hipdate.Host) (bool, error) {
	return redis.Bool(hb.r.Do("EXISTS", h.Key()))
}

func (hb *HipacheBackend) hostDelete(h hipdate.Host) error {
	if _, err := hb.r.Do("DEL", h.Key()); err != nil {
		return err
	}
	log.Println("Deleted", h)

	return nil
}

func (hb *HipacheBackend) hostCreate(h hipdate.Host) error {
	if _, err := hb.r.Do("RPUSH", h.Key(), h); err != nil {
		return err
	}
	log.Println("Created", h)

	return nil
}

func (hb *HipacheBackend) hostClear(h hipdate.Host) error {
	if err := hb.hostDelete(h); err != nil {
		return err
	}

	if err := hb.hostCreate(h); err != nil {
		return err
	}

	log.Println("Initialised", h)
	return nil
}

func NewHipacheBackend(r redis.Conn) *HipacheBackend {
	return &HipacheBackend{
		r: r,
	}
}
<|endoftext|>