<commit_before>package cmd

import (
	"os"
	"os/signal"
	"strconv"
	"syscall"
	"time"

	"github.com/containrrr/watchtower/internal/actions"
	"github.com/containrrr/watchtower/internal/flags"
	"github.com/containrrr/watchtower/pkg/api"
	"github.com/containrrr/watchtower/pkg/container"
	"github.com/containrrr/watchtower/pkg/filters"
	"github.com/containrrr/watchtower/pkg/notifications"
	t "github.com/containrrr/watchtower/pkg/types"
	"github.com/robfig/cron"
	log "github.com/sirupsen/logrus"

	"github.com/spf13/cobra"
)

var (
	client         container.Client
	scheduleSpec   string
	cleanup        bool
	noRestart      bool
	monitorOnly    bool
	enableLabel    bool
	notifier       *notifications.Notifier
	timeout        time.Duration
	lifecycleHooks bool
)

var rootCmd = &cobra.Command{
	Use:   "watchtower",
	Short: "Automatically updates running Docker containers",
	Long: `
Watchtower automatically updates running Docker containers whenever a new image is released.
More information available at https://github.com/containrrr/watchtower/.
`,
	Run:    Run,
	PreRun: PreRun,
}

func init() {
	flags.SetDefaults()
	flags.RegisterDockerFlags(rootCmd)
	flags.RegisterSystemFlags(rootCmd)
	flags.RegisterNotificationFlags(rootCmd)
}

// Execute the root func and exit in case of errors
func Execute() {
	if err := rootCmd.Execute(); err != nil {
		log.Fatal(err)
	}
}

// PreRun is a lifecycle hook that runs before the command is executed.
func PreRun(cmd *cobra.Command, args []string) {
	f := cmd.PersistentFlags()

	if enabled, _ := f.GetBool("debug"); enabled {
		log.SetLevel(log.DebugLevel)
	}
	if enabled, _ := f.GetBool("trace"); enabled {
		log.SetLevel(log.TraceLevel)
	}

	pollingSet := f.Changed("interval")
	schedule, _ := f.GetString("schedule")
	cronLen := len(schedule)

	if pollingSet && cronLen > 0 {
		log.Fatal("Only schedule or interval can be defined, not both.")
	} else if cronLen > 0 {
		scheduleSpec, _ = f.GetString("schedule")
	} else {
		interval, _ := f.GetInt("interval")
		scheduleSpec = "@every " + strconv.Itoa(interval) + "s"
	}

	flags.GetSecretsFromFiles(cmd)
	cleanup, noRestart, monitorOnly, timeout = flags.ReadFlags(cmd)

	if timeout < 0 {
		log.Fatal("Please specify a positive value for timeout value.")
	}

	enableLabel, _ = f.GetBool("label-enable")
	lifecycleHooks, _ = f.GetBool("enable-lifecycle-hooks")

	// configure environment vars for client
	err := flags.EnvConfig(cmd)
	if err != nil {
		log.Fatal(err)
	}

	noPull, _ := f.GetBool("no-pull")
	includeStopped, _ := f.GetBool("include-stopped")
	reviveStopped, _ := f.GetBool("revive-stopped")
	removeVolumes, _ := f.GetBool("remove-volumes")

	client = container.NewClient(
		!noPull,
		includeStopped,
		reviveStopped,
		removeVolumes,
	)

	notifier = notifications.NewNotifier(cmd)
}

// Run is the main execution flow of the command
func Run(c *cobra.Command, names []string) {
	filter := filters.BuildFilter(names, enableLabel)
	runOnce, _ := c.PersistentFlags().GetBool("run-once")
	httpAPI, _ := c.PersistentFlags().GetBool("http-api")

	if httpAPI {
		apiToken, _ := c.PersistentFlags().GetString("http-api-token")

		if err := api.SetupHTTPUpdates(apiToken, func() { runUpdatesWithNotifications(filter) }); err != nil {
			log.Fatal(err)
			os.Exit(1)
		}

		api.WaitForHTTPUpdates()
	}

	if runOnce {
		if noStartupMessage, _ := c.PersistentFlags().GetBool("no-startup-message"); !noStartupMessage {
			log.Info("Running a one time update.")
		}
		runUpdatesWithNotifications(filter)
		os.Exit(0)
		return
	}

	if err := actions.CheckForMultipleWatchtowerInstances(client, cleanup); err != nil {
		log.Fatal(err)
	}

	if err := runUpgradesOnSchedule(c, filter); err != nil {
		log.Error(err)
	}

	os.Exit(1)
}

func runUpgradesOnSchedule(c *cobra.Command, filter t.Filter) error {
	tryLockSem := make(chan bool, 1)
	tryLockSem <- true

	cron := cron.New()
	err := cron.AddFunc(
		scheduleSpec,
		func() {
			select {
			case v := <-tryLockSem:
				defer func() { tryLockSem <- v }()
				runUpdatesWithNotifications(filter)
			default:
				log.Debug("Skipped another update already running.")
			}

			nextRuns := cron.Entries()
			if len(nextRuns) > 0 {
				log.Debug("Scheduled next run: " + nextRuns[0].Next.String())
			}
		})

	if err != nil {
		return err
	}

	if noStartupMessage, _ := c.PersistentFlags().GetBool("no-startup-message"); !noStartupMessage {
		log.Info("Starting Watchtower and scheduling first run: " + cron.Entries()[0].Schedule.Next(time.Now()).String())
	}

	cron.Start()

	// Graceful shut-down on SIGINT/SIGTERM
	interrupt := make(chan os.Signal, 1)
	signal.Notify(interrupt, os.Interrupt)
	signal.Notify(interrupt, syscall.SIGTERM)

	<-interrupt
	cron.Stop()
	log.Info("Waiting for running update to be finished...")
	<-tryLockSem
	return nil
}

func runUpdatesWithNotifications(filter t.Filter) {
	notifier.StartNotification()
	updateParams := t.UpdateParams{
		Filter:         filter,
		Cleanup:        cleanup,
		NoRestart:      noRestart,
		Timeout:        timeout,
		MonitorOnly:    monitorOnly,
		LifecycleHooks: lifecycleHooks,
	}
	err := actions.Update(client, updateParams)
	if err != nil {
		log.Println(err)
	}
	notifier.SendNotification()
}
<commit_msg>Warning if `WATCHTOWER_NO_PULL` and` WATCHTOWER_MONITOR_ONLY` are used simultaneously. (#594)<commit_after>package cmd

import (
	"os"
	"os/signal"
	"strconv"
	"syscall"
	"time"

	"github.com/containrrr/watchtower/internal/actions"
	"github.com/containrrr/watchtower/internal/flags"
	"github.com/containrrr/watchtower/pkg/api"
	"github.com/containrrr/watchtower/pkg/container"
	"github.com/containrrr/watchtower/pkg/filters"
	"github.com/containrrr/watchtower/pkg/notifications"
	t "github.com/containrrr/watchtower/pkg/types"
	"github.com/robfig/cron"
	log "github.com/sirupsen/logrus"

	"github.com/spf13/cobra"
)

var (
	client         container.Client
	scheduleSpec   string
	cleanup        bool
	noRestart      bool
	monitorOnly    bool
	enableLabel    bool
	notifier       *notifications.Notifier
	timeout        time.Duration
	lifecycleHooks bool
)

var rootCmd = &cobra.Command{
	Use:   "watchtower",
	Short: "Automatically updates running Docker containers",
	Long: `
Watchtower automatically updates running Docker containers whenever a new image is released.
More information available at https://github.com/containrrr/watchtower/.
`,
	Run:    Run,
	PreRun: PreRun,
}

func init() {
	flags.SetDefaults()
	flags.RegisterDockerFlags(rootCmd)
	flags.RegisterSystemFlags(rootCmd)
	flags.RegisterNotificationFlags(rootCmd)
}

// Execute the root func and exit in case of errors
func Execute() {
	if err := rootCmd.Execute(); err != nil {
		log.Fatal(err)
	}
}

// PreRun is a lifecycle hook that runs before the command is executed.
func PreRun(cmd *cobra.Command, args []string) {
	f := cmd.PersistentFlags()

	if enabled, _ := f.GetBool("debug"); enabled {
		log.SetLevel(log.DebugLevel)
	}
	if enabled, _ := f.GetBool("trace"); enabled {
		log.SetLevel(log.TraceLevel)
	}

	pollingSet := f.Changed("interval")
	schedule, _ := f.GetString("schedule")
	cronLen := len(schedule)

	if pollingSet && cronLen > 0 {
		log.Fatal("Only schedule or interval can be defined, not both.")
	} else if cronLen > 0 {
		scheduleSpec, _ = f.GetString("schedule")
	} else {
		interval, _ := f.GetInt("interval")
		scheduleSpec = "@every " + strconv.Itoa(interval) + "s"
	}

	flags.GetSecretsFromFiles(cmd)
	cleanup, noRestart, monitorOnly, timeout = flags.ReadFlags(cmd)

	if timeout < 0 {
		log.Fatal("Please specify a positive value for timeout value.")
	}

	enableLabel, _ = f.GetBool("label-enable")
	lifecycleHooks, _ = f.GetBool("enable-lifecycle-hooks")

	// configure environment vars for client
	err := flags.EnvConfig(cmd)
	if err != nil {
		log.Fatal(err)
	}

	noPull, _ := f.GetBool("no-pull")
	includeStopped, _ := f.GetBool("include-stopped")
	reviveStopped, _ := f.GetBool("revive-stopped")
	removeVolumes, _ := f.GetBool("remove-volumes")

	if monitorOnly && noPull {
		log.Warn("Using `WATCHTOWER_NO_PULL` and `WATCHTOWER_MONITOR_ONLY` simultaneously might lead to no action being taken at all. If this is intentional, you may safely ignore this message.")
	}

	client = container.NewClient(
		!noPull,
		includeStopped,
		reviveStopped,
		removeVolumes,
	)

	notifier = notifications.NewNotifier(cmd)
}

// Run is the main execution flow of the command
func Run(c *cobra.Command, names []string) {
	filter := filters.BuildFilter(names, enableLabel)
	runOnce, _ := c.PersistentFlags().GetBool("run-once")
	httpAPI, _ := c.PersistentFlags().GetBool("http-api")

	if httpAPI {
		apiToken, _ := c.PersistentFlags().GetString("http-api-token")

		if err := api.SetupHTTPUpdates(apiToken, func() { runUpdatesWithNotifications(filter) }); err != nil {
			log.Fatal(err)
			os.Exit(1)
		}

		api.WaitForHTTPUpdates()
	}

	if runOnce {
		if noStartupMessage, _ := c.PersistentFlags().GetBool("no-startup-message"); !noStartupMessage {
			log.Info("Running a one time update.")
		}
		runUpdatesWithNotifications(filter)
		os.Exit(0)
		return
	}

	if err := actions.CheckForMultipleWatchtowerInstances(client, cleanup); err != nil {
		log.Fatal(err)
	}

	if err := runUpgradesOnSchedule(c, filter); err != nil {
		log.Error(err)
	}

	os.Exit(1)
}

func runUpgradesOnSchedule(c *cobra.Command, filter t.Filter) error {
	tryLockSem := make(chan bool, 1)
	tryLockSem <- true

	cron := cron.New()
	err := cron.AddFunc(
		scheduleSpec,
		func() {
			select {
			case v := <-tryLockSem:
				defer func() { tryLockSem <- v }()
				runUpdatesWithNotifications(filter)
			default:
				log.Debug("Skipped another update already running.")
			}

			nextRuns := cron.Entries()
			if len(nextRuns) > 0 {
				log.Debug("Scheduled next run: " + nextRuns[0].Next.String())
			}
		})

	if err != nil {
		return err
	}

	if noStartupMessage, _ := c.PersistentFlags().GetBool("no-startup-message"); !noStartupMessage {
		log.Info("Starting Watchtower and scheduling first run: " + cron.Entries()[0].Schedule.Next(time.Now()).String())
	}

	cron.Start()

	// Graceful shut-down on SIGINT/SIGTERM
	interrupt := make(chan os.Signal, 1)
	signal.Notify(interrupt, os.Interrupt)
	signal.Notify(interrupt, syscall.SIGTERM)

	<-interrupt
	cron.Stop()
	log.Info("Waiting for running update to be finished...")
	<-tryLockSem
	return nil
}

func runUpdatesWithNotifications(filter t.Filter) {
	notifier.StartNotification()
	updateParams := t.UpdateParams{
		Filter:         filter,
		Cleanup:        cleanup,
		NoRestart:      noRestart,
		Timeout:        timeout,
		MonitorOnly:    monitorOnly,
		LifecycleHooks: lifecycleHooks,
	}
	err := actions.Update(client, updateParams)
	if err != nil {
		log.Println(err)
	}
	notifier.SendNotification()
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"github.com/qiniu/qshell/v2/docs"
	"github.com/qiniu/qshell/v2/iqshell"
	"github.com/qiniu/qshell/v2/iqshell/common/config"
	"github.com/qiniu/qshell/v2/iqshell/common/data"
	"github.com/qiniu/qshell/v2/iqshell/common/version"
	"github.com/spf13/cobra"
	"os"
)

const (
	bash_completion_func = `__qshell_parse_get()
{
    local qshell_output out
    if qshell_output=$(qshell user ls --name 2>/dev/null); then
        out=($(echo "${qshell_output}"))
        COMPREPLY=( $( compgen -W "${out[*]}" -- "$cur" ) )
    fi
}

__qshell_get_resource()
{
    __qshell_parse_get
    if [[ $? -eq 0 ]]; then
        return 0
    fi
}

__custom_func() {
    case ${last_command} in
        qshell_user_cu)
            __qshell_get_resource
            return
            ;;
        *)
            ;;
    esac
}
`
)

var rootCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
	cmd := &cobra.Command{
		Use:                    "qshell",
		Short:                  "Qiniu commandline tool for managing your bucket and CDN",
		Version:                version.Version(),
		BashCompletionFunction: bash_completion_func,
		Run: func(cmd *cobra.Command, args []string) {
			cfg.CmdCfg.CmdId = docs.QShellType
			docs.ShowCmdDocument(docs.QShellType)
		},
	}
	cmd.PersistentFlags().BoolVarP(&cfg.StdoutColorful, "colorful", "", false, "console colorful mode")
	cmd.PersistentFlags().BoolVarP(&cfg.DebugEnable, "silence", "", false, "silence mode, The console only outputs warnings、errors and some important information")
	cmd.PersistentFlags().BoolVarP(&cfg.DebugEnable, "debug", "d", false, "debug mode")
	// ddebug 开启 client debug
	cmd.PersistentFlags().BoolVarP(&cfg.DDebugEnable, "ddebug", "D", false, "deep debug mode")
	cmd.PersistentFlags().StringVarP(&cfg.ConfigFilePath, "config", "C", "", "set config file (default is $HOME/.qshell.json)")
	cmd.PersistentFlags().BoolVarP(&cfg.Local, "local", "L", false, "use current directory qshell workspace (default is $HOME/.qshell)")
	cmd.PersistentFlags().BoolVarP(&cfg.Document, "doc", "", false, "document of command")
	return cmd
}

func Execute() {
	var cfg = &iqshell.Config{
		Document:       false,
		DebugEnable:    false,
		DDebugEnable:   false,
		ConfigFilePath: "",
		Local:          false,
		CmdCfg: config.Config{
			Log: &config.LogSetting{},
		},
	}

	rootCmd := rootCmdBuilder(cfg)
	load(rootCmd, cfg)

	if err := rootCmd.Execute(); err != nil {
		_, _ = fmt.Fprintf(os.Stderr, "%v\n", err)
		data.SetCmdStatusError()
	}

	if !data.IsTestMode() && data.GetCmdStatus() != data.StatusOK {
		os.Exit(data.GetCmdStatus())
	}
}

type Loader func(superCmd *cobra.Command, cfg *iqshell.Config)

var loaders = make([]Loader, 0, 20)

func registerLoader(l Loader) {
	if l != nil {
		loaders = append(loaders, l)
	}
}

func load(superCmd *cobra.Command, cfg *iqshell.Config) {
	for _, l := range loaders {
		if l != nil {
			l(superCmd, cfg)
		}
	}
}
<commit_msg>change silence<commit_after>package cmd

import (
	"fmt"
	"github.com/qiniu/qshell/v2/docs"
	"github.com/qiniu/qshell/v2/iqshell"
	"github.com/qiniu/qshell/v2/iqshell/common/config"
	"github.com/qiniu/qshell/v2/iqshell/common/data"
	"github.com/qiniu/qshell/v2/iqshell/common/version"
	"github.com/spf13/cobra"
	"os"
)

const (
	bash_completion_func = `__qshell_parse_get()
{
    local qshell_output out
    if qshell_output=$(qshell user ls --name 2>/dev/null); then
        out=($(echo "${qshell_output}"))
        COMPREPLY=( $( compgen -W "${out[*]}" -- "$cur" ) )
    fi
}

__qshell_get_resource()
{
    __qshell_parse_get
    if [[ $? -eq 0 ]]; then
        return 0
    fi
}

__custom_func() {
    case ${last_command} in
        qshell_user_cu)
            __qshell_get_resource
            return
            ;;
        *)
            ;;
    esac
}
`
)

var rootCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
	cmd := &cobra.Command{
		Use:                    "qshell",
		Short:                  "Qiniu commandline tool for managing your bucket and CDN",
		Version:                version.Version(),
		BashCompletionFunction: bash_completion_func,
		Run: func(cmd *cobra.Command, args []string) {
			cfg.CmdCfg.CmdId = docs.QShellType
			docs.ShowCmdDocument(docs.QShellType)
		},
	}
	cmd.PersistentFlags().BoolVarP(&cfg.StdoutColorful, "colorful", "", false, "console colorful mode")
	cmd.PersistentFlags().BoolVarP(&cfg.Silence, "silence", "", false, "silence mode, The console only outputs warnings、errors and some important information")
	cmd.PersistentFlags().BoolVarP(&cfg.DebugEnable, "debug", "d", false, "debug mode")
	// ddebug 开启 client debug
	cmd.PersistentFlags().BoolVarP(&cfg.DDebugEnable, "ddebug", "D", false, "deep debug mode")
	cmd.PersistentFlags().StringVarP(&cfg.ConfigFilePath, "config", "C", "", "set config file (default is $HOME/.qshell.json)")
	cmd.PersistentFlags().BoolVarP(&cfg.Local, "local", "L", false, "use current directory qshell workspace (default is $HOME/.qshell)")
	cmd.PersistentFlags().BoolVarP(&cfg.Document, "doc", "", false, "document of command")
	return cmd
}

func Execute() {
	var cfg = &iqshell.Config{
		Document:       false,
		DebugEnable:    false,
		DDebugEnable:   false,
		ConfigFilePath: "",
		Local:          false,
		CmdCfg: config.Config{
			Log: &config.LogSetting{},
		},
	}

	rootCmd := rootCmdBuilder(cfg)
	load(rootCmd, cfg)

	if err := rootCmd.Execute(); err != nil {
		_, _ = fmt.Fprintf(os.Stderr, "%v\n", err)
		data.SetCmdStatusError()
	}

	if !data.IsTestMode() && data.GetCmdStatus() != data.StatusOK {
		os.Exit(data.GetCmdStatus())
	}
}

type Loader func(superCmd *cobra.Command, cfg *iqshell.Config)

var loaders = make([]Loader, 0, 20)

func registerLoader(l Loader) {
	if l != nil {
		loaders = append(loaders, l)
	}
}

func load(superCmd *cobra.Command, cfg *iqshell.Config) {
	for _, l := range loaders {
		if l != nil {
			l(superCmd, cfg)
		}
	}
}
<|endoftext|>


<commit_before>package daemon

import (
	"fmt"
	"runtime"
	"strings"
	"syscall"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/container"
	"github.com/docker/docker/pkg/signal"
)

type errNoSuchProcess struct {
	pid    int
	signal int
}

func (e errNoSuchProcess) Error() string {
	return fmt.Sprintf("Cannot kill process (pid=%d) with signal %d: no such process.", e.pid, e.signal)
}

// isErrNoSuchProcess returns true if the error
// is an instance of errNoSuchProcess.
func isErrNoSuchProcess(err error) bool {
	_, ok := err.(errNoSuchProcess)
	return ok
}

// ContainerKill sends signal to the container
// If no signal is given (sig 0), then Kill with SIGKILL and wait
// for the container to exit.
// If a signal is given, then just send it to the container and return.
func (daemon *Daemon) ContainerKill(name string, sig uint64) error {
	container, err := daemon.GetContainer(name)
	if err != nil {
		return err
	}

	if sig != 0 && !signal.ValidSignalForPlatform(syscall.Signal(sig)) {
		return fmt.Errorf("The %s daemon does not support signal %d", runtime.GOOS, sig)
	}

	// If no signal is passed, or SIGKILL, perform regular Kill (SIGKILL + wait())
	if sig == 0 || syscall.Signal(sig) == syscall.SIGKILL {
		return daemon.Kill(container)
	}
	return daemon.killWithSignal(container, int(sig))
}

// killWithSignal sends the container the given signal. This wrapper for the
// host specific kill command prepares the container before attempting
// to send the signal. An error is returned if the container is paused
// or not running, or if there is a problem returned from the
// underlying kill command.
func (daemon *Daemon) killWithSignal(container *container.Container, sig int) error {
	logrus.Debugf("Sending %d to %s", sig, container.ID)
	container.Lock()
	defer container.Unlock()

	// We could unpause the container for them rather than returning this error
	if container.Paused {
		return fmt.Errorf("Container %s is paused. Unpause the container before stopping", container.ID)
	}

	if !container.Running {
		return errNotRunning{container.ID}
	}

	container.ExitOnNext()

	if !daemon.IsShuttingDown() {
		container.HasBeenManuallyStopped = true
	}

	// if the container is currently restarting we do not need to send the signal
	// to the process. Telling the monitor that it should exit on its next event
	// loop is enough
	if container.Restarting {
		return nil
	}

	if err := daemon.kill(container, sig); err != nil {
		err = fmt.Errorf("Cannot kill container %s: %s", container.ID, err)
		// if container or process not exists, ignore the error
		if strings.Contains(err.Error(), "container not found") ||
			strings.Contains(err.Error(), "no such process") {
			logrus.Warnf("container kill failed because of 'container not found' or 'no such process': %s", err.Error())
		} else {
			return err
		}
	}

	attributes := map[string]string{
		"signal": fmt.Sprintf("%d", sig),
	}
	daemon.LogContainerEventWithAttributes(container, "kill", attributes)
	return nil
}

// Kill forcefully terminates a container.
func (daemon *Daemon) Kill(container *container.Container) error {
	if !container.IsRunning() {
		return errNotRunning{container.ID}
	}

	// 1. Send SIGKILL
	if err := daemon.killPossiblyDeadProcess(container, int(syscall.SIGKILL)); err != nil {
		// While normally we might "return err" here we're not going to
		// because if we can't stop the container by this point then
		// its probably because its already stopped. Meaning, between
		// the time of the IsRunning() call above and now it stopped.
		// Also, since the err return will be environment specific we can't
		// look for any particular (common) error that would indicate
		// that the process is already dead vs something else going wrong.
		// So, instead we'll give it up to 2 more seconds to complete and if
		// by that time the container is still running, then the error
		// we got is probably valid and so we return it to the caller.
		if isErrNoSuchProcess(err) {
			return nil
		}

		if container.IsRunning() {
			container.WaitStop(2 * time.Second)
			if container.IsRunning() {
				return err
			}
		}
	}

	// 2. Wait for the process to die, in last resort, try to kill the process directly
	if err := killProcessDirectly(container); err != nil {
		if isErrNoSuchProcess(err) {
			return nil
		}
		return err
	}

	container.WaitStop(-1 * time.Second)
	return nil
}

// killPossibleDeadProcess is a wrapper around killSig() suppressing "no such process" error.
func (daemon *Daemon) killPossiblyDeadProcess(container *container.Container, sig int) error {
	err := daemon.killWithSignal(container, sig)
	if err == syscall.ESRCH {
		e := errNoSuchProcess{container.GetPID(), sig}
		logrus.Debug(e)
		return e
	}
	return err
}

func (daemon *Daemon) kill(c *container.Container, sig int) error {
	return daemon.containerd.Signal(c.ID, sig)
}
<commit_msg>make container kill debug log readable<commit_after>package daemon

import (
	"fmt"
	"runtime"
	"strings"
	"syscall"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/container"
	"github.com/docker/docker/pkg/signal"
)

type errNoSuchProcess struct {
	pid    int
	signal int
}

func (e errNoSuchProcess) Error() string {
	return fmt.Sprintf("Cannot kill process (pid=%d) with signal %d: no such process.", e.pid, e.signal)
}

// isErrNoSuchProcess returns true if the error
// is an instance of errNoSuchProcess.
func isErrNoSuchProcess(err error) bool {
	_, ok := err.(errNoSuchProcess)
	return ok
}

// ContainerKill sends signal to the container
// If no signal is given (sig 0), then Kill with SIGKILL and wait
// for the container to exit.
// If a signal is given, then just send it to the container and return.
func (daemon *Daemon) ContainerKill(name string, sig uint64) error {
	container, err := daemon.GetContainer(name)
	if err != nil {
		return err
	}

	if sig != 0 && !signal.ValidSignalForPlatform(syscall.Signal(sig)) {
		return fmt.Errorf("The %s daemon does not support signal %d", runtime.GOOS, sig)
	}

	// If no signal is passed, or SIGKILL, perform regular Kill (SIGKILL + wait())
	if sig == 0 || syscall.Signal(sig) == syscall.SIGKILL {
		return daemon.Kill(container)
	}
	return daemon.killWithSignal(container, int(sig))
}

// killWithSignal sends the container the given signal. This wrapper for the
// host specific kill command prepares the container before attempting
// to send the signal. An error is returned if the container is paused
// or not running, or if there is a problem returned from the
// underlying kill command.
func (daemon *Daemon) killWithSignal(container *container.Container, sig int) error {
	logrus.Debugf("Sending kill signal %d to container %s", sig, container.ID)
	container.Lock()
	defer container.Unlock()

	// We could unpause the container for them rather than returning this error
	if container.Paused {
		return fmt.Errorf("Container %s is paused. Unpause the container before stopping", container.ID)
	}

	if !container.Running {
		return errNotRunning{container.ID}
	}

	container.ExitOnNext()

	if !daemon.IsShuttingDown() {
		container.HasBeenManuallyStopped = true
	}

	// if the container is currently restarting we do not need to send the signal
	// to the process. Telling the monitor that it should exit on its next event
	// loop is enough
	if container.Restarting {
		return nil
	}

	if err := daemon.kill(container, sig); err != nil {
		err = fmt.Errorf("Cannot kill container %s: %s", container.ID, err)
		// if container or process not exists, ignore the error
		if strings.Contains(err.Error(), "container not found") ||
			strings.Contains(err.Error(), "no such process") {
			logrus.Warnf("container kill failed because of 'container not found' or 'no such process': %s", err.Error())
		} else {
			return err
		}
	}

	attributes := map[string]string{
		"signal": fmt.Sprintf("%d", sig),
	}
	daemon.LogContainerEventWithAttributes(container, "kill", attributes)
	return nil
}

// Kill forcefully terminates a container.
func (daemon *Daemon) Kill(container *container.Container) error {
	if !container.IsRunning() {
		return errNotRunning{container.ID}
	}

	// 1. Send SIGKILL
	if err := daemon.killPossiblyDeadProcess(container, int(syscall.SIGKILL)); err != nil {
		// While normally we might "return err" here we're not going to
		// because if we can't stop the container by this point then
		// its probably because its already stopped. Meaning, between
		// the time of the IsRunning() call above and now it stopped.
		// Also, since the err return will be environment specific we can't
		// look for any particular (common) error that would indicate
		// that the process is already dead vs something else going wrong.
		// So, instead we'll give it up to 2 more seconds to complete and if
		// by that time the container is still running, then the error
		// we got is probably valid and so we return it to the caller.
		if isErrNoSuchProcess(err) {
			return nil
		}

		if container.IsRunning() {
			container.WaitStop(2 * time.Second)
			if container.IsRunning() {
				return err
			}
		}
	}

	// 2. Wait for the process to die, in last resort, try to kill the process directly
	if err := killProcessDirectly(container); err != nil {
		if isErrNoSuchProcess(err) {
			return nil
		}
		return err
	}

	container.WaitStop(-1 * time.Second)
	return nil
}

// killPossibleDeadProcess is a wrapper around killSig() suppressing "no such process" error.
func (daemon *Daemon) killPossiblyDeadProcess(container *container.Container, sig int) error {
	err := daemon.killWithSignal(container, sig)
	if err == syscall.ESRCH {
		e := errNoSuchProcess{container.GetPID(), sig}
		logrus.Debug(e)
		return e
	}
	return err
}

func (daemon *Daemon) kill(c *container.Container, sig int) error {
	return daemon.containerd.Signal(c.ID, sig)
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/spf13/cobra"

	"github.com/stormforger/cli/api"
)

var (
	RootCmd = &cobra.Command{
		Use:   "forge",
		Short: "This is the StormForger command line client!",
		Long:  `This is the StormForger command line client!`,
	}

	rootOpts struct {
		APIEndpoint string
	}
)

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}

func NewClient() *api.Client {
	return api.NewClient(rootOpts.APIEndpoint, readJwt())
}

func readJwt() string {
	jwt, _ := ioutil.ReadFile("./.stormforger_jwt")

	return strings.TrimSpace(string(jwt))
}

func init() {
	RootCmd.PersistentFlags().StringVar(&rootOpts.APIEndpoint, "api-endpoint", "https://api.stormforger.com", "API Domain")
}
<commit_msg>fix --api-endpoint description<commit_after>package cmd

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/spf13/cobra"

	"github.com/stormforger/cli/api"
)

var (
	RootCmd = &cobra.Command{
		Use:   "forge",
		Short: "This is the StormForger command line client!",
		Long:  `This is the StormForger command line client!`,
	}

	rootOpts struct {
		APIEndpoint string
	}
)

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}

func NewClient() *api.Client {
	return api.NewClient(rootOpts.APIEndpoint, readJwt())
}

func readJwt() string {
	jwt, _ := ioutil.ReadFile("./.stormforger_jwt")

	return strings.TrimSpace(string(jwt))
}

func init() {
	RootCmd.PersistentFlags().StringVar(&rootOpts.APIEndpoint, "api-endpoint", "https://api.stormforger.com", "API Endpoint")
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"os"

	log "github.com/Sirupsen/logrus"
	"github.com/cozy/cozy-stack/config"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

// ConfigFilename is the default configuration filename that cozy
// search for
const ConfigFilename = "cozy"

// ConfigPaths is the list of directories used to search for a
// configuration file
var ConfigPaths = []string{
	".cozy",
	"$HOME/.cozy",
	"/etc/cozy",
}

// DefaultStorageDirectory is the default directory name in which data
// is stored relatively to the cozy-stack binary.
const DefaultStorageDirectory = "storage"

// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use:   "cozy-stack",
	Short: "cozy-stack is the main command",
	Long: `Cozy is a platform that brings all your web services in the same private space.
With it, your web apps and your devices can share data easily, providing you
with a new experience. You can install Cozy on your own hardware where no one
profiles you.`,
	RunE: func(cmd *cobra.Command, args []string) error {
		if err := Configure(); err != nil {
			return err
		}

		// Display the usage/help by default
		return cmd.Help()
	},
	// Do not display usage on error
	SilenceUsage: true,
}

var cfgFile string

func init() {
	pwd, err := os.Getwd()
	if err != nil {
		panic(err)
	}

	flags := RootCmd.PersistentFlags()
	flags.StringVarP(&cfgFile, "config", "c", "", "configuration file (default \"$HOME/.cozy.yaml\")")

	flags.StringP("mode", "m", config.BuildMode, "server mode: development or production")
	viper.BindPFlag("mode", flags.Lookup("mode"))

	flags.StringP("host", "", "localhost", "server host")
	viper.BindPFlag("host", flags.Lookup("host"))

	flags.IntP("port", "p", 8080, "server port")
	viper.BindPFlag("port", flags.Lookup("port"))

	flags.String("fs-url", fmt.Sprintf("file://localhost%s/%s", pwd, DefaultStorageDirectory), "filesystem url")
	viper.BindPFlag("fs.url", flags.Lookup("fs-url"))

	flags.String("couchdb-host", "localhost", "couchdbdb host")
	viper.BindPFlag("couchdb.host", flags.Lookup("couchdb-host"))

	flags.Int("couchdb-port", 5984, "couchdbdb port")
	viper.BindPFlag("couchdb.port", flags.Lookup("couchdb-port"))

	flags.String("log-level", "info", "define the log level")
	viper.BindPFlag("log.level", flags.Lookup("log-level"))
}

// Configure Viper to read the environment and the optional config file
func Configure() error {
	viper.SetEnvPrefix("cozy")
	viper.AutomaticEnv()

	if cfgFile != "" {
		// Read given config file and skip other paths
		viper.SetConfigFile(cfgFile)
	} else {
		viper.SetConfigName(ConfigFilename)
		for _, cfgPath := range ConfigPaths {
			viper.AddConfigPath(cfgPath)
		}
	}

	if err := viper.ReadInConfig(); err != nil {
		if _, isParseErr := err.(viper.ConfigParseError); isParseErr {
			fmt.Fprintf(os.Stderr, "Error: While reading cozy-stack configurations from %s\n", viper.ConfigFileUsed())
			return err
		}

		if cfgFile != "" {
			return fmt.Errorf("Unable to locate config file: %s\n", cfgFile)
		}
	}

	if viper.ConfigFileUsed() != "" {
		fmt.Println("Using config file:", viper.ConfigFileUsed())
	}

	if err := config.UseViper(viper.GetViper()); err != nil {
		return err
	}

	if err := configureLogger(); err != nil {
		return err
	}

	return nil
}

func configureLogger() error {
	loggerCfg := config.GetConfig().Logger

	logLevel, err := log.ParseLevel(loggerCfg.Level)
	if err != nil {
		return err
	}

	log.SetLevel(logLevel)
	return nil
}
<commit_msg>Make ./storage relative to binary directory instead of pwd<commit_after>package cmd

import (
	"fmt"
	"os"
	"path/filepath"

	log "github.com/Sirupsen/logrus"
	"github.com/cozy/cozy-stack/config"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

// ConfigFilename is the default configuration filename that cozy
// search for
const ConfigFilename = "cozy"

// ConfigPaths is the list of directories used to search for a
// configuration file
var ConfigPaths = []string{
	".cozy",
	"$HOME/.cozy",
	"/etc/cozy",
}

// DefaultStorageDirectory is the default directory name in which data
// is stored relatively to the cozy-stack binary.
const DefaultStorageDirectory = "storage"

// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use:   "cozy-stack",
	Short: "cozy-stack is the main command",
	Long: `Cozy is a platform that brings all your web services in the same private space.
With it, your web apps and your devices can share data easily, providing you
with a new experience. You can install Cozy on your own hardware where no one
profiles you.`,
	RunE: func(cmd *cobra.Command, args []string) error {
		if err := Configure(); err != nil {
			return err
		}

		// Display the usage/help by default
		return cmd.Help()
	},
	// Do not display usage on error
	SilenceUsage: true,
}

var cfgFile string

func init() {
	binDir, err := filepath.Abs(filepath.Dir(os.Args[0]))
	if err != nil {
		panic(err)
	}

	flags := RootCmd.PersistentFlags()
	flags.StringVarP(&cfgFile, "config", "c", "", "configuration file (default \"$HOME/.cozy.yaml\")")

	flags.StringP("mode", "m", config.BuildMode, "server mode: development or production")
	viper.BindPFlag("mode", flags.Lookup("mode"))

	flags.StringP("host", "", "localhost", "server host")
	viper.BindPFlag("host", flags.Lookup("host"))

	flags.IntP("port", "p", 8080, "server port")
	viper.BindPFlag("port", flags.Lookup("port"))

	flags.String("fs-url", fmt.Sprintf("file://localhost%s/%s", binDir, DefaultStorageDirectory), "filesystem url")
	viper.BindPFlag("fs.url", flags.Lookup("fs-url"))

	flags.String("couchdb-host", "localhost", "couchdbdb host")
	viper.BindPFlag("couchdb.host", flags.Lookup("couchdb-host"))

	flags.Int("couchdb-port", 5984, "couchdbdb port")
	viper.BindPFlag("couchdb.port", flags.Lookup("couchdb-port"))

	flags.String("log-level", "info", "define the log level")
	viper.BindPFlag("log.level", flags.Lookup("log-level"))
}

// Configure Viper to read the environment and the optional config file
func Configure() error {
	viper.SetEnvPrefix("cozy")
	viper.AutomaticEnv()

	if cfgFile != "" {
		// Read given config file and skip other paths
		viper.SetConfigFile(cfgFile)
	} else {
		viper.SetConfigName(ConfigFilename)
		for _, cfgPath := range ConfigPaths {
			viper.AddConfigPath(cfgPath)
		}
	}

	if err := viper.ReadInConfig(); err != nil {
		if _, isParseErr := err.(viper.ConfigParseError); isParseErr {
			fmt.Fprintf(os.Stderr, "Error: While reading cozy-stack configurations from %s\n", viper.ConfigFileUsed())
			return err
		}

		if cfgFile != "" {
			return fmt.Errorf("Unable to locate config file: %s\n", cfgFile)
		}
	}

	if viper.ConfigFileUsed() != "" {
		fmt.Println("Using config file:", viper.ConfigFileUsed())
	}

	if err := config.UseViper(viper.GetViper()); err != nil {
		return err
	}

	if err := configureLogger(); err != nil {
		return err
	}

	return nil
}

func configureLogger() error {
	loggerCfg := config.GetConfig().Logger

	logLevel, err := log.ParseLevel(loggerCfg.Level)
	if err != nil {
		return err
	}

	log.SetLevel(logLevel)
	return nil
}
<|endoftext|>


<commit_before>// Copyright © 2017 Aqua Security Software Ltd. <info@aquasec.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"fmt"
	"os"

	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

var (
	cfgDir  = "./cfg"
	cfgFile string

	jsonFmt       bool
	checkList     string
	groupList     string
	masterFile    string
	nodeFile      string
	federatedFile string

	kubeConfDir    string
	etcdConfDir    string
	flannelConfDir string
)

// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use:   os.Args[0],
	Short: "Run CIS Benchmarks checks against a Kubernetes deployment",
	Long:  `This tool runs the CIS Kubernetes 1.6 Benchmark v1.0.0 checks.`,
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}

func init() {
	cobra.OnInitialize(initConfig)

	RootCmd.PersistentFlags().BoolVar(&jsonFmt, "json", false, "Output results as JSON")
	RootCmd.PersistentFlags().StringVarP(&checkList,
		"check",
		"c",
		"",
		`A comma-delimited list of checks to run as specified in CIS document. Example --check="1.1.1,1.1.2"`,
	)
	RootCmd.PersistentFlags().StringVarP(&groupList,
		"group",
		"g",
		"",
		`Run all the checks under this comma-delimited list of groups. Example --group="1.1"`,
	)

}

// initConfig reads in config file and ENV variables if set.
func initConfig() {
	if cfgFile != "" { // enable ability to specify config file via flag
		viper.SetConfigFile(cfgFile)
	}

	viper.SetConfigName("config") // name of config file (without extension)
	viper.AddConfigPath(cfgDir)   // adding home directory as first search path

	viper.SetEnvPrefix("CISK8S")
	viper.AutomaticEnv() // read in environment variables that match

	// Set defaults
	viper.SetDefault("kubeConfDir", "/etc/kubernetes")
	viper.SetDefault("etcdConfDir", "/etc/etcd")
	viper.SetDefault("flanneldConfDir", "/etc/sysconfig")

	viper.SetDefault("masterFile", cfgDir+"/master.yaml")
	viper.SetDefault("nodeFile", cfgDir+"/node.yaml")
	viper.SetDefault("federatedFile", cfgDir+"/federated.yaml")

	// If a config file is found, read it in.
	if err := viper.ReadInConfig(); err == nil {
		fmt.Println("Using config file:", viper.ConfigFileUsed())
	}
}
<commit_msg>Use colorPrint for config file info too<commit_after>// Copyright © 2017 Aqua Security Software Ltd. <info@aquasec.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"fmt"
	"os"

	"github.com/aquasecurity/kube-bench/check"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

var (
	cfgDir  = "./cfg"
	cfgFile string

	jsonFmt       bool
	checkList     string
	groupList     string
	masterFile    string
	nodeFile      string
	federatedFile string

	kubeConfDir    string
	etcdConfDir    string
	flannelConfDir string
)

// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use:   os.Args[0],
	Short: "Run CIS Benchmarks checks against a Kubernetes deployment",
	Long:  `This tool runs the CIS Kubernetes 1.6 Benchmark v1.0.0 checks.`,
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}

func init() {
	cobra.OnInitialize(initConfig)

	RootCmd.PersistentFlags().BoolVar(&jsonFmt, "json", false, "Output results as JSON")
	RootCmd.PersistentFlags().StringVarP(&checkList,
		"check",
		"c",
		"",
		`A comma-delimited list of checks to run as specified in CIS document. Example --check="1.1.1,1.1.2"`,
	)
	RootCmd.PersistentFlags().StringVarP(&groupList,
		"group",
		"g",
		"",
		`Run all the checks under this comma-delimited list of groups. Example --group="1.1"`,
	)

}

// initConfig reads in config file and ENV variables if set.
func initConfig() {
	if cfgFile != "" { // enable ability to specify config file via flag
		viper.SetConfigFile(cfgFile)
	}

	viper.SetConfigName("config") // name of config file (without extension)
	viper.AddConfigPath(cfgDir)   // adding home directory as first search path

	viper.SetEnvPrefix("CISK8S")
	viper.AutomaticEnv() // read in environment variables that match

	// Set defaults
	viper.SetDefault("kubeConfDir", "/etc/kubernetes")
	viper.SetDefault("etcdConfDir", "/etc/etcd")
	viper.SetDefault("flanneldConfDir", "/etc/sysconfig")

	viper.SetDefault("masterFile", cfgDir+"/master.yaml")
	viper.SetDefault("nodeFile", cfgDir+"/node.yaml")
	viper.SetDefault("federatedFile", cfgDir+"/federated.yaml")

	// If a config file is found, read it in.
	if err := viper.ReadInConfig(); err == nil {
		colorPrint(check.INFO, fmt.Sprintf("Using config file: %s\n", viper.ConfigFileUsed()))
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"os"
	"os/exec"
	"regexp"
	"strings"

	"github.com/aquasecurity/kube-bench/check"
	"github.com/fatih/color"
	"github.com/golang/glog"
	"github.com/spf13/viper"
)

var (
	// Print colors
	colors = map[check.State]*color.Color{
		check.PASS: color.New(color.FgGreen),
		check.FAIL: color.New(color.FgRed),
		check.WARN: color.New(color.FgYellow),
		check.INFO: color.New(color.FgBlue),
	}
)

var psFunc func(string) string
var statFunc func(string) (os.FileInfo, error)

func init() {
	psFunc = ps
	statFunc = os.Stat
}

func printlnWarn(msg string) {
	fmt.Fprintf(os.Stderr, "[%s] %s\n",
		colors[check.WARN].Sprintf("%s", check.WARN),
		msg,
	)
}

func sprintlnWarn(msg string) string {
	return fmt.Sprintf("[%s] %s",
		colors[check.WARN].Sprintf("%s", check.WARN),
		msg,
	)
}

func exitWithError(err error) {
	fmt.Fprintf(os.Stderr, "\n%v\n", err)
	os.Exit(1)
}

func continueWithError(err error, msg string) string {
	if err != nil {
		glog.V(2).Info(err)
	}

	if msg != "" {
		fmt.Fprintf(os.Stderr, "%s\n", msg)
	}

	return ""
}

func cleanIDs(list string) []string {
	list = strings.Trim(list, ",")
	ids := strings.Split(list, ",")

	for _, id := range ids {
		id = strings.Trim(id, " ")
	}

	return ids
}

// ps execs out to the ps command; it's separated into a function so we can write tests
func ps(proc string) string {
	cmd := exec.Command("ps", "-C", proc, "-o", "cmd", "--no-headers")
	out, err := cmd.Output()
	if err != nil {
		continueWithError(fmt.Errorf("%s: %s", cmd.Args, err), "")
	}

	return string(out)
}

// getBinaries finds which of the set of candidate executables are running
func getBinaries(v *viper.Viper) map[string]string {
	binmap := make(map[string]string)

	for _, component := range v.GetStringSlice("components") {
		s := v.Sub(component)
		if s == nil {
			continue
		}

		optional := s.GetBool("optional")
		bins := s.GetStringSlice("bins")
		if len(bins) > 0 {
			bin, err := findExecutable(bins)
			if err != nil && !optional {
				exitWithError(fmt.Errorf("need %s executable but none of the candidates are running", component))
			}

			// Default the executable name that we'll substitute to the name of the component
			if bin == "" {
				bin = component
				glog.V(2).Info(fmt.Sprintf("Component %s not running", component))
			} else {
				glog.V(2).Info(fmt.Sprintf("Component %s uses running binary %s", component, bin))
			}
			binmap[component] = bin
		}
	}

	return binmap
}

// getConfigFiles finds which of the set of candidate config files exist
// accepts a string 't' which indicates the type of config file, conf,
// podspec or untifile.
func getConfigFiles(v *viper.Viper) map[string]string {
	confmap := make(map[string]string)

	for _, component := range v.GetStringSlice("components") {
		s := v.Sub(component)
		if s == nil {
			continue
		}

		// See if any of the candidate config files exist
		conf := findConfigFile(s.GetStringSlice("confs"))
		if conf == "" {
			if s.IsSet("defaultconf") {
				conf = s.GetString("defaultconf")
				glog.V(2).Info(fmt.Sprintf("Using default config file name '%s' for component %s", conf, component))
			} else {
				// Default the config file name that we'll substitute to the name of the component
				glog.V(2).Info(fmt.Sprintf("Missing config file for %s", component))
				conf = component
			}
		} else {
			glog.V(2).Info(fmt.Sprintf("Component %s uses config file '%s'", component, conf))
		}

		confmap[component] = conf
	}

	return confmap
}

// verifyBin checks that the binary specified is running
func verifyBin(bin string) bool {

	// Strip any quotes
	bin = strings.Trim(bin, "'\"")

	// bin could consist of more than one word
	// We'll search for running processes with the first word, and then check the whole
	// proc as supplied is included in the results
	proc := strings.Fields(bin)[0]
	out := psFunc(proc)

	// There could be multiple lines in the ps output
	// The binary needs to be the first word in the ps output, except that it could be preceded by a path
	// e.g. /usr/bin/kubelet is a match for kubelet
	// but apiserver is not a match for kube-apiserver
	reFirstWord := regexp.MustCompile(`^(\S*\/)*` + bin)
	lines := strings.Split(out, "\n")
	for _, l := range lines {
		if reFirstWord.Match([]byte(l)) {
			return true
		}
	}

	return false
}

// fundConfigFile looks through a list of possible config files and finds the first one that exists
func findConfigFile(candidates []string) string {
	for _, c := range candidates {
		_, err := statFunc(c)
		if err == nil {
			return c
		}
		if !os.IsNotExist(err) {
			exitWithError(fmt.Errorf("error looking for file %s: %v", c, err))
		}
	}

	return ""
}

// findExecutable looks through a list of possible executable names and finds the first one that's running
func findExecutable(candidates []string) (string, error) {
	for _, c := range candidates {
		if verifyBin(c) {
			return c, nil
		}
		glog.V(1).Info(fmt.Sprintf("executable '%s' not running", c))
	}

	return "", fmt.Errorf("no candidates running")
}

func multiWordReplace(s string, subname string, sub string) string {
	f := strings.Fields(sub)
	if len(f) > 1 {
		sub = "'" + sub + "'"
	}

	return strings.Replace(s, subname, sub, -1)
}

func getKubeVersion() string {
	// These executables might not be on the user's path.
	_, err := exec.LookPath("kubectl")
	if err != nil {
		exitWithError(fmt.Errorf("kubernetes version check failed: %v", err))
	}

	cmd := exec.Command("kubectl", "version", "--short")
	out, err := cmd.CombinedOutput()
	if err != nil {
		continueWithError(fmt.Errorf("%s", out), "")
	}

	return getVersionFromKubectlOutput(string(out))
}

func getVersionFromKubectlOutput(s string) string {
	serverVersionRe := regexp.MustCompile(`Server Version: v(\d+.\d+)`)
	subs := serverVersionRe.FindStringSubmatch(s)
	if len(subs) < 2 {
		printlnWarn(fmt.Sprintf("Unable to get kubectl version, using default version: %s", defaultKubeVersion))
		return defaultKubeVersion
	}
	return subs[1]
}

func makeSubstitutions(s string, ext string, m map[string]string) string {
	for k, v := range m {
		subst := "$" + k + ext
		if v == "" {
			glog.V(2).Info(fmt.Sprintf("No subsitution for '%s'\n", subst))
			continue
		}
		glog.V(1).Info(fmt.Sprintf("Substituting %s with '%s'\n", subst, v))
		s = multiWordReplace(s, subst, v)
	}

	return s
}
<commit_msg>Try to use kubelet when kubectl is unavailable<commit_after>package cmd

import (
	"fmt"
	"os"
	"os/exec"
	"regexp"
	"strings"

	"github.com/aquasecurity/kube-bench/check"
	"github.com/fatih/color"
	"github.com/golang/glog"
	"github.com/spf13/viper"
)

var (
	// Print colors
	colors = map[check.State]*color.Color{
		check.PASS: color.New(color.FgGreen),
		check.FAIL: color.New(color.FgRed),
		check.WARN: color.New(color.FgYellow),
		check.INFO: color.New(color.FgBlue),
	}
)

var psFunc func(string) string
var statFunc func(string) (os.FileInfo, error)

func init() {
	psFunc = ps
	statFunc = os.Stat
}

func printlnWarn(msg string) {
	fmt.Fprintf(os.Stderr, "[%s] %s\n",
		colors[check.WARN].Sprintf("%s", check.WARN),
		msg,
	)
}

func sprintlnWarn(msg string) string {
	return fmt.Sprintf("[%s] %s",
		colors[check.WARN].Sprintf("%s", check.WARN),
		msg,
	)
}

func exitWithError(err error) {
	fmt.Fprintf(os.Stderr, "\n%v\n", err)
	os.Exit(1)
}

func continueWithError(err error, msg string) string {
	if err != nil {
		glog.V(2).Info(err)
	}

	if msg != "" {
		fmt.Fprintf(os.Stderr, "%s\n", msg)
	}

	return ""
}

func cleanIDs(list string) []string {
	list = strings.Trim(list, ",")
	ids := strings.Split(list, ",")

	for _, id := range ids {
		id = strings.Trim(id, " ")
	}

	return ids
}

// ps execs out to the ps command; it's separated into a function so we can write tests
func ps(proc string) string {
	cmd := exec.Command("ps", "-C", proc, "-o", "cmd", "--no-headers")
	out, err := cmd.Output()
	if err != nil {
		continueWithError(fmt.Errorf("%s: %s", cmd.Args, err), "")
	}

	return string(out)
}

// getBinaries finds which of the set of candidate executables are running
func getBinaries(v *viper.Viper) map[string]string {
	binmap := make(map[string]string)

	for _, component := range v.GetStringSlice("components") {
		s := v.Sub(component)
		if s == nil {
			continue
		}

		optional := s.GetBool("optional")
		bins := s.GetStringSlice("bins")
		if len(bins) > 0 {
			bin, err := findExecutable(bins)
			if err != nil && !optional {
				exitWithError(fmt.Errorf("need %s executable but none of the candidates are running", component))
			}

			// Default the executable name that we'll substitute to the name of the component
			if bin == "" {
				bin = component
				glog.V(2).Info(fmt.Sprintf("Component %s not running", component))
			} else {
				glog.V(2).Info(fmt.Sprintf("Component %s uses running binary %s", component, bin))
			}
			binmap[component] = bin
		}
	}

	return binmap
}

// getConfigFiles finds which of the set of candidate config files exist
// accepts a string 't' which indicates the type of config file, conf,
// podspec or untifile.
func getConfigFiles(v *viper.Viper) map[string]string {
	confmap := make(map[string]string)

	for _, component := range v.GetStringSlice("components") {
		s := v.Sub(component)
		if s == nil {
			continue
		}

		// See if any of the candidate config files exist
		conf := findConfigFile(s.GetStringSlice("confs"))
		if conf == "" {
			if s.IsSet("defaultconf") {
				conf = s.GetString("defaultconf")
				glog.V(2).Info(fmt.Sprintf("Using default config file name '%s' for component %s", conf, component))
			} else {
				// Default the config file name that we'll substitute to the name of the component
				glog.V(2).Info(fmt.Sprintf("Missing config file for %s", component))
				conf = component
			}
		} else {
			glog.V(2).Info(fmt.Sprintf("Component %s uses config file '%s'", component, conf))
		}

		confmap[component] = conf
	}

	return confmap
}

// verifyBin checks that the binary specified is running
func verifyBin(bin string) bool {

	// Strip any quotes
	bin = strings.Trim(bin, "'\"")

	// bin could consist of more than one word
	// We'll search for running processes with the first word, and then check the whole
	// proc as supplied is included in the results
	proc := strings.Fields(bin)[0]
	out := psFunc(proc)

	// There could be multiple lines in the ps output
	// The binary needs to be the first word in the ps output, except that it could be preceded by a path
	// e.g. /usr/bin/kubelet is a match for kubelet
	// but apiserver is not a match for kube-apiserver
	reFirstWord := regexp.MustCompile(`^(\S*\/)*` + bin)
	lines := strings.Split(out, "\n")
	for _, l := range lines {
		if reFirstWord.Match([]byte(l)) {
			return true
		}
	}

	return false
}

// fundConfigFile looks through a list of possible config files and finds the first one that exists
func findConfigFile(candidates []string) string {
	for _, c := range candidates {
		_, err := statFunc(c)
		if err == nil {
			return c
		}
		if !os.IsNotExist(err) {
			exitWithError(fmt.Errorf("error looking for file %s: %v", c, err))
		}
	}

	return ""
}

// findExecutable looks through a list of possible executable names and finds the first one that's running
func findExecutable(candidates []string) (string, error) {
	for _, c := range candidates {
		if verifyBin(c) {
			return c, nil
		}
		glog.V(1).Info(fmt.Sprintf("executable '%s' not running", c))
	}

	return "", fmt.Errorf("no candidates running")
}

func multiWordReplace(s string, subname string, sub string) string {
	f := strings.Fields(sub)
	if len(f) > 1 {
		sub = "'" + sub + "'"
	}

	return strings.Replace(s, subname, sub, -1)
}

func getKubeVersion() string {
	// These executables might not be on the user's path.
	_, err := exec.LookPath("kubectl")

	if err != nil {
		_, err = exec.LookPath("kubelet")
		if err != nil {
			exitWithError(fmt.Errorf("Version check failed: need kubectl or kubelet binaries to get kubernetes version"))
		}
		return getKubeVersionFromKubelet()
	}

	return getKubeVersionFromKubectl()
}

func getKubeVersionFromKubectl() string {
	cmd := exec.Command("kubectl", "version", "--short")
	out, err := cmd.CombinedOutput()
	if err != nil {
		continueWithError(fmt.Errorf("%s", out), "")
	}

	return getVersionFromKubectlOutput(string(out))
}

func getKubeVersionFromKubelet() string {
	cmd := exec.Command("kubelet", "--version")
	out, err := cmd.CombinedOutput()

	if err != nil {
		continueWithError(fmt.Errorf("%s", out), "")
	}

	return getVersionFromKubeletOutput(string(out))
}

func getVersionFromKubectlOutput(s string) string {
	serverVersionRe := regexp.MustCompile(`Server Version: v(\d+.\d+)`)
	subs := serverVersionRe.FindStringSubmatch(s)
	if len(subs) < 2 {
		printlnWarn(fmt.Sprintf("Unable to get kubectl version, using default version: %s", defaultKubeVersion))
		return defaultKubeVersion
	}
	return subs[1]
}

func getVersionFromKubeletOutput(s string) string {
	serverVersionRe := regexp.MustCompile(`Kubernetes v(\d+.\d+)`)
	subs := serverVersionRe.FindStringSubmatch(s)
	if len(subs) < 2 {
		printlnWarn(fmt.Sprintf("Unable to get kubelet version, using default version: %s", defaultKubeVersion))
		return defaultKubeVersion
	}
	return subs[1]
}

func makeSubstitutions(s string, ext string, m map[string]string) string {
	for k, v := range m {
		subst := "$" + k + ext
		if v == "" {
			glog.V(2).Info(fmt.Sprintf("No subsitution for '%s'\n", subst))
			continue
		}
		glog.V(1).Info(fmt.Sprintf("Substituting %s with '%s'\n", subst, v))
		s = multiWordReplace(s, subst, v)
	}

	return s
}
<|endoftext|>


<commit_before>// bucket.go

package base

import (
	"log"
	"strings"
	"sync"

	"github.com/couchbaselabs/go-couchbase"
	"github.com/couchbaselabs/walrus"
)

type Bucket walrus.Bucket

// Implementation of walrus.Bucket that talks to a Couchbase server
type couchbaseBucket struct {
	*couchbase.Bucket
}

func (bucket couchbaseBucket) GetName() string {
	return bucket.Name
}

func (bucket couchbaseBucket) Update(k string, exp int, callback walrus.UpdateFunc) error {
	return bucket.Bucket.Update(k, exp, couchbase.UpdateFunc(callback))
}

func (bucket couchbaseBucket) View(ddoc, name string, params map[string]interface{}) (walrus.ViewResult, error) {
	vres := walrus.ViewResult{}
	return vres, bucket.Bucket.ViewCustom(ddoc, name, params, &vres)
}

// Creates a Bucket that talks to a real live Couchbase server.
func GetCouchbaseBucket(couchbaseURL, poolName, bucketName string) (bucket Bucket, err error) {
	cbbucket, err := couchbase.GetBucket(couchbaseURL, poolName, bucketName)
	if err == nil {
		bucket = couchbaseBucket{cbbucket}
	}
	return
}

var buckets map[[3]string]Bucket
var bucketsLock sync.Mutex

func GetWalrusBucket(couchbaseURL, poolName, bucketName string) (Bucket, error) {
	bucketsLock.Lock()
	defer bucketsLock.Unlock()

	if buckets == nil {
		buckets = make(map[[3]string]Bucket)
	}
	key := [3]string{couchbaseURL, poolName, bucketName}
	bucket := buckets[key]
	if bucket == nil {
		bucket = walrus.NewBucket(bucketName)
		buckets[key] = bucket
		log.Printf("Created Walrus bucket %+v", key)
	}
	return bucket, nil
}

func GetBucket(url, poolName, bucketName string) (bucket Bucket, err error) {
	if strings.HasPrefix(url, "walrus:") {
		return GetWalrusBucket(url, poolName, bucketName)
	}
	return GetCouchbaseBucket(url, poolName, bucketName)
}
<commit_msg>Use walrus.GetBucket instead of walrus.NewBucket<commit_after>// bucket.go

package base

import (
	"strings"

	"github.com/couchbaselabs/go-couchbase"
	"github.com/couchbaselabs/walrus"
)

type Bucket walrus.Bucket

// Implementation of walrus.Bucket that talks to a Couchbase server
type couchbaseBucket struct {
	*couchbase.Bucket
}

func (bucket couchbaseBucket) GetName() string {
	return bucket.Name
}

func (bucket couchbaseBucket) Update(k string, exp int, callback walrus.UpdateFunc) error {
	return bucket.Bucket.Update(k, exp, couchbase.UpdateFunc(callback))
}

func (bucket couchbaseBucket) View(ddoc, name string, params map[string]interface{}) (walrus.ViewResult, error) {
	vres := walrus.ViewResult{}
	return vres, bucket.Bucket.ViewCustom(ddoc, name, params, &vres)
}

// Creates a Bucket that talks to a real live Couchbase server.
func GetCouchbaseBucket(couchbaseURL, poolName, bucketName string) (bucket Bucket, err error) {
	cbbucket, err := couchbase.GetBucket(couchbaseURL, poolName, bucketName)
	if err == nil {
		bucket = couchbaseBucket{cbbucket}
	}
	return
}

func GetBucket(url, poolName, bucketName string) (bucket Bucket, err error) {
	if strings.HasPrefix(url, "walrus:") {
		return walrus.GetBucket(url, poolName, bucketName)
	}
	return GetCouchbaseBucket(url, poolName, bucketName)
}
<|endoftext|>


<commit_before>// Package batch provides tools for various batch processing actions,
// such as reprocessing a month or day or data, and handling deduplication.
package batch

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"sync"
	"time"

	"cloud.google.com/go/storage"
	"google.golang.org/api/iterator"
)

// HTTPClientIntf defines interface for fake injection.
type HTTPClientIntf interface {
	Get(url string) (resp *http.Response, err error)
}

// This is used to intercept Get requests to the queue_pusher when invoked
// with -dry_run.
type dryRunHTTP struct{}

func (dr *dryRunHTTP) Get(url string) (resp *http.Response, err error) {
	resp = &http.Response{}
	resp.Body = ioutil.NopCloser(bytes.NewReader([]byte{}))
	resp.Status = "200 OK"
	resp.StatusCode = 200
	return
}

// A Queuer provides the environment for queuing reprocessing requests.
type Queuer struct {
	Project    string                // project containing task queue
	QueueBase  string                // task queue base name
	NumQueues  int                   // number of task queues.
	BucketName string                // name of bucket containing task files
	Bucket     *storage.BucketHandle // bucket handle
	HTTPClient HTTPClientIntf        // Client to be used for http requests.
}

// NewQueuer creates a Queuer struct from provided parameters.
func NewQueuer(altHTTP HTTPClientIntf, queueBase string, numQueues int, project, bucketName string, dryRun bool) (Queuer, error) {
	var httpClient HTTPClientIntf
	if altHTTP != nil {
		httpClient = altHTTP
	} else if dryRun {
		httpClient = &dryRunHTTP{}
	} else {
		httpClient = http.DefaultClient
	}

	storageClient, err := storage.NewClient(context.Background())
	if err != nil {
		log.Println(err)
		panic(err)
	}

	bucket := storageClient.Bucket(bucketName)
	// Check that the bucket is valid, by fetching it's attributes.
	_, err = bucket.Attrs(context.Background())
	if err != nil {
		return Queuer{}, err
	}

	return Queuer{project, queueBase, numQueues, bucketName, bucket, httpClient}, nil
}

// Initially this used a hash, but using day ordinal is better
// as it distributes across the queues more evenly.
func (q Queuer) queueForDate(date time.Time) string {
	day := date.Unix() / (24 * 60 * 60)
	return fmt.Sprintf("%s%d", q.QueueBase, int(day)%q.NumQueues)
}

// postOne sends a single https request to the queue pusher to add a task.
// Iff dryRun is true, this does nothing.
// TODO - move retry into this method.
// TODO - should use AddMulti - should be much faster.
//   however - be careful not to exceed quotas
func (q Queuer) postOneTask(queue, fn string) error {
	reqStr := fmt.Sprintf("https://queue-pusher-dot-%s.appspot.com/receiver?queue=%s&filename=gs://%s/%s", q.Project, queue, q.BucketName, fn)

	resp, err := q.HTTPClient.Get(reqStr)
	if err != nil {
		// TODO - we don't see errors here or below when the queue doesn't exist.
		// That seems bad.
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return errors.New("http error: " + resp.Status)
	}

	return nil
}

// postOneDay posts all items in an ObjectIterator into the appropriate queue.
func (q Queuer) postOneDay(wg *sync.WaitGroup, queue string, it *storage.ObjectIterator) {
	qpErrCount := 0
	gcsErrCount := 0
	fileCount := 0
	if wg != nil {
		defer wg.Done()
	}
	for o, err := it.Next(); err != iterator.Done; o, err = it.Next() {
		if err != nil {
			// TODO - should this retry?
			log.Println(err, "on it.Next")
			gcsErrCount++
			if gcsErrCount > 3 {
				log.Printf("Failed after %d files to %s.\n", fileCount, queue)
				return
			}
			continue
		}

		err = q.postOneTask(queue, o.Name)
		if err != nil {
			// TODO - should this retry?
			log.Println(err, o.Name, "Retrying")
			// Retry
			time.Sleep(10 * time.Second)
			err = q.postOneTask(queue, o.Name)

			if err != nil {
				log.Println(err, o.Name, "FAILED")
				qpErrCount++
				if qpErrCount > 3 {
					log.Printf("Failed after %d files to %s (on %s).\n", fileCount, queue, o.Name)
					return
				}
			}
		} else {
			fileCount++
		}
	}
	log.Println("Added ", fileCount, " tasks to ", queue)
}

// PostDay fetches an iterator over the objects with ndt/YYYY/MM/DD prefix,
// and passes the iterator to postDay with appropriate queue.
// Iff wq is not nil, PostDay will call done on wg when finished
// posting.
// This typically takes about 10 minutes, whether single or parallel days.
func (q Queuer) PostDay(wg *sync.WaitGroup, prefix string) {
	date, err := time.Parse("ndt/2006/01/02/", prefix)
	if err != nil {
		log.Println("Failed parsing date from ", prefix)
		log.Println(err)
		if wg != nil {
			wg.Done()
		}
		return
	}
	queue := q.queueForDate(date)
	log.Println("Adding ", prefix, " to ", queue)
	qry := storage.Query{
		Delimiter: "/",
		Prefix:    prefix,
	}
	// TODO - can this error?  Or do errors only occur on iterator ops?
	it := q.Bucket.Objects(context.Background(), &qry)
	if wg != nil {
		go q.postOneDay(wg, queue, it)
	} else {
		q.postOneDay(nil, queue, it)
	}
}

// PostMonth adds all of the files from dates within a specified month.
// It is difficult to test without hitting GCS.  8-(
// This typically takes about 10 minutes, processing all days concurrently.
func (q Queuer) PostMonth(prefix string) {
	qry := storage.Query{
		Delimiter: "/",
		// TODO - validate.
		Prefix: prefix,
	}
	it := q.Bucket.Objects(context.Background(), &qry)

	var wg sync.WaitGroup
	for o, err := it.Next(); err != iterator.Done; o, err = it.Next() {
		if err != nil {
			log.Println(err)
		} else if o.Prefix != "" {
			wg.Add(1)
			q.PostDay(&wg, o.Prefix)
		} else {
			log.Println("Skipping: ", o.Name)
		}
	}
	wg.Wait()
}
<commit_msg>avoid check if dryrun<commit_after>// Package batch provides tools for various batch processing actions,
// such as reprocessing a month or day or data, and handling deduplication.
package batch

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"sync"
	"time"

	"cloud.google.com/go/storage"
	"google.golang.org/api/iterator"
)

// HTTPClientIntf defines interface for fake injection.
type HTTPClientIntf interface {
	Get(url string) (resp *http.Response, err error)
}

// This is used to intercept Get requests to the queue_pusher when invoked
// with -dry_run.
type dryRunHTTP struct{}

func (dr *dryRunHTTP) Get(url string) (resp *http.Response, err error) {
	resp = &http.Response{}
	resp.Body = ioutil.NopCloser(bytes.NewReader([]byte{}))
	resp.Status = "200 OK"
	resp.StatusCode = 200
	return
}

// A Queuer provides the environment for queuing reprocessing requests.
type Queuer struct {
	Project    string                // project containing task queue
	QueueBase  string                // task queue base name
	NumQueues  int                   // number of task queues.
	BucketName string                // name of bucket containing task files
	Bucket     *storage.BucketHandle // bucket handle
	HTTPClient HTTPClientIntf        // Client to be used for http requests.
}

// NewQueuer creates a Queuer struct from provided parameters.
func NewQueuer(altHTTP HTTPClientIntf, queueBase string, numQueues int, project, bucketName string, dryRun bool) (Queuer, error) {
	var httpClient HTTPClientIntf
	if altHTTP != nil {
		httpClient = altHTTP
	} else if dryRun {
		httpClient = &dryRunHTTP{}
	} else {
		httpClient = http.DefaultClient
	}

	storageClient, err := storage.NewClient(context.Background())
	if err != nil {
		log.Println(err)
		panic(err)
	}

	bucket := storageClient.Bucket(bucketName)
	// Check that the bucket is valid, by fetching it's attributes.
	// Bypass check if we are running travis tests.
	if !dryRun {
		_, err = bucket.Attrs(context.Background())
		if err != nil {
			return Queuer{}, err
		}

	}

	return Queuer{project, queueBase, numQueues, bucketName, bucket, httpClient}, nil
}

// Initially this used a hash, but using day ordinal is better
// as it distributes across the queues more evenly.
func (q Queuer) queueForDate(date time.Time) string {
	day := date.Unix() / (24 * 60 * 60)
	return fmt.Sprintf("%s%d", q.QueueBase, int(day)%q.NumQueues)
}

// postOne sends a single https request to the queue pusher to add a task.
// Iff dryRun is true, this does nothing.
// TODO - move retry into this method.
// TODO - should use AddMulti - should be much faster.
//   however - be careful not to exceed quotas
func (q Queuer) postOneTask(queue, fn string) error {
	reqStr := fmt.Sprintf("https://queue-pusher-dot-%s.appspot.com/receiver?queue=%s&filename=gs://%s/%s", q.Project, queue, q.BucketName, fn)

	resp, err := q.HTTPClient.Get(reqStr)
	if err != nil {
		// TODO - we don't see errors here or below when the queue doesn't exist.
		// That seems bad.
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return errors.New("http error: " + resp.Status)
	}

	return nil
}

// postOneDay posts all items in an ObjectIterator into the appropriate queue.
func (q Queuer) postOneDay(wg *sync.WaitGroup, queue string, it *storage.ObjectIterator) {
	qpErrCount := 0
	gcsErrCount := 0
	fileCount := 0
	if wg != nil {
		defer wg.Done()
	}
	for o, err := it.Next(); err != iterator.Done; o, err = it.Next() {
		if err != nil {
			// TODO - should this retry?
			log.Println(err, "on it.Next")
			gcsErrCount++
			if gcsErrCount > 3 {
				log.Printf("Failed after %d files to %s.\n", fileCount, queue)
				return
			}
			continue
		}

		err = q.postOneTask(queue, o.Name)
		if err != nil {
			// TODO - should this retry?
			log.Println(err, o.Name, "Retrying")
			// Retry
			time.Sleep(10 * time.Second)
			err = q.postOneTask(queue, o.Name)

			if err != nil {
				log.Println(err, o.Name, "FAILED")
				qpErrCount++
				if qpErrCount > 3 {
					log.Printf("Failed after %d files to %s (on %s).\n", fileCount, queue, o.Name)
					return
				}
			}
		} else {
			fileCount++
		}
	}
	log.Println("Added ", fileCount, " tasks to ", queue)
}

// PostDay fetches an iterator over the objects with ndt/YYYY/MM/DD prefix,
// and passes the iterator to postDay with appropriate queue.
// Iff wq is not nil, PostDay will call done on wg when finished
// posting.
// This typically takes about 10 minutes, whether single or parallel days.
func (q Queuer) PostDay(wg *sync.WaitGroup, prefix string) {
	date, err := time.Parse("ndt/2006/01/02/", prefix)
	if err != nil {
		log.Println("Failed parsing date from ", prefix)
		log.Println(err)
		if wg != nil {
			wg.Done()
		}
		return
	}
	queue := q.queueForDate(date)
	log.Println("Adding ", prefix, " to ", queue)
	qry := storage.Query{
		Delimiter: "/",
		Prefix:    prefix,
	}
	// TODO - can this error?  Or do errors only occur on iterator ops?
	it := q.Bucket.Objects(context.Background(), &qry)
	if wg != nil {
		go q.postOneDay(wg, queue, it)
	} else {
		q.postOneDay(nil, queue, it)
	}
}

// PostMonth adds all of the files from dates within a specified month.
// It is difficult to test without hitting GCS.  8-(
// This typically takes about 10 minutes, processing all days concurrently.
func (q Queuer) PostMonth(prefix string) {
	qry := storage.Query{
		Delimiter: "/",
		// TODO - validate.
		Prefix: prefix,
	}
	it := q.Bucket.Objects(context.Background(), &qry)

	var wg sync.WaitGroup
	for o, err := it.Next(); err != iterator.Done; o, err = it.Next() {
		if err != nil {
			log.Println(err)
		} else if o.Prefix != "" {
			wg.Add(1)
			q.PostDay(&wg, o.Prefix)
		} else {
			log.Println("Skipping: ", o.Name)
		}
	}
	wg.Wait()
}
<|endoftext|>


<commit_before>package middleware

import (
	"bytes"
	"context"
	"encoding/json"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httptest"
	"net/http/httputil"
	"strings"
	"time"

	"github.com/gilcrest/go-API-template/pkg/env"
	"github.com/rs/xid"
	"github.com/rs/zerolog/log"
)

// const (
// 	dumpReq   = "dumpRequest"
// 	logReq    = "logRequest"
// 	logResp   = "logResponse"
// 	logReq2db = "logReq2db"
// 	// m["reqwbody2db"] = true
// 	// m["resp2db"] = true
// 	// m["respwbody2db"] = true
// 	// m["limited2db"] = true
// )

// APIAudit struct holds the http request and other attributes needed
// for auditing an http request
type APIAudit struct {
	ctx          context.Context
	RequestID    string        `json:"request_id"`
	TimeStarted  time.Time     `json:"time_started"`
	TimeFinished time.Time     `json:"time_finished"`
	TimeInMillis time.Duration `json:"time_in_millis"`
	ResponseCode int           `json:"response_code"`
	request
	response request
}

type request struct {
	Proto            string `json:"protocol"`
	ProtoMajor       int    `json:"protocol_major"`
	ProtoMinor       int    `json:"protocol_minor"`
	Method           string `json:"request_method"`
	Scheme           string `json:"scheme"`
	Host             string `json:"host"`
	Port             string `json:"port"`
	Path             string `json:"path"`
	Header           string `json:"header"`
	Body             string `json:"body"`
	ContentLength    int64  `json:"content_length"`
	TransferEncoding string `json:"transfer_encoding"`
	Close            bool   `json:"close"`
	Trailer          string `json:"trailer"`
	RemoteAddr       string `json:"remote_address"`
	RequestURI       string `json:"request_uri"`
}

// LogRequest wraps several optional logging functions
//   printRequest - sends request output from httputil.DumpRequest to STDERR
//   logRequest - uses logger util to log requests
//   log2DB - logs request to relational database TODO - not yet implemented
func LogRequest(env *env.Env, aud *APIAudit) Adapter {
	return func(h http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			log.Print("Start LogRequest")
			defer log.Print("Finish LogRequest")
			err := logReqDispatch(env, aud, r)
			if err != nil {
				http.Error(w, "Unable to log request", http.StatusBadRequest)
				return
			}
			h.ServeHTTP(w, r) // call original
		})
	}
}

// logReqDispatch determines which, if any, of the logging methods
// you wish to use will be employed.  Using a cache mechanism I haven't
// implemented yet, you will be able to turn on/off these methods on demand
// as of right now, it's doing all of them, which is ridiculous
func logReqDispatch(env *env.Env, aud *APIAudit, req *http.Request) error {

	var err error

	err = SetRequest(aud, req)
	if err != nil {
		return err
	}

	if env.LogOpts.DumpRequest.Write {
		// func DumpRequest(req *http.Request, body bool) ([]byte, error)
		requestDump, err := httputil.DumpRequest(req, env.LogOpts.DumpRequest.Body)
		if err != nil {
			return err
		}
		log.Print(string(requestDump))
		return nil
	}

	if env.LogOpts.Log2StdOut.Request.Write {
		err = logRequest(env, aud)
		if err != nil {
			return err
		}
	}

	// if logSwitch(env, logReq2db) {
	// 	err = logRequest2Db(env, aud)
	// 	if err != nil {
	// 		return err
	// 	}
	// }

	return nil

}

// func logSwitch(env *env.Env, logTest string) bool {

// 	// If nothing is in the LogMap, then return false
// 	if len(env.LogMap) == 0 {
// 		return false
// 	}

// 	// for dumprequest, validate if true in map
// 	if logTest == dumpReq {
// 		if mv := env.LogMap[dumpReq]; mv {
// 			return true
// 		}
// 	}

// 	// for logRequest, validate if true in map
// 	if logTest == logReq {
// 		if mv := env.LogMap[logReq]; mv {
// 			return true
// 		}
// 	}

// 	// for logResponse, validate if true in map
// 	if logTest == logResp {
// 		if mv := env.LogMap[logResp]; mv {
// 			return true
// 		}
// 	}

// 	return false
// }

// SetRequest populates several core fields (TimeStarted, ctx and RequestID)
// for the APIAudit struct being passed into the function
func SetRequest(aud *APIAudit, req *http.Request) error {

	var (
		scheme string
	)

	// split host and port out for cleaner logging
	host, port, err := net.SplitHostPort(req.Host)
	if err != nil {
		return err
	}

	// determine if the request is an HTTPS request
	isHTTPS := req.TLS != nil

	if isHTTPS {
		scheme = "https"
	} else {
		scheme = "http"
	}

	// convert the Header map from the request to a JSON string
	headerJSON, err := convertHeader(req.Header)
	if err != nil {
		return err
	}

	// convert the Trailer map from the request to a JSON string
	trailerJSON, err := convertHeader(req.Trailer)
	if err != nil {
		return err
	}

	// get byte Array representation of guid from xid package (12 bytes)
	guid := xid.New()

	// use the String method of the guid object to convert byte array to string (20 bytes)
	rID := guid.String()

	body, err := dumpBody(req)
	if err != nil {
		return err
	}

	aud.ctx = req.Context() // retrieve the context from the http.Request
	aud.RequestID = rID
	aud.request.Proto = req.Proto
	aud.request.ProtoMajor = req.ProtoMajor
	aud.request.ProtoMinor = req.ProtoMinor
	aud.request.Method = req.Method
	aud.request.Scheme = scheme
	aud.request.Host = host
	aud.request.Body = body
	aud.request.Port = port
	aud.request.Path = req.URL.Path
	aud.request.Header = headerJSON
	aud.request.ContentLength = req.ContentLength
	aud.request.TransferEncoding = strings.Join(req.TransferEncoding, ",")
	aud.request.Close = req.Close
	aud.request.Trailer = trailerJSON
	aud.request.RemoteAddr = req.RemoteAddr
	aud.request.RequestURI = req.RequestURI

	return nil

}

// SetResponse sets the response elements of the APIAudit payload
func SetResponse(aud *APIAudit, rec *httptest.ResponseRecorder) error {
	// set ResponseCode from ResponseRecorder
	aud.ResponseCode = rec.Code

	// set Header JSON from Header map in ResponseRecorder
	headerJSON, err := convertHeader(rec.HeaderMap)
	if err != nil {
		return err
	}
	aud.response.Header = headerJSON

	// Dump body to text using dumpBody function - need an http request
	// struct, so use httptest.NewRequest to get one
	req := httptest.NewRequest("POST", "http://example.com/foo", rec.Body)

	body, err := dumpBody(req)
	if err != nil {
		return err
	}
	aud.response.Body = body

	return nil
}

// convertHeader returns a JSON string representation of an http.Header map
func convertHeader(hdr http.Header) (string, error) {
	// convert the http.Header map from the request to a slice of bytes
	jsonBytes, err := json.Marshal(hdr)
	if err != nil {
		return "", err
	}

	// convert the slice of bytes to a JSON string
	headerJSON := string(jsonBytes)

	return headerJSON, nil

}

// drainBody reads all of b to memory and then returns two equivalent
// ReadClosers yielding the same bytes.
//
// It returns an error if the initial slurp of all bytes fails. It does not attempt
// to make the returned ReadClosers have identical error-matching behavior.
// Function lifted straight from net/http/httputil package
func drainBody(b io.ReadCloser) (r1, r2 io.ReadCloser, err error) {
	if b == http.NoBody {
		// No copying needed. Preserve the magic sentinel meaning of NoBody.
		return http.NoBody, http.NoBody, nil
	}
	var buf bytes.Buffer
	if _, err = buf.ReadFrom(b); err != nil {
		return nil, b, err
	}
	if err = b.Close(); err != nil {
		return nil, b, err
	}
	return ioutil.NopCloser(&buf), ioutil.NopCloser(bytes.NewReader(buf.Bytes())), nil
}

func dumpBody(req *http.Request) (string, error) {
	var err error
	save := req.Body
	save, req.Body, err = drainBody(req.Body)
	if err != nil {
		return "", err
	}
	var b bytes.Buffer

	chunked := len(req.TransferEncoding) > 0 && req.TransferEncoding[0] == "chunked"

	if req.Body != nil {
		var dest io.Writer = &b
		if chunked {
			dest = httputil.NewChunkedWriter(dest)
		}
		_, err = io.Copy(dest, req.Body)
		if chunked {
			dest.(io.Closer).Close()
			io.WriteString(&b, "\r\n")
		}
	}

	req.Body = save
	if err != nil {
		return "", err
	}
	return string(b.Bytes()), nil
}

// func logFormValues(lgr zerolog.Logger, req *http.Request) (zerolog.Logger, error) {

// 	var i int

// 	err := req.ParseForm()
// 	if err != nil {
// 		return nil, err
// 	}

// 	for key, valSlice := range req.Form {
// 		for _, val := range valSlice {
// 			i++
// 			formValue := fmt.Sprintf("%s: %s", key, val)
// 			lgr = lgr.With().Str(fmt.Sprintf("Form(%d)", i), formValue))
// 		}
// 	}

// 	for key, valSlice := range req.PostForm {
// 		for _, val := range valSlice {
// 			i++
// 			formValue := fmt.Sprintf("%s: %s", key, val)
// 			lgr = lgr.With(Str(fmt.Sprintf("PostForm(%d)", i), formValue))
// 		}
// 	}

// 	return lgr, nil
// }

func logRequest(env *env.Env, aud *APIAudit) error {

	//var err error

	log := env.Logger

	log.Debug().Msg("logRequest started")
	defer log.Debug().Msg("logRequest ended")

	// logger, err = logFormValues(logger, req)
	// if err != nil {
	// 	return err
	// }

	// All header key:value pairs written to JSON
	if env.LogOpts.Log2StdOut.Request.Header {
		log = log.With().Str("header_json", aud.request.Header).Logger()
	}

	if env.LogOpts.Log2StdOut.Request.Body {
		log = log.With().Str("body", aud.request.Body).Logger()
	}

	log.Info().
		Str("request_id", aud.RequestID).
		Str("method", aud.request.Method).
		// most url.URL components split out
		Str("scheme", aud.request.Scheme).
		Str("host", aud.request.Host).
		Str("port", aud.request.Port).
		Str("path", aud.request.Path).
		// The protocol version for incoming server requests.
		Str("protocol", aud.request.Proto).
		Int("proto_major", aud.request.ProtoMajor).
		Int("proto_minor", aud.request.ProtoMinor).
		Int64("Content Length", aud.request.ContentLength).
		Str("Transfer-Encoding", aud.request.TransferEncoding).
		Bool("Close", aud.request.Close).
		Str("RemoteAddr", aud.request.RemoteAddr).
		Str("RequestURI", aud.request.RequestURI).
		Msg("Request received")

	return nil
}

// Creates a record in the appUser table using a stored function
func logRequest2Db(env *env.Env, aud *APIAudit) error {

	var (
		rowsInserted int
		respHdr      interface{}
		reqHdr       interface{}
	)

	respHdr = headerNil(aud.response.Header)
	reqHdr = headerNil(aud.request.Header)

	// Calls the BeginTx method of the MainDb opened database
	tx, err := env.DS.MainDb.BeginTx(aud.ctx, nil)
	if err != nil {
		return err
	}

	// Prepare the sql statement using bind variables
	stmt, err := tx.PrepareContext(aud.ctx, `select api.log_request
		(
		p_request_id => $1,
		p_request_timestamp => $2,
		p_response_code => $3,
		p_response_timestamp => $4,
		p_duration_in_millis => $5,
		p_protocol => $6,
		p_protocol_major => $7,
		p_protocol_minor => $8,
		p_request_method => $9,
		p_scheme => $10,
		p_host => $11,
		p_port => $12,
		p_path => $13,
		p_remote_address => $14,
		p_request_header => $15,
		p_request_content_length => $16,
		p_request_body => $17,
		p_response_header => $18,
		p_response_body => $19)`)

	if err != nil {
		return err
	}
	defer stmt.Close()

	rows, err := stmt.QueryContext(aud.ctx,
		aud.RequestID,          //$1
		aud.TimeStarted,        //$2
		aud.ResponseCode,       //$3
		aud.TimeFinished,       //$4
		aud.TimeInMillis,       //$5
		aud.request.Proto,      //$6
		aud.request.ProtoMajor, //$7
		aud.request.ProtoMinor, //$8
		aud.request.Method,     //$9
		aud.request.Scheme,     //$10
		aud.request.Host,       //$11
		aud.request.Port,       //$12
		aud.request.Path,       //$13
		aud.request.RemoteAddr, //$14
		reqHdr,                 //$15
		aud.request.ContentLength, //$16
		aud.request.Body,          //$17
		respHdr,                   //$18
		aud.response.Body)         //$19

	if err != nil {
		log.Print(err)
		return err
	}
	defer rows.Close()

	// Iterate through the returned record(s)
	for rows.Next() {
		if err := rows.Scan(&rowsInserted); err != nil {
			log.Print(err)
			return err
		}
	}

	err = rows.Err()
	if err != nil {
		return err
	}

	// If we have successfully written rows to the db
	// we commit the transaction
	err = tx.Commit()
	if err != nil {
		return err
	}

	return nil

}

// headerNil checks if the header field is an empty string
// (the empty value for the string type) and switches it to
// a nil.  An empty string is not allowed to be passed to a
// JSONB type in postgres, however, a nil works
func headerNil(hdr string) interface{} {
	var respHdr interface{}

	if hdr == "" {
		respHdr = nil
	} else {
		respHdr = hdr
	}

	return respHdr
}

func logResponse(env *env.Env, aud *APIAudit) error {

	//var err error

	logger := env.Logger

	logger.Debug().Msg("logResponse started")
	defer logger.Debug().Msg("logResponse ended")

	// logger, err = logFormValues(logger, req)
	// if err != nil {
	// 	return err
	// }

	logger.Info().
		Str("request_id", aud.RequestID).
		Int("response_code", aud.ResponseCode).
		Str("header_json", aud.response.Header).
		Str("body", aud.response.Body).
		Msg("Response Sent")

	return nil
}
<commit_msg>re-org<commit_after>package middleware

import (
	"bytes"
	"context"
	"encoding/json"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httputil"
	"strings"
	"time"

	"github.com/gilcrest/go-API-template/pkg/env"
	"github.com/rs/xid"
	"github.com/rs/zerolog/log"
)

// APIAudit struct holds the http request and other attributes needed
// for auditing an http request
type APIAudit struct {
	ctx          context.Context
	RequestID    string        `json:"request_id"`
	TimeStarted  time.Time     `json:"time_started"`
	TimeFinished time.Time     `json:"time_finished"`
	Duration     time.Duration `json:"time_in_millis"`
	ResponseCode int           `json:"response_code"`
	request
	response request
}

type request struct {
	Proto            string `json:"protocol"`
	ProtoMajor       int    `json:"protocol_major"`
	ProtoMinor       int    `json:"protocol_minor"`
	Method           string `json:"request_method"`
	Scheme           string `json:"scheme"`
	Host             string `json:"host"`
	Port             string `json:"port"`
	Path             string `json:"path"`
	Header           string `json:"header"`
	Body             string `json:"body"`
	ContentLength    int64  `json:"content_length"`
	TransferEncoding string `json:"transfer_encoding"`
	Close            bool   `json:"close"`
	Trailer          string `json:"trailer"`
	RemoteAddr       string `json:"remote_address"`
	RequestURI       string `json:"request_uri"`
}

// LogRequest wraps several optional logging functions
//   printRequest - sends request output from httputil.DumpRequest to STDERR
//   logRequest - uses logger util to log requests
//   log2DB - logs request to relational database TODO - not yet implemented
func LogRequest(env *env.Env, aud *APIAudit) Adapter {
	return func(h http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			log.Print("Start LogRequest")
			defer log.Print("Finish LogRequest")
			err := logReqDispatch(env, aud, r)
			if err != nil {
				http.Error(w, "Unable to log request", http.StatusBadRequest)
				return
			}
			h.ServeHTTP(w, r) // call original
		})
	}
}

// logReqDispatch determines which, if any, of the logging methods
// you wish to use will be employed.  Using a cache mechanism I haven't
// implemented yet, you will be able to turn on/off these methods on demand
// as of right now, it's doing all of them, which is ridiculous
func logReqDispatch(env *env.Env, aud *APIAudit, req *http.Request) error {

	var err error

	err = setRequest(aud, req)
	if err != nil {
		return err
	}

	if env.LogOpts.DumpRequest.Write {
		requestDump, err := httputil.DumpRequest(req, env.LogOpts.DumpRequest.Body)
		if err != nil {
			log.Error().Err(err).Msg("")
			return err
		}
		log.Print(string(requestDump))
		return nil
	}

	if env.LogOpts.Log2StdOut.Request.Write {
		err = logReq2Stdout(env, aud)
		if err != nil {
			log.Error().Err(err).Msg("")
			return err
		}
	}

	return nil
}

// setRequest populates several core fields (TimeStarted, ctx and RequestID)
// for the APIAudit struct being passed into the function
func setRequest(aud *APIAudit, req *http.Request) error {

	var (
		scheme string
	)

	// split host and port out for cleaner logging
	host, port, err := net.SplitHostPort(req.Host)
	if err != nil {
		log.Error().Err(err).Msg("")
		return err
	}

	// determine if the request is an HTTPS request
	isHTTPS := req.TLS != nil

	if isHTTPS {
		scheme = "https"
	} else {
		scheme = "http"
	}

	// convert the Header map from the request to a JSON string
	headerJSON, err := convertHeader(req.Header)
	if err != nil {
		log.Error().Err(err).Msg("")
		return err
	}

	// convert the Trailer map from the request to a JSON string
	trailerJSON, err := convertHeader(req.Trailer)
	if err != nil {
		log.Error().Err(err).Msg("")
		return err
	}

	// get byte Array representation of guid from xid package (12 bytes)
	guid := xid.New()

	// use the String method of the guid object to convert byte array to string (20 bytes)
	rID := guid.String()

	body, err := dumpBody(req)
	if err != nil {
		log.Error().Err(err).Msg("")
		return err
	}

	aud.ctx = req.Context() // retrieve the context from the http.Request
	aud.RequestID = rID
	aud.request.Proto = req.Proto
	aud.request.ProtoMajor = req.ProtoMajor
	aud.request.ProtoMinor = req.ProtoMinor
	aud.request.Method = req.Method
	aud.request.Scheme = scheme
	aud.request.Host = host
	aud.request.Body = body
	aud.request.Port = port
	aud.request.Path = req.URL.Path
	aud.request.Header = headerJSON
	aud.request.ContentLength = req.ContentLength
	aud.request.TransferEncoding = strings.Join(req.TransferEncoding, ",")
	aud.request.Close = req.Close
	aud.request.Trailer = trailerJSON
	aud.request.RemoteAddr = req.RemoteAddr
	aud.request.RequestURI = req.RequestURI

	return nil
}

// convertHeader returns a JSON string representation of an http.Header map
func convertHeader(hdr http.Header) (string, error) {
	// convert the http.Header map from the request to a slice of bytes
	jsonBytes, err := json.Marshal(hdr)
	if err != nil {
		log.Error().Err(err).Msg("")
		return "", err
	}

	// convert the slice of bytes to a JSON string
	headerJSON := string(jsonBytes)

	return headerJSON, nil

}

// drainBody reads all of b to memory and then returns two equivalent
// ReadClosers yielding the same bytes.
//
// It returns an error if the initial slurp of all bytes fails. It does not attempt
// to make the returned ReadClosers have identical error-matching behavior.
// Function lifted straight from net/http/httputil package
func drainBody(b io.ReadCloser) (r1, r2 io.ReadCloser, err error) {
	if b == http.NoBody {
		// No copying needed. Preserve the magic sentinel meaning of NoBody.
		return http.NoBody, http.NoBody, nil
	}
	var buf bytes.Buffer
	if _, err = buf.ReadFrom(b); err != nil {
		return nil, b, err
	}
	if err = b.Close(); err != nil {
		return nil, b, err
	}
	return ioutil.NopCloser(&buf), ioutil.NopCloser(bytes.NewReader(buf.Bytes())), nil
}

func dumpBody(req *http.Request) (string, error) {
	var err error
	save := req.Body
	save, req.Body, err = drainBody(req.Body)
	if err != nil {
		return "", err
	}
	var b bytes.Buffer

	chunked := len(req.TransferEncoding) > 0 && req.TransferEncoding[0] == "chunked"

	if req.Body != nil {
		var dest io.Writer = &b
		if chunked {
			dest = httputil.NewChunkedWriter(dest)
		}
		_, err = io.Copy(dest, req.Body)
		if chunked {
			dest.(io.Closer).Close()
			io.WriteString(&b, "\r\n")
		}
	}

	req.Body = save
	if err != nil {
		return "", err
	}
	return string(b.Bytes()), nil
}

// func logFormValues(lgr zerolog.Logger, req *http.Request) (zerolog.Logger, error) {

// 	var i int

// 	err := req.ParseForm()
// 	if err != nil {
// 		return nil, err
// 	}

// 	for key, valSlice := range req.Form {
// 		for _, val := range valSlice {
// 			i++
// 			formValue := fmt.Sprintf("%s: %s", key, val)
// 			lgr = lgr.With().Str(fmt.Sprintf("Form(%d)", i), formValue))
// 		}
// 	}

// 	for key, valSlice := range req.PostForm {
// 		for _, val := range valSlice {
// 			i++
// 			formValue := fmt.Sprintf("%s: %s", key, val)
// 			lgr = lgr.With(Str(fmt.Sprintf("PostForm(%d)", i), formValue))
// 		}
// 	}

// 	return lgr, nil
// }

func logReq2Stdout(env *env.Env, aud *APIAudit) error {

	//var err error

	log := env.Logger

	log.Debug().Msg("logRequest started")
	defer log.Debug().Msg("logRequest ended")

	// logger, err = logFormValues(logger, req)
	// if err != nil {
	// 	return err
	// }

	// All header key:value pairs written to JSON
	if env.LogOpts.Log2StdOut.Request.Header {
		log = log.With().Str("header_json", aud.request.Header).Logger()
	}

	if env.LogOpts.Log2StdOut.Request.Body {
		log = log.With().Str("body", aud.request.Body).Logger()
	}

	log.Info().
		Str("request_id", aud.RequestID).
		Str("method", aud.request.Method).
		// most url.URL components split out
		Str("scheme", aud.request.Scheme).
		Str("host", aud.request.Host).
		Str("port", aud.request.Port).
		Str("path", aud.request.Path).
		// The protocol version for incoming server requests.
		Str("protocol", aud.request.Proto).
		Int("proto_major", aud.request.ProtoMajor).
		Int("proto_minor", aud.request.ProtoMinor).
		Int64("Content Length", aud.request.ContentLength).
		Str("Transfer-Encoding", aud.request.TransferEncoding).
		Bool("Close", aud.request.Close).
		Str("RemoteAddr", aud.request.RemoteAddr).
		Str("RequestURI", aud.request.RequestURI).
		Msg("Request received")

	return nil
}
<|endoftext|>