<commit_before>package application

import (
	"fmt"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/cloudfoundry/cli/cf"
	"github.com/cloudfoundry/cli/cf/api"
	"github.com/cloudfoundry/cli/cf/api/app_instances"
	"github.com/cloudfoundry/cli/cf/api/applications"
	"github.com/cloudfoundry/cli/cf/command_registry"
	"github.com/cloudfoundry/cli/cf/configuration/core_config"
	. "github.com/cloudfoundry/cli/cf/i18n"
	"github.com/cloudfoundry/cli/cf/models"
	"github.com/cloudfoundry/cli/cf/requirements"
	"github.com/cloudfoundry/cli/cf/terminal"
	"github.com/cloudfoundry/loggregatorlib/logmessage"
	"github.com/cloudfoundry/sonde-go/events"
	"github.com/simonleung8/flags"
)

const (
	DefaultStagingTimeout = 15 * time.Minute
	DefaultStartupTimeout = 5 * time.Minute
	DefaultPingerThrottle = 5 * time.Second
)

const LogMessageTypeStaging = "STG"

type ApplicationStagingWatcher interface {
	ApplicationWatchStaging(app models.Application, orgName string, spaceName string, startCommand func(app models.Application) (models.Application, error)) (updatedApp models.Application, err error)
}

//go:generate counterfeiter -o ../../../testhelpers/commands/fake_application_starter.go . ApplicationStarter
type ApplicationStarter interface {
	command_registry.Command
	SetStartTimeoutInSeconds(timeout int)
	ApplicationStart(app models.Application, orgName string, spaceName string) (updatedApp models.Application, err error)
}

type Start struct {
	ui               terminal.UI
	config           core_config.Reader
	appDisplayer     ApplicationDisplayer
	appReq           requirements.ApplicationRequirement
	appRepo          applications.ApplicationRepository
	appInstancesRepo app_instances.AppInstancesRepository
	oldLogsRepo      api.OldLogsRepository
	logRepo          api.LogsNoaaRepository

	LogServerConnectionTimeout time.Duration
	StartupTimeout             time.Duration
	StagingTimeout             time.Duration
	PingerThrottle             time.Duration
}

func init() {
	command_registry.Register(&Start{})
}

func (cmd *Start) MetaData() command_registry.CommandMetadata {
	return command_registry.CommandMetadata{
		Name:        "start",
		ShortName:   "st",
		Description: T("Start an app"),
		Usage:       T("CF_NAME start APP_NAME"),
	}
}

func (cmd *Start) Requirements(requirementsFactory requirements.Factory, fc flags.FlagContext) (reqs []requirements.Requirement, err error) {
	if len(fc.Args()) != 1 {
		cmd.ui.Failed(T("Incorrect Usage. Requires an argument\n\n") + command_registry.Commands.CommandUsage("start"))
	}

	cmd.appReq = requirementsFactory.NewApplicationRequirement(fc.Args()[0])

	reqs = []requirements.Requirement{requirementsFactory.NewLoginRequirement(), requirementsFactory.NewTargetedSpaceRequirement(), cmd.appReq}
	return
}

func (cmd *Start) SetDependency(deps command_registry.Dependency, pluginCall bool) command_registry.Command {
	cmd.ui = deps.Ui
	cmd.config = deps.Config
	cmd.appRepo = deps.RepoLocator.GetApplicationRepository()
	cmd.appInstancesRepo = deps.RepoLocator.GetAppInstancesRepository()
	cmd.logRepo = deps.RepoLocator.GetLogsNoaaRepository()
	cmd.oldLogsRepo = deps.RepoLocator.GetOldLogsRepository()
	cmd.LogServerConnectionTimeout = 20 * time.Second
	cmd.PingerThrottle = DefaultPingerThrottle

	if os.Getenv("CF_STAGING_TIMEOUT") != "" {
		duration, err := strconv.ParseInt(os.Getenv("CF_STAGING_TIMEOUT"), 10, 64)
		if err != nil {
			cmd.ui.Failed(T("invalid value for env var CF_STAGING_TIMEOUT\n{{.Err}}",
				map[string]interface{}{"Err": err}))
		}
		cmd.StagingTimeout = time.Duration(duration) * time.Minute
	} else {
		cmd.StagingTimeout = DefaultStagingTimeout
	}

	if os.Getenv("CF_STARTUP_TIMEOUT") != "" {
		duration, err := strconv.ParseInt(os.Getenv("CF_STARTUP_TIMEOUT"), 10, 64)
		if err != nil {
			cmd.ui.Failed(T("invalid value for env var CF_STARTUP_TIMEOUT\n{{.Err}}",
				map[string]interface{}{"Err": err}))
		}
		cmd.StartupTimeout = time.Duration(duration) * time.Minute
	} else {
		cmd.StartupTimeout = DefaultStartupTimeout
	}

	appCommand := command_registry.Commands.FindCommand("app")
	appCommand = appCommand.SetDependency(deps, false)
	cmd.appDisplayer = appCommand.(ApplicationDisplayer)

	return cmd
}

func (cmd *Start) Execute(c flags.FlagContext) {
	cmd.ApplicationStart(cmd.appReq.GetApplication(), cmd.config.OrganizationFields().Name, cmd.config.SpaceFields().Name)
}

func (cmd *Start) ApplicationStart(app models.Application, orgName, spaceName string) (updatedApp models.Application, err error) {
	if app.State == "started" {
		cmd.ui.Say(terminal.WarningColor(T("App ") + app.Name + T(" is already started")))
		return
	}

	return cmd.ApplicationWatchStaging(app, orgName, spaceName, func(app models.Application) (models.Application, error) {
		cmd.ui.Say(T("Starting app {{.AppName}} in org {{.OrgName}} / space {{.SpaceName}} as {{.CurrentUser}}...",
			map[string]interface{}{
				"AppName":     terminal.EntityNameColor(app.Name),
				"OrgName":     terminal.EntityNameColor(orgName),
				"SpaceName":   terminal.EntityNameColor(spaceName),
				"CurrentUser": terminal.EntityNameColor(cmd.config.Username())}))

		state := "STARTED"
		return cmd.appRepo.Update(app.Guid, models.AppParams{State: &state})
	})
}

func (cmd *Start) ApplicationWatchStaging(app models.Application, orgName, spaceName string, start func(app models.Application) (models.Application, error)) (updatedApp models.Application, err error) {
	var isConnected bool
	loggingStartedChan := make(chan bool)
	doneLoggingChan := make(chan bool)

	go cmd.tailStagingLogs(app, loggingStartedChan, doneLoggingChan)
	timeout := make(chan struct{})
	go func() {
		time.Sleep(cmd.LogServerConnectionTimeout)
		close(timeout)
	}()

	select {
	case <-timeout:
		cmd.ui.Warn("timeout connecting to log server, no log will be shown")
		break
	case <-loggingStartedChan: // block until we have established connection to Loggregator
		isConnected = true
		break
	}

	updatedApp, apiErr := start(app)
	if apiErr != nil {
		cmd.ui.Failed(apiErr.Error())
		return
	}

	isStaged := cmd.waitForInstancesToStage(updatedApp)

	if isConnected { //only close when actually connected, else CLI hangs at closing consumer connection
		// cmd.logRepo.Close()
		cmd.oldLogsRepo.Close()
	}

	<-doneLoggingChan

	cmd.ui.Say("")

	if !isStaged {
		cmd.ui.Failed(fmt.Sprintf("%s failed to stage within %f minutes", app.Name, cmd.StagingTimeout.Minutes()))
	}

	cmd.waitForOneRunningInstance(updatedApp)
	cmd.ui.Say(terminal.HeaderColor(T("\nApp started\n")))
	cmd.ui.Say("")
	cmd.ui.Ok()

	//detectedstartcommand on first push is not present until starting completes
	startedApp, apiErr := cmd.appRepo.Read(updatedApp.Name)
	if err != nil {
		cmd.ui.Failed(apiErr.Error())
		return
	}

	var appStartCommand string
	if app.Command == "" {
		appStartCommand = startedApp.DetectedStartCommand
	} else {
		appStartCommand = startedApp.Command
	}

	cmd.ui.Say(T("\nApp {{.AppName}} was started using this command `{{.Command}}`\n",
		map[string]interface{}{
			"AppName": terminal.EntityNameColor(startedApp.Name),
			"Command": appStartCommand,
		}))

	cmd.appDisplayer.ShowApp(startedApp, orgName, spaceName)
	return
}

func (cmd *Start) SetStartTimeoutInSeconds(timeout int) {
	cmd.StartupTimeout = time.Duration(timeout) * time.Second
}

func simpleLogMessageOutput(logMsg *logmessage.LogMessage) (msgText string) {
	msgText = string(logMsg.GetMessage())
	reg, err := regexp.Compile("[\n\r]+$")
	if err != nil {
		return
	}
	msgText = reg.ReplaceAllString(msgText, "")
	return
}

func (cmd *Start) tailStagingLogs(app models.Application, startChan, doneChan chan bool) {
	onConnect := func() {
		startChan <- true
	}

	err := cmd.oldLogsRepo.TailLogsFor(app.Guid, onConnect, func(msg *logmessage.LogMessage) {
		if msg.GetSourceName() == LogMessageTypeStaging {
			cmd.ui.Say(simpleLogMessageOutput(msg))
		}
	})
	// err := cmd.logRepo.TailNoaaLogsFor(app.Guid, onConnect, func(msg *events.LogMessage) {
	// 	if msg.GetSourceType() == LogMessageTypeStaging {
	// 		cmd.ui.Say(simpleLogMessageOutput(msg))
	// 	}
	// })

	if err != nil {
		cmd.ui.Warn(T("Warning: error tailing logs"))
		cmd.ui.Say("%s", err)
		close(startChan)
	}

	close(doneChan)
}

func (cmd *Start) waitForInstancesToStage(app models.Application) bool {
	stagingStartTime := time.Now()

	var err error

	if cmd.StagingTimeout == 0 {
		app, err = cmd.appRepo.GetApp(app.Guid)
	} else {
		for app.PackageState != "STAGED" && app.PackageState != "FAILED" && time.Since(stagingStartTime) < cmd.StagingTimeout {
			app, err = cmd.appRepo.GetApp(app.Guid)
			if err != nil {
				break
			}
			cmd.ui.Wait(cmd.PingerThrottle)
		}
	}

	if err != nil {
		cmd.ui.Failed(err.Error())
	}

	if app.PackageState == "FAILED" {
		cmd.ui.Say("")
		if app.StagingFailedReason == "NoAppDetectedError" {
			cmd.ui.Failed(T(`{{.Err}}

TIP: Buildpacks are detected when the "{{.PushCommand}}" is executed from within the directory that contains the app source code.

Use '{{.BuildpackCommand}}' to see a list of supported buildpacks.

Use '{{.Command}}' for more in depth log information.`,
				map[string]interface{}{
					"Err":              app.StagingFailedReason,
					"PushCommand":      terminal.CommandColor(fmt.Sprintf("%s push", cf.Name())),
					"BuildpackCommand": terminal.CommandColor(fmt.Sprintf("%s buildpacks", cf.Name())),
					"Command":          terminal.CommandColor(fmt.Sprintf("%s logs %s --recent", cf.Name(), app.Name))}))
		} else {
			cmd.ui.Failed(T("{{.Err}}\n\nTIP: use '{{.Command}}' for more information",
				map[string]interface{}{
					"Err":     app.StagingFailedReason,
					"Command": terminal.CommandColor(fmt.Sprintf("%s logs %s --recent", cf.Name(), app.Name))}))
		}
	}

	if time.Since(stagingStartTime) >= cmd.StagingTimeout {
		return false
	}

	return true
}

func (cmd *Start) waitForOneRunningInstance(app models.Application) {
	startupStartTime := time.Now()

	for {
		if time.Since(startupStartTime) > cmd.StartupTimeout {
			cmd.ui.Failed(fmt.Sprintf(T("Start app timeout\n\nTIP: use '{{.Command}}' for more information",
				map[string]interface{}{
					"Command": terminal.CommandColor(fmt.Sprintf("%s logs %s --recent", cf.Name(), app.Name))})))
			return
		}

		count, err := cmd.fetchInstanceCount(app.Guid)
		if err != nil {
			cmd.ui.Wait(cmd.PingerThrottle)
			continue
		}

		cmd.ui.Say(instancesDetails(count))

		if count.running > 0 {
			return
		}

		if count.flapping > 0 || count.crashed > 0 {
			cmd.ui.Failed(fmt.Sprintf(T("Start unsuccessful\n\nTIP: use '{{.Command}}' for more information",
				map[string]interface{}{"Command": terminal.CommandColor(fmt.Sprintf("%s logs %s --recent", cf.Name(), app.Name))})))
			return
		}

		cmd.ui.Wait(cmd.PingerThrottle)
	}
}

type instanceCount struct {
	running         int
	starting        int
	startingDetails map[string]struct{}
	flapping        int
	down            int
	crashed         int
	total           int
}

func (cmd Start) fetchInstanceCount(appGuid string) (instanceCount, error) {
	count := instanceCount{
		startingDetails: make(map[string]struct{}),
	}

	instances, apiErr := cmd.appInstancesRepo.GetInstances(appGuid)
	if apiErr != nil {
		return instanceCount{}, apiErr
	}

	count.total = len(instances)

	for _, inst := range instances {
		switch inst.State {
		case models.InstanceRunning:
			count.running++
		case models.InstanceStarting:
			count.starting++
			if inst.Details != "" {
				count.startingDetails[inst.Details] = struct{}{}
			}
		case models.InstanceFlapping:
			count.flapping++
		case models.InstanceDown:
			count.down++
		case models.InstanceCrashed:
			count.crashed++
		}
	}

	return count, nil
}

func instancesDetails(count instanceCount) string {
	details := []string{fmt.Sprintf(T("{{.RunningCount}} of {{.TotalCount}} instances running",
		map[string]interface{}{"RunningCount": count.running, "TotalCount": count.total}))}

	if count.starting > 0 {
		if len(count.startingDetails) == 0 {
			details = append(details, fmt.Sprintf(T("{{.StartingCount}} starting",
				map[string]interface{}{"StartingCount": count.starting})))
		} else {
			info := []string{}
			for d, _ := range count.startingDetails {
				info = append(info, d)
			}
			sort.Strings(info)
			details = append(details, fmt.Sprintf(T("{{.StartingCount}} starting ({{.Details}})",
				map[string]interface{}{
					"StartingCount": count.starting,
					"Details":       strings.Join(info, ", "),
				})))
		}
	}

	if count.down > 0 {
		details = append(details, fmt.Sprintf(T("{{.DownCount}} down",
			map[string]interface{}{"DownCount": count.down})))
	}

	if count.flapping > 0 {
		details = append(details, fmt.Sprintf(T("{{.FlappingCount}} failing",
			map[string]interface{}{"FlappingCount": count.flapping})))
	}

	if count.crashed > 0 {
		details = append(details, fmt.Sprintf(T("{{.CrashedCount}} crashed",
			map[string]interface{}{"CrashedCount": count.crashed})))
	}

	return strings.Join(details, ", ")
}
<commit_msg>remove unused import<commit_after>package application

import (
	"fmt"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/cloudfoundry/cli/cf"
	"github.com/cloudfoundry/cli/cf/api"
	"github.com/cloudfoundry/cli/cf/api/app_instances"
	"github.com/cloudfoundry/cli/cf/api/applications"
	"github.com/cloudfoundry/cli/cf/command_registry"
	"github.com/cloudfoundry/cli/cf/configuration/core_config"
	. "github.com/cloudfoundry/cli/cf/i18n"
	"github.com/cloudfoundry/cli/cf/models"
	"github.com/cloudfoundry/cli/cf/requirements"
	"github.com/cloudfoundry/cli/cf/terminal"
	"github.com/cloudfoundry/loggregatorlib/logmessage"
	"github.com/simonleung8/flags"
)

const (
	DefaultStagingTimeout = 15 * time.Minute
	DefaultStartupTimeout = 5 * time.Minute
	DefaultPingerThrottle = 5 * time.Second
)

const LogMessageTypeStaging = "STG"

type ApplicationStagingWatcher interface {
	ApplicationWatchStaging(app models.Application, orgName string, spaceName string, startCommand func(app models.Application) (models.Application, error)) (updatedApp models.Application, err error)
}

//go:generate counterfeiter -o ../../../testhelpers/commands/fake_application_starter.go . ApplicationStarter
type ApplicationStarter interface {
	command_registry.Command
	SetStartTimeoutInSeconds(timeout int)
	ApplicationStart(app models.Application, orgName string, spaceName string) (updatedApp models.Application, err error)
}

type Start struct {
	ui               terminal.UI
	config           core_config.Reader
	appDisplayer     ApplicationDisplayer
	appReq           requirements.ApplicationRequirement
	appRepo          applications.ApplicationRepository
	appInstancesRepo app_instances.AppInstancesRepository
	oldLogsRepo      api.OldLogsRepository
	logRepo          api.LogsNoaaRepository

	LogServerConnectionTimeout time.Duration
	StartupTimeout             time.Duration
	StagingTimeout             time.Duration
	PingerThrottle             time.Duration
}

func init() {
	command_registry.Register(&Start{})
}

func (cmd *Start) MetaData() command_registry.CommandMetadata {
	return command_registry.CommandMetadata{
		Name:        "start",
		ShortName:   "st",
		Description: T("Start an app"),
		Usage:       T("CF_NAME start APP_NAME"),
	}
}

func (cmd *Start) Requirements(requirementsFactory requirements.Factory, fc flags.FlagContext) (reqs []requirements.Requirement, err error) {
	if len(fc.Args()) != 1 {
		cmd.ui.Failed(T("Incorrect Usage. Requires an argument\n\n") + command_registry.Commands.CommandUsage("start"))
	}

	cmd.appReq = requirementsFactory.NewApplicationRequirement(fc.Args()[0])

	reqs = []requirements.Requirement{requirementsFactory.NewLoginRequirement(), requirementsFactory.NewTargetedSpaceRequirement(), cmd.appReq}
	return
}

func (cmd *Start) SetDependency(deps command_registry.Dependency, pluginCall bool) command_registry.Command {
	cmd.ui = deps.Ui
	cmd.config = deps.Config
	cmd.appRepo = deps.RepoLocator.GetApplicationRepository()
	cmd.appInstancesRepo = deps.RepoLocator.GetAppInstancesRepository()
	cmd.logRepo = deps.RepoLocator.GetLogsNoaaRepository()
	cmd.oldLogsRepo = deps.RepoLocator.GetOldLogsRepository()
	cmd.LogServerConnectionTimeout = 20 * time.Second
	cmd.PingerThrottle = DefaultPingerThrottle

	if os.Getenv("CF_STAGING_TIMEOUT") != "" {
		duration, err := strconv.ParseInt(os.Getenv("CF_STAGING_TIMEOUT"), 10, 64)
		if err != nil {
			cmd.ui.Failed(T("invalid value for env var CF_STAGING_TIMEOUT\n{{.Err}}",
				map[string]interface{}{"Err": err}))
		}
		cmd.StagingTimeout = time.Duration(duration) * time.Minute
	} else {
		cmd.StagingTimeout = DefaultStagingTimeout
	}

	if os.Getenv("CF_STARTUP_TIMEOUT") != "" {
		duration, err := strconv.ParseInt(os.Getenv("CF_STARTUP_TIMEOUT"), 10, 64)
		if err != nil {
			cmd.ui.Failed(T("invalid value for env var CF_STARTUP_TIMEOUT\n{{.Err}}",
				map[string]interface{}{"Err": err}))
		}
		cmd.StartupTimeout = time.Duration(duration) * time.Minute
	} else {
		cmd.StartupTimeout = DefaultStartupTimeout
	}

	appCommand := command_registry.Commands.FindCommand("app")
	appCommand = appCommand.SetDependency(deps, false)
	cmd.appDisplayer = appCommand.(ApplicationDisplayer)

	return cmd
}

func (cmd *Start) Execute(c flags.FlagContext) {
	cmd.ApplicationStart(cmd.appReq.GetApplication(), cmd.config.OrganizationFields().Name, cmd.config.SpaceFields().Name)
}

func (cmd *Start) ApplicationStart(app models.Application, orgName, spaceName string) (updatedApp models.Application, err error) {
	if app.State == "started" {
		cmd.ui.Say(terminal.WarningColor(T("App ") + app.Name + T(" is already started")))
		return
	}

	return cmd.ApplicationWatchStaging(app, orgName, spaceName, func(app models.Application) (models.Application, error) {
		cmd.ui.Say(T("Starting app {{.AppName}} in org {{.OrgName}} / space {{.SpaceName}} as {{.CurrentUser}}...",
			map[string]interface{}{
				"AppName":     terminal.EntityNameColor(app.Name),
				"OrgName":     terminal.EntityNameColor(orgName),
				"SpaceName":   terminal.EntityNameColor(spaceName),
				"CurrentUser": terminal.EntityNameColor(cmd.config.Username())}))

		state := "STARTED"
		return cmd.appRepo.Update(app.Guid, models.AppParams{State: &state})
	})
}

func (cmd *Start) ApplicationWatchStaging(app models.Application, orgName, spaceName string, start func(app models.Application) (models.Application, error)) (updatedApp models.Application, err error) {
	var isConnected bool
	loggingStartedChan := make(chan bool)
	doneLoggingChan := make(chan bool)

	go cmd.tailStagingLogs(app, loggingStartedChan, doneLoggingChan)
	timeout := make(chan struct{})
	go func() {
		time.Sleep(cmd.LogServerConnectionTimeout)
		close(timeout)
	}()

	select {
	case <-timeout:
		cmd.ui.Warn("timeout connecting to log server, no log will be shown")
		break
	case <-loggingStartedChan: // block until we have established connection to Loggregator
		isConnected = true
		break
	}

	updatedApp, apiErr := start(app)
	if apiErr != nil {
		cmd.ui.Failed(apiErr.Error())
		return
	}

	isStaged := cmd.waitForInstancesToStage(updatedApp)

	if isConnected { //only close when actually connected, else CLI hangs at closing consumer connection
		// cmd.logRepo.Close()
		cmd.oldLogsRepo.Close()
	}

	<-doneLoggingChan

	cmd.ui.Say("")

	if !isStaged {
		cmd.ui.Failed(fmt.Sprintf("%s failed to stage within %f minutes", app.Name, cmd.StagingTimeout.Minutes()))
	}

	cmd.waitForOneRunningInstance(updatedApp)
	cmd.ui.Say(terminal.HeaderColor(T("\nApp started\n")))
	cmd.ui.Say("")
	cmd.ui.Ok()

	//detectedstartcommand on first push is not present until starting completes
	startedApp, apiErr := cmd.appRepo.Read(updatedApp.Name)
	if err != nil {
		cmd.ui.Failed(apiErr.Error())
		return
	}

	var appStartCommand string
	if app.Command == "" {
		appStartCommand = startedApp.DetectedStartCommand
	} else {
		appStartCommand = startedApp.Command
	}

	cmd.ui.Say(T("\nApp {{.AppName}} was started using this command `{{.Command}}`\n",
		map[string]interface{}{
			"AppName": terminal.EntityNameColor(startedApp.Name),
			"Command": appStartCommand,
		}))

	cmd.appDisplayer.ShowApp(startedApp, orgName, spaceName)
	return
}

func (cmd *Start) SetStartTimeoutInSeconds(timeout int) {
	cmd.StartupTimeout = time.Duration(timeout) * time.Second
}

func simpleLogMessageOutput(logMsg *logmessage.LogMessage) (msgText string) {
	msgText = string(logMsg.GetMessage())
	reg, err := regexp.Compile("[\n\r]+$")
	if err != nil {
		return
	}
	msgText = reg.ReplaceAllString(msgText, "")
	return
}

func (cmd *Start) tailStagingLogs(app models.Application, startChan, doneChan chan bool) {
	onConnect := func() {
		startChan <- true
	}

	err := cmd.oldLogsRepo.TailLogsFor(app.Guid, onConnect, func(msg *logmessage.LogMessage) {
		if msg.GetSourceName() == LogMessageTypeStaging {
			cmd.ui.Say(simpleLogMessageOutput(msg))
		}
	})
	// err := cmd.logRepo.TailNoaaLogsFor(app.Guid, onConnect, func(msg *events.LogMessage) {
	// 	if msg.GetSourceType() == LogMessageTypeStaging {
	// 		cmd.ui.Say(simpleLogMessageOutput(msg))
	// 	}
	// })

	if err != nil {
		cmd.ui.Warn(T("Warning: error tailing logs"))
		cmd.ui.Say("%s", err)
		close(startChan)
	}

	close(doneChan)
}

func (cmd *Start) waitForInstancesToStage(app models.Application) bool {
	stagingStartTime := time.Now()

	var err error

	if cmd.StagingTimeout == 0 {
		app, err = cmd.appRepo.GetApp(app.Guid)
	} else {
		for app.PackageState != "STAGED" && app.PackageState != "FAILED" && time.Since(stagingStartTime) < cmd.StagingTimeout {
			app, err = cmd.appRepo.GetApp(app.Guid)
			if err != nil {
				break
			}
			cmd.ui.Wait(cmd.PingerThrottle)
		}
	}

	if err != nil {
		cmd.ui.Failed(err.Error())
	}

	if app.PackageState == "FAILED" {
		cmd.ui.Say("")
		if app.StagingFailedReason == "NoAppDetectedError" {
			cmd.ui.Failed(T(`{{.Err}}

TIP: Buildpacks are detected when the "{{.PushCommand}}" is executed from within the directory that contains the app source code.

Use '{{.BuildpackCommand}}' to see a list of supported buildpacks.

Use '{{.Command}}' for more in depth log information.`,
				map[string]interface{}{
					"Err":              app.StagingFailedReason,
					"PushCommand":      terminal.CommandColor(fmt.Sprintf("%s push", cf.Name())),
					"BuildpackCommand": terminal.CommandColor(fmt.Sprintf("%s buildpacks", cf.Name())),
					"Command":          terminal.CommandColor(fmt.Sprintf("%s logs %s --recent", cf.Name(), app.Name))}))
		} else {
			cmd.ui.Failed(T("{{.Err}}\n\nTIP: use '{{.Command}}' for more information",
				map[string]interface{}{
					"Err":     app.StagingFailedReason,
					"Command": terminal.CommandColor(fmt.Sprintf("%s logs %s --recent", cf.Name(), app.Name))}))
		}
	}

	if time.Since(stagingStartTime) >= cmd.StagingTimeout {
		return false
	}

	return true
}

func (cmd *Start) waitForOneRunningInstance(app models.Application) {
	startupStartTime := time.Now()

	for {
		if time.Since(startupStartTime) > cmd.StartupTimeout {
			cmd.ui.Failed(fmt.Sprintf(T("Start app timeout\n\nTIP: use '{{.Command}}' for more information",
				map[string]interface{}{
					"Command": terminal.CommandColor(fmt.Sprintf("%s logs %s --recent", cf.Name(), app.Name))})))
			return
		}

		count, err := cmd.fetchInstanceCount(app.Guid)
		if err != nil {
			cmd.ui.Wait(cmd.PingerThrottle)
			continue
		}

		cmd.ui.Say(instancesDetails(count))

		if count.running > 0 {
			return
		}

		if count.flapping > 0 || count.crashed > 0 {
			cmd.ui.Failed(fmt.Sprintf(T("Start unsuccessful\n\nTIP: use '{{.Command}}' for more information",
				map[string]interface{}{"Command": terminal.CommandColor(fmt.Sprintf("%s logs %s --recent", cf.Name(), app.Name))})))
			return
		}

		cmd.ui.Wait(cmd.PingerThrottle)
	}
}

type instanceCount struct {
	running         int
	starting        int
	startingDetails map[string]struct{}
	flapping        int
	down            int
	crashed         int
	total           int
}

func (cmd Start) fetchInstanceCount(appGuid string) (instanceCount, error) {
	count := instanceCount{
		startingDetails: make(map[string]struct{}),
	}

	instances, apiErr := cmd.appInstancesRepo.GetInstances(appGuid)
	if apiErr != nil {
		return instanceCount{}, apiErr
	}

	count.total = len(instances)

	for _, inst := range instances {
		switch inst.State {
		case models.InstanceRunning:
			count.running++
		case models.InstanceStarting:
			count.starting++
			if inst.Details != "" {
				count.startingDetails[inst.Details] = struct{}{}
			}
		case models.InstanceFlapping:
			count.flapping++
		case models.InstanceDown:
			count.down++
		case models.InstanceCrashed:
			count.crashed++
		}
	}

	return count, nil
}

func instancesDetails(count instanceCount) string {
	details := []string{fmt.Sprintf(T("{{.RunningCount}} of {{.TotalCount}} instances running",
		map[string]interface{}{"RunningCount": count.running, "TotalCount": count.total}))}

	if count.starting > 0 {
		if len(count.startingDetails) == 0 {
			details = append(details, fmt.Sprintf(T("{{.StartingCount}} starting",
				map[string]interface{}{"StartingCount": count.starting})))
		} else {
			info := []string{}
			for d, _ := range count.startingDetails {
				info = append(info, d)
			}
			sort.Strings(info)
			details = append(details, fmt.Sprintf(T("{{.StartingCount}} starting ({{.Details}})",
				map[string]interface{}{
					"StartingCount": count.starting,
					"Details":       strings.Join(info, ", "),
				})))
		}
	}

	if count.down > 0 {
		details = append(details, fmt.Sprintf(T("{{.DownCount}} down",
			map[string]interface{}{"DownCount": count.down})))
	}

	if count.flapping > 0 {
		details = append(details, fmt.Sprintf(T("{{.FlappingCount}} failing",
			map[string]interface{}{"FlappingCount": count.flapping})))
	}

	if count.crashed > 0 {
		details = append(details, fmt.Sprintf(T("{{.CrashedCount}} crashed",
			map[string]interface{}{"CrashedCount": count.crashed})))
	}

	return strings.Join(details, ", ")
}
<|endoftext|>


<commit_before>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package avalanche

import (
	"github.com/ava-labs/gecko/ids"
	"github.com/ava-labs/gecko/snow"
	"github.com/ava-labs/gecko/snow/choices"
	"github.com/ava-labs/gecko/snow/consensus/snowstorm"
)

// TODO: Implement pruning of accepted decisions.
// To perfectly preserve the protocol, this implementation will need to store
// the hashes of all accepted decisions. It is possible to add a heuristic that
// removes sufficiently old decisions. However, that will need to be analyzed to
// ensure safety. It is doable with a weak syncrony assumption.

// Consensus represents a general avalanche instance that can be used directly
// to process a series of partially ordered elements.
type Consensus interface {
	// Takes in alpha, beta1, beta2, the accepted frontier, the join statuses,
	// the mutation statuses, and the consumer statuses. If accept or reject is
	// called, the status maps should be immediately updated accordingly.
	// Assumes each element in the accepted frontier will return accepted from
	// the join status map.
	Initialize(*snow.Context, Parameters, []Vertex)

	// Returns the parameters that describe this avalanche instance
	Parameters() Parameters

	// Returns true if the transaction is virtuous.
	// That is, no transaction has been added that conflicts with it
	IsVirtuous(snowstorm.Tx) bool

	// Adds a new decision. Assumes the dependencies have already been added.
	// Assumes that mutations don't conflict with themselves. Returns if a
	// critical error has occurred.
	Add(Vertex) error

	// VertexIssued returns true iff Vertex has been added
	VertexIssued(Vertex) bool

	// TxIssued returns true if a vertex containing this transanction has been added
	TxIssued(snowstorm.Tx) bool

	// Returns the set of transaction IDs that are virtuous but not contained in
	// any preferred vertices.
	Orphans() ids.Set

	// Returns a set of vertex IDs that were virtuous at the last update.
	Virtuous() ids.Set

	// Returns a set of vertex IDs that are preferred
	Preferences() ids.Set

	// RecordPoll collects the results of a network poll. If a result has not
	// been added, the result is dropped. Returns if a critical error has
	// occurred.
	RecordPoll(ids.UniqueBag) error

	// Quiesce returns true iff all vertices that have been added but not been accepted or rejected are rogue.
	// Note, it is possible that after returning quiesce, a new decision may be added such
	// that this instance should no longer quiesce.
	Quiesce() bool

	// Finalized returns true if all transactions that have been added have been
	// finalized. Note, it is possible that after returning finalized, a new
	// decision may be added such that this instance is no longer finalized.
	Finalized() bool
}

// Vertex is a collection of multiple transactions tied to other vertices
type Vertex interface {
	choices.Decidable

	// Returns the vertices this vertex depends on
	Parents() []Vertex

	// Returns the height of this vertex
	Height() uint64

	// Returns a series of state transitions to be performed on acceptance
	Txs() []snowstorm.Tx

	Bytes() []byte
}
<commit_msg>defined vertex height<commit_after>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package avalanche

import (
	"github.com/ava-labs/gecko/ids"
	"github.com/ava-labs/gecko/snow"
	"github.com/ava-labs/gecko/snow/choices"
	"github.com/ava-labs/gecko/snow/consensus/snowstorm"
)

// TODO: Implement pruning of accepted decisions.
// To perfectly preserve the protocol, this implementation will need to store
// the hashes of all accepted decisions. It is possible to add a heuristic that
// removes sufficiently old decisions. However, that will need to be analyzed to
// ensure safety. It is doable with a weak syncrony assumption.

// Consensus represents a general avalanche instance that can be used directly
// to process a series of partially ordered elements.
type Consensus interface {
	// Takes in alpha, beta1, beta2, the accepted frontier, the join statuses,
	// the mutation statuses, and the consumer statuses. If accept or reject is
	// called, the status maps should be immediately updated accordingly.
	// Assumes each element in the accepted frontier will return accepted from
	// the join status map.
	Initialize(*snow.Context, Parameters, []Vertex)

	// Returns the parameters that describe this avalanche instance
	Parameters() Parameters

	// Returns true if the transaction is virtuous.
	// That is, no transaction has been added that conflicts with it
	IsVirtuous(snowstorm.Tx) bool

	// Adds a new decision. Assumes the dependencies have already been added.
	// Assumes that mutations don't conflict with themselves. Returns if a
	// critical error has occurred.
	Add(Vertex) error

	// VertexIssued returns true iff Vertex has been added
	VertexIssued(Vertex) bool

	// TxIssued returns true if a vertex containing this transanction has been added
	TxIssued(snowstorm.Tx) bool

	// Returns the set of transaction IDs that are virtuous but not contained in
	// any preferred vertices.
	Orphans() ids.Set

	// Returns a set of vertex IDs that were virtuous at the last update.
	Virtuous() ids.Set

	// Returns a set of vertex IDs that are preferred
	Preferences() ids.Set

	// RecordPoll collects the results of a network poll. If a result has not
	// been added, the result is dropped. Returns if a critical error has
	// occurred.
	RecordPoll(ids.UniqueBag) error

	// Quiesce returns true iff all vertices that have been added but not been accepted or rejected are rogue.
	// Note, it is possible that after returning quiesce, a new decision may be added such
	// that this instance should no longer quiesce.
	Quiesce() bool

	// Finalized returns true if all transactions that have been added have been
	// finalized. Note, it is possible that after returning finalized, a new
	// decision may be added such that this instance is no longer finalized.
	Finalized() bool
}

// Vertex is a collection of multiple transactions tied to other vertices
type Vertex interface {
	choices.Decidable

	// Returns the vertices this vertex depends on
	Parents() []Vertex

	// Returns the height of this vertex. A vertex's height is defined by one
	// greater than the maximum height of the parents.
	Height() uint64

	// Returns a series of state transitions to be performed on acceptance
	Txs() []snowstorm.Tx

	Bytes() []byte
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
	"log"
	"time"

	"github.com/satori/go.uuid"

)

func (s *supervisor) permission(q *msg.Request) {
	result := msg.Result{Type: `supervisor`, Action: `permission`}

	log.Printf(LogStrReq, q.Type, fmt.Sprintf("%s/%s", q.Action, q.Super.Action), q.User, q.RemoteAddr)

	if s.readonly && (q.Super.Action == `add` || q.Super.Action == `delete`) {
		result.Conflict(fmt.Errorf(`Readonly instance`))
		goto dispatch
	}

	switch q.Super.Action {
	case `list`:
		fallthrough
	case `search/name`:
		fallthrough
	case `show`:
		s.permission_read(q)
		return
	case `add`:
		fallthrough
	case `delete`:
		s.permission_write(q)
		return
	}

dispatch:
	q.Reply <- result
}

func (s *supervisor) permission_read(q *msg.Request) {
	result := msg.Result{Type: `supervisor`, Action: `permission`}
	var (
		rows                     *sql.Rows
		err                      error
		id, name, category, user string
		ts                       time.Time
	)

	switch q.Super.Action {
	case `list`:
		fallthrough
	case `search/name`:
		result.Permission = []proto.Permission{}
		switch q.Super.Action {
		case `list`:
			rows, err = s.stmt_ListPermission.Query()
		case `search/name`:
			rows, err = s.stmt_SearchPerm.Query(q.Permission.Name)
		}
		if err != nil {
			result.ServerError(err)
			goto dispatch
		}
		defer rows.Close()

		for rows.Next() {
			if err = rows.Scan(
				&id,
				&name,
			); err != nil {
				result.ServerError(err)
				result.Clear(q.Action)
				goto dispatch
			}
			result.Permission = append(result.Permission,
				proto.Permission{Id: id, Name: name})
		}
		if err = rows.Err(); err != nil {
			result.ServerError(err)
			result.Clear(q.Action)
		}
	case `show`:
		if err = s.stmt_ShowPermission.QueryRow(q.Permission.Name).Scan(
			&id,
			&name,
			&category,
			&user,
			&ts,
		); err == sql.ErrNoRows {
			result.NotFound(err)
			goto dispatch
		} else if err != nil {
			result.ServerError(err)
			goto dispatch
		}
		result.Permission = []proto.Permission{proto.Permission{
			Id:       id,
			Name:     name,
			Category: category,
			Details: &proto.PermissionDetails{
				CreatedAt: ts.Format(rfc3339Milli),
				CreatedBy: user,
			},
		}}
	}

dispatch:
	q.Reply <- result
}

func (s *supervisor) permission_write(q *msg.Request) {
	result := msg.Result{Type: `supervisor`, Action: `permission`}
	userUUID, ok := s.id_user_rev.get(q.User)
	if !ok {
		userUUID = `00000000-0000-0000-0000-000000000000`
	}

	var (
		res sql.Result
		err error
		id  string
	)

	switch q.Super.Action {
	case `add`:
		q.Permission.Id = uuid.NewV4().String()
		res, err = s.stmt_AddPermission.Exec(
			q.Permission.Id,
			q.Permission.Name,
			q.Permission.Category,
			userUUID,
		)
	case `delete`:
		if id, ok = s.id_permission.get(q.Permission.Name); !ok {
			result.NotFound(fmt.Errorf(`Supervisor: unknown`))
			goto dispatch
		}
		res, err = s.stmt_DelPermission.Exec(
			id,
		)
	}
	if err != nil {
		result.ServerError(err)
		goto dispatch
	}

	if result.RowCnt(res.RowsAffected()) {
		result.Permission = []proto.Permission{q.Permission}
		// keep lookup maps in sync
		switch q.Super.Action {
		case `add`:
			s.id_permission.insert(q.Permission.Name, q.Permission.Id)
		case `delete`:
			s.id_permission.remove(q.Permission.Name)
		}
	}

dispatch:
	q.Reply <- result
}

// vim: ts=4 sw=4 sts=4 noet fenc=utf-8 ffs=unix
<commit_msg>FIX: set supervisor/permission result code for non-error<commit_after>package main

import (
	"database/sql"
	"fmt"
	"log"
	"time"

	"github.com/satori/go.uuid"

)

func (s *supervisor) permission(q *msg.Request) {
	result := msg.Result{Type: `supervisor`, Action: `permission`}

	log.Printf(LogStrReq, q.Type, fmt.Sprintf("%s/%s", q.Action, q.Super.Action), q.User, q.RemoteAddr)

	if s.readonly && (q.Super.Action == `add` || q.Super.Action == `delete`) {
		result.Conflict(fmt.Errorf(`Readonly instance`))
		goto dispatch
	}

	switch q.Super.Action {
	case `list`:
		fallthrough
	case `search/name`:
		fallthrough
	case `show`:
		s.permission_read(q)
		return
	case `add`:
		fallthrough
	case `delete`:
		s.permission_write(q)
		return
	}

dispatch:
	q.Reply <- result
}

func (s *supervisor) permission_read(q *msg.Request) {
	result := msg.Result{Type: `supervisor`, Action: `permission`}
	var (
		rows                     *sql.Rows
		err                      error
		id, name, category, user string
		ts                       time.Time
	)

	switch q.Super.Action {
	case `list`:
		fallthrough
	case `search/name`:
		result.Permission = []proto.Permission{}
		switch q.Super.Action {
		case `list`:
			rows, err = s.stmt_ListPermission.Query()
		case `search/name`:
			rows, err = s.stmt_SearchPerm.Query(q.Permission.Name)
		}
		if err != nil {
			result.ServerError(err)
			goto dispatch
		}
		defer rows.Close()

		for rows.Next() {
			if err = rows.Scan(
				&id,
				&name,
			); err != nil {
				result.ServerError(err)
				result.Clear(q.Action)
				goto dispatch
			}
			result.Permission = append(result.Permission,
				proto.Permission{Id: id, Name: name})
		}
		if err = rows.Err(); err != nil {
			result.ServerError(err)
			result.Clear(q.Action)
		}
		result.OK()
	case `show`:
		if err = s.stmt_ShowPermission.QueryRow(q.Permission.Name).Scan(
			&id,
			&name,
			&category,
			&user,
			&ts,
		); err == sql.ErrNoRows {
			result.NotFound(err)
			goto dispatch
		} else if err != nil {
			result.ServerError(err)
			goto dispatch
		}
		result.Permission = []proto.Permission{proto.Permission{
			Id:       id,
			Name:     name,
			Category: category,
			Details: &proto.PermissionDetails{
				CreatedAt: ts.Format(rfc3339Milli),
				CreatedBy: user,
			},
		}}
		result.OK()
	}

dispatch:
	q.Reply <- result
}

func (s *supervisor) permission_write(q *msg.Request) {
	result := msg.Result{Type: `supervisor`, Action: `permission`}
	userUUID, ok := s.id_user_rev.get(q.User)
	if !ok {
		userUUID = `00000000-0000-0000-0000-000000000000`
	}

	var (
		res sql.Result
		err error
		id  string
	)

	switch q.Super.Action {
	case `add`:
		q.Permission.Id = uuid.NewV4().String()
		res, err = s.stmt_AddPermission.Exec(
			q.Permission.Id,
			q.Permission.Name,
			q.Permission.Category,
			userUUID,
		)
	case `delete`:
		if id, ok = s.id_permission.get(q.Permission.Name); !ok {
			result.NotFound(fmt.Errorf(`Supervisor: unknown`))
			goto dispatch
		}
		res, err = s.stmt_DelPermission.Exec(
			id,
		)
	}
	if err != nil {
		result.ServerError(err)
		goto dispatch
	}

	if result.RowCnt(res.RowsAffected()) {
		result.Permission = []proto.Permission{q.Permission}
		// keep lookup maps in sync
		switch q.Super.Action {
		case `add`:
			s.id_permission.insert(q.Permission.Name, q.Permission.Id)
		case `delete`:
			s.id_permission.remove(q.Permission.Name)
		}
	}

dispatch:
	q.Reply <- result
}

// vim: ts=4 sw=4 sts=4 noet fenc=utf-8 ffs=unix
<|endoftext|>


<commit_before>package processing

import (
	"fmt"
	"regexp"
	"strconv"
	"time"

	"github.com/pkg/errors"
)

// Logsync Interfface to use for uploading to different services
type Logsync interface {
	Process(bool) error
}

// FileMatch is used to keep track of path, matched strings, and rotated Time
type FileMatch struct {
	path        string
	matches     map[string]string
	rotatedTime time.Time
}

func NewFileMatch(path string, matches map[string]string) (*FileMatch, error) {
	f := &FileMatch{
		path:    path,
		matches: matches,
	}

	tm, err := f.ParseTime()
	if err != nil {
		return f, errors.Wrapf(err, "Failed to parse rotated time for %s", path)
	}
	f.rotatedTime = tm
	return f, nil
}

// Parse the Key from matches
func (f *FileMatch) parseInt(key string) (int, error) {
	extras, ok := f.matches[key]
	if !ok {
		return -1, fmt.Errorf("No field(%s) matched, unexpected filename %s", key, f.path)
	}
	extraInt, err := strconv.Atoi(extras)
	if err != nil {
		return -1, errors.Wrapf(err, "Cannot parse the field(%s) as int", key)
	}
	return extraInt, nil
}

// ParseTime relies on the matches map containing
// year, month, day and extras field
// if extras can be parsed as Unix timestamp, that's picked
// otherwise the last field is checked if it's an 0 <= hour <= 23
// the time is constructed if
func (f *FileMatch) ParseTime() (time.Time, error) {
	extraInt, err := f.parseInt("extra")
	if err != nil {
		return time.Time{}, errors.Wrapf(err, "Failed to parse for extra")
	}
	day, err := f.parseInt("day")
	if err != nil {
		return time.Time{}, errors.Wrapf(err, "Failed to parse for day")
	}
	month, err := f.parseInt("month")
	if err != nil {
		return time.Time{}, errors.Wrapf(err, "Failed to parse for month")
	}
	year, err := f.parseInt("year")
	if err != nil {
		return time.Time{}, errors.Wrapf(err, "Failed to parse for year")
	}
	if extraInt >= 0 && extraInt <= 23 {
		hour := extraInt
		return time.Date(year, time.Month(month), day, hour, 0, 0, 0, time.UTC), nil
	}
	// assume it's unix time
	tm := time.Unix(int64(extraInt), 0)
	if tm.Year() != year || tm.Month() != time.Month(month) || tm.Day() != day {
		return time.Time{}, errors.Wrapf(err, "Unix time doesn't match the Y/M/D, probably invalid file pattern")
	}
	return tm, nil

}

func matchFileNames(name string, matchPattern *regexp.Regexp) map[string]string {
	matches := matchPattern.FindAllStringSubmatch(name, -1)
	if len(matches) == 0 {
		return nil
	}
	subExps := matchPattern.SubexpNames()
	elements := matches[0]
	md := map[string]string{}
	for i := 1; i < len(elements); i++ {
		md[subExps[i]] = elements[i]
	}
	return md
}
<commit_msg>comment for golint<commit_after>package processing

import (
	"fmt"
	"regexp"
	"strconv"
	"time"

	"github.com/pkg/errors"
)

// Logsync Interfface to use for uploading to different services
type Logsync interface {
	Process(bool) error
}

// FileMatch is used to keep track of path, matched strings, and rotated Time
type FileMatch struct {
	path        string
	matches     map[string]string
	rotatedTime time.Time
}

// NewFileMatch takes the path and the matched map, and parses the timestamp from the matches
func NewFileMatch(path string, matches map[string]string) (*FileMatch, error) {
	f := &FileMatch{
		path:    path,
		matches: matches,
	}

	tm, err := f.ParseTime()
	if err != nil {
		return f, errors.Wrapf(err, "Failed to parse rotated time for %s", path)
	}
	f.rotatedTime = tm
	return f, nil
}

// Parse the Key from matches
func (f *FileMatch) parseInt(key string) (int, error) {
	extras, ok := f.matches[key]
	if !ok {
		return -1, fmt.Errorf("No field(%s) matched, unexpected filename %s", key, f.path)
	}
	extraInt, err := strconv.Atoi(extras)
	if err != nil {
		return -1, errors.Wrapf(err, "Cannot parse the field(%s) as int", key)
	}
	return extraInt, nil
}

// ParseTime relies on the matches map containing
// year, month, day and extras field
// if extras can be parsed as Unix timestamp, that's picked
// otherwise the last field is checked if it's an 0 <= hour <= 23
// the time is constructed if
func (f *FileMatch) ParseTime() (time.Time, error) {
	extraInt, err := f.parseInt("extra")
	if err != nil {
		return time.Time{}, errors.Wrapf(err, "Failed to parse for extra")
	}
	day, err := f.parseInt("day")
	if err != nil {
		return time.Time{}, errors.Wrapf(err, "Failed to parse for day")
	}
	month, err := f.parseInt("month")
	if err != nil {
		return time.Time{}, errors.Wrapf(err, "Failed to parse for month")
	}
	year, err := f.parseInt("year")
	if err != nil {
		return time.Time{}, errors.Wrapf(err, "Failed to parse for year")
	}
	if extraInt >= 0 && extraInt <= 23 {
		hour := extraInt
		return time.Date(year, time.Month(month), day, hour, 0, 0, 0, time.UTC), nil
	}
	// assume it's unix time
	tm := time.Unix(int64(extraInt), 0)
	if tm.Year() != year || tm.Month() != time.Month(month) || tm.Day() != day {
		return time.Time{}, errors.Wrapf(err, "Unix time doesn't match the Y/M/D, probably invalid file pattern")
	}
	return tm, nil

}

func matchFileNames(name string, matchPattern *regexp.Regexp) map[string]string {
	matches := matchPattern.FindAllStringSubmatch(name, -1)
	if len(matches) == 0 {
		return nil
	}
	subExps := matchPattern.SubexpNames()
	elements := matches[0]
	md := map[string]string{}
	for i := 1; i < len(elements); i++ {
		md[subExps[i]] = elements[i]
	}
	return md
}
<|endoftext|>


<commit_before>// Copyright 2017-2019 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package launch

import (
	"context"
	"fmt"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/cilium/cilium/api/v1/models"
	"github.com/cilium/cilium/pkg/datapath/linux/route"
	"github.com/cilium/cilium/pkg/defaults"
	"github.com/cilium/cilium/pkg/endpoint"
	"github.com/cilium/cilium/pkg/endpoint/connector"
	"github.com/cilium/cilium/pkg/endpoint/regeneration"
	healthDefaults "github.com/cilium/cilium/pkg/health/defaults"
	"github.com/cilium/cilium/pkg/health/probe"
	"github.com/cilium/cilium/pkg/labels"
	"github.com/cilium/cilium/pkg/launcher"
	"github.com/cilium/cilium/pkg/logging/logfields"
	"github.com/cilium/cilium/pkg/metrics"
	"github.com/cilium/cilium/pkg/mtu"
	"github.com/cilium/cilium/pkg/netns"
	"github.com/cilium/cilium/pkg/node"
	"github.com/cilium/cilium/pkg/option"
	"github.com/cilium/cilium/pkg/pidfile"

	"github.com/containernetworking/plugins/pkg/ns"
	"github.com/vishvananda/netlink"
	"golang.org/x/sys/unix"
)

const (
	ciliumHealth = "cilium-health"
	netNSName    = "cilium-health"
	binaryName   = "cilium-health-responder"
)

var (
	// vethName is the host-side veth link device name for cilium-health EP
	// (veth mode only).
	vethName = "lxc_health"

	// legacyVethName is the host-side cilium-health EP device name used in
	// older Cilium versions. Used for removal only.
	legacyVethName = "cilium_health"

	// epIfaceName is the endpoint-side link device name for cilium-health.
	epIfaceName = "cilium"

	// PidfilePath
	PidfilePath = "health-endpoint.pid"

	// LaunchTime is the expected time within which the health endpoint
	// should be able to be successfully run and its BPF program attached.
	LaunchTime = 30 * time.Second
)

func configureHealthRouting(netns, dev string, addressing *models.NodeAddressing, mtuConfig mtu.Configuration) error {
	routes := []route.Route{}

	if option.Config.EnableIPv4 {
		v4Routes, err := connector.IPv4Routes(addressing, mtuConfig.GetRouteMTU())
		if err == nil {
			routes = append(routes, v4Routes...)
		} else {
			log.Debugf("Couldn't get IPv4 routes for health routing")
		}
	}

	if option.Config.EnableIPv6 {
		v6Routes, err := connector.IPv6Routes(addressing, mtuConfig.GetRouteMTU())
		if err != nil {
			return fmt.Errorf("Failed to get IPv6 routes")
		}
		routes = append(routes, v6Routes...)
	}

	prog := "ip"
	args := []string{"netns", "exec", netns, "bash", "-c"}
	routeCmds := []string{}
	for _, rt := range routes {
		cmd := strings.Join(rt.ToIPCommand(dev), " ")
		log.WithField("netns", netns).WithField("command", cmd).Debug("Adding route")
		routeCmds = append(routeCmds, cmd)
	}
	cmd := strings.Join(routeCmds, " && ")
	args = append(args, cmd)

	log.Debugf("Running \"%s %+v\"", prog, args)
	out, err := exec.Command(prog, args...).CombinedOutput()
	if err == nil && len(out) > 0 {
		log.Warn(out)
	}

	return err
}

func configureHealthInterface(netNS ns.NetNS, ifName string, ip4Addr, ip6Addr *net.IPNet) error {
	return netNS.Do(func(_ ns.NetNS) error {
		link, err := netlink.LinkByName(ifName)
		if err != nil {
			return err
		}

		if ip6Addr != nil {
			if err = netlink.AddrAdd(link, &netlink.Addr{IPNet: ip6Addr}); err != nil {
				return err
			}
		}

		if ip4Addr != nil {
			if err = netlink.AddrAdd(link, &netlink.Addr{IPNet: ip4Addr}); err != nil {
				return err
			}
		}

		if err = netlink.LinkSetUp(link); err != nil {
			return err
		}

		lo, err := netlink.LinkByName("lo")
		if err != nil {
			return err
		}

		if err = netlink.LinkSetUp(lo); err != nil {
			return err
		}

		return nil
	})
}

// Client wraps a client to a specific cilium-health endpoint instance, to
// provide convenience methods such as PingEndpoint().
type Client struct {
	*probe.Client
}

// PingEndpoint attempts to make an API ping request to the local cilium-health
// endpoint, and returns whether this was successful.
func (c *Client) PingEndpoint() error {
	return c.Client.GetHello()
}

// KillEndpoint attempts to kill any existing cilium-health endpoint if it
// exists.
//
// This is intended to be invoked in multiple situations:
// * The health endpoint has never been run before
// * The health endpoint was run during a previous run of the Cilium agent
// * The health endpoint crashed during the current run of the Cilium agent
//   and needs to be cleaned up before it is restarted.
func KillEndpoint() {
	path := filepath.Join(option.Config.StateDir, PidfilePath)
	scopedLog := log.WithField(logfields.PIDFile, path)
	scopedLog.Debug("Killing old health endpoint process")
	pid, err := pidfile.Kill(path)
	if err != nil {
		scopedLog.WithError(err).Warning("Failed to kill cilium-health-responder")
	} else if pid != 0 {
		scopedLog.WithField(logfields.PID, pid).Debug("Killed endpoint process")
	}
}

// CleanupEndpoint cleans up remaining resources associated with the health
// endpoint.
//
// This is expected to be called after the process is killed and the endpoint
// is removed from the endpointmanager.
func CleanupEndpoint() {
	// Removes the interfaces used for the endpoint process, followed by the
	// deletion of the health namespace itself. The removal of the interfaces
	// is needed, because network namespace removal does not always trigger the
	// deletion of associated interfaces immediately (e.g. when a process in the
	// namespace marked for deletion has not yet been terminated).
	switch option.Config.DatapathMode {
	case option.DatapathModeVeth:
		for _, iface := range []string{legacyVethName, vethName} {
			scopedLog := log.WithField(logfields.Veth, iface)
			if link, err := netlink.LinkByName(iface); err == nil {
				err = netlink.LinkDel(link)
				if err != nil {
					scopedLog.WithError(err).Info("Couldn't delete cilium-health veth device")
				}
			} else {
				scopedLog.WithError(err).Debug("Didn't find existing device")
			}
		}
	case option.DatapathModeIpvlan:
		if err := netns.RemoveIfFromNetNSWithNameIfBothExist(netNSName, epIfaceName); err != nil {
			log.WithError(err).WithField(logfields.Ipvlan, epIfaceName).
				Info("Couldn't delete cilium-health ipvlan slave device")
		}
	}

	if err := netns.RemoveNetNSWithName(netNSName); err != nil {
		log.WithError(err).Debug("Unable to remove cilium-health namespace")
	}
}

// EndpointAdder is any type which adds an endpoint to be managed by Cilium.
type EndpointAdder interface {
	AddEndpoint(owner regeneration.Owner, ep *endpoint.Endpoint, reason string) error
}

// LaunchAsEndpoint launches the cilium-health agent in a nested network
// namespace and attaches it to Cilium the same way as any other endpoint,
// but with special reserved labels.
//
// CleanupEndpoint() must be called before calling LaunchAsEndpoint() to ensure
// cleanup of prior cilium-health endpoint instances.
func LaunchAsEndpoint(baseCtx context.Context, owner regeneration.Owner, n *node.Node, mtuConfig mtu.Configuration, epMgr EndpointAdder) (*Client, error) {
	var (
		cmd  = launcher.Launcher{}
		info = &models.EndpointChangeRequest{
			ContainerName: ciliumHealth,
			State:         models.EndpointStateWaitingForIdentity,
			Addressing:    &models.AddressPair{},
		}
		healthIP               net.IP
		ip4Address, ip6Address *net.IPNet
	)

	if n.IPv4HealthIP != nil {
		healthIP = n.IPv4HealthIP
		info.Addressing.IPV4 = healthIP.String()
		ip4Address = &net.IPNet{IP: healthIP, Mask: defaults.ContainerIPv4Mask}
	}

	if n.IPv6HealthIP != nil {
		healthIP = n.IPv6HealthIP
		info.Addressing.IPV6 = healthIP.String()
		ip6Address = &net.IPNet{IP: healthIP, Mask: defaults.ContainerIPv6Mask}
	}

	netNS, err := netns.ReplaceNetNSWithName(netNSName)
	if err != nil {
		return nil, err
	}

	switch option.Config.DatapathMode {
	case option.DatapathModeVeth:
		_, epLink, err := connector.SetupVethWithNames(vethName, epIfaceName, mtuConfig.GetDeviceMTU(), info)
		if err != nil {
			return nil, fmt.Errorf("Error while creating veth: %s", err)
		}

		if err = netlink.LinkSetNsFd(*epLink, int(netNS.Fd())); err != nil {
			return nil, fmt.Errorf("failed to move device %q to health namespace: %s", epIfaceName, err)
		}

	case option.DatapathModeIpvlan:
		mapFD, err := connector.CreateAndSetupIpvlanSlave("",
			epIfaceName, netNS, mtuConfig.GetDeviceMTU(),
			option.Config.Ipvlan.MasterDeviceIndex,
			option.Config.Ipvlan.OperationMode, info)
		if err != nil {
			if errDel := netns.RemoveNetNSWithName(netNSName); errDel != nil {
				log.WithError(errDel).WithField(logfields.NetNSName, netNSName).
					Warning("Unable to remove network namespace")
			}
			return nil, err
		}
		defer unix.Close(mapFD)

	}

	if err = configureHealthInterface(netNS, epIfaceName, ip4Address, ip6Address); err != nil {
		return nil, fmt.Errorf("failed configure health interface %q: %s", epIfaceName, err)
	}

	pidfile := filepath.Join(option.Config.StateDir, PidfilePath)
	prog := "ip"
	args := []string{"netns", "exec", netNSName, binaryName, "--pidfile", pidfile}
	cmd.SetTarget(prog)
	cmd.SetArgs(args)
	log.Infof("Spawning health endpoint with command %q %q", prog, args)
	if err := cmd.Run(); err != nil {
		return nil, err
	}

	// Create the endpoint
	ep, err := endpoint.NewEndpointFromChangeModel(owner, info)
	if err != nil {
		return nil, fmt.Errorf("Error while creating endpoint model: %s", err)
	}

	// Wait until the cilium-health endpoint is running before setting up routes
	deadline := time.Now().Add(1 * time.Minute)
	for {
		if _, err := os.Stat(pidfile); err == nil {
			log.WithField("pidfile", pidfile).Debug("cilium-health agent running")
			break
		} else if time.Now().After(deadline) {
			return nil, fmt.Errorf("Endpoint failed to run: %s", err)
		} else {
			time.Sleep(1 * time.Second)
		}
	}

	// Set up the endpoint routes
	hostAddressing := node.GetNodeAddressing()
	if err = configureHealthRouting(info.ContainerName, epIfaceName, hostAddressing, mtuConfig); err != nil {
		return nil, fmt.Errorf("Error while configuring routes: %s", err)
	}

	if err := epMgr.AddEndpoint(owner, ep, "Create cilium-health endpoint"); err != nil {
		return nil, fmt.Errorf("Error while adding endpoint: %s", err)
	}

	if err := ep.LockAlive(); err != nil {
		return nil, err
	}
	ep.PinDatapathMap()
	ep.Unlock()

	// Give the endpoint a security identity
	ctx, cancel := context.WithTimeout(baseCtx, LaunchTime)
	defer cancel()
	ep.UpdateLabels(ctx, labels.LabelHealth, nil, true)

	// Initialize the health client to talk to this instance. This is why
	// the caller must limit usage of this package to a single goroutine.
	client, err := probe.NewClient("http://" + net.JoinHostPort(healthIP.String(), fmt.Sprintf("%d", healthDefaults.HTTPPathPort)))
	if err != nil {
		return nil, fmt.Errorf("Cannot establish connection to health endpoint: %s", err)
	}
	metrics.SubprocessStart.WithLabelValues(ciliumHealth).Inc()

	return &Client{Client: client}, nil
}
<commit_msg>health: Prefer contacting health EP over IPv4<commit_after>// Copyright 2017-2019 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package launch

import (
	"context"
	"fmt"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/cilium/cilium/api/v1/models"
	"github.com/cilium/cilium/pkg/datapath/linux/route"
	"github.com/cilium/cilium/pkg/defaults"
	"github.com/cilium/cilium/pkg/endpoint"
	"github.com/cilium/cilium/pkg/endpoint/connector"
	"github.com/cilium/cilium/pkg/endpoint/regeneration"
	healthDefaults "github.com/cilium/cilium/pkg/health/defaults"
	"github.com/cilium/cilium/pkg/health/probe"
	"github.com/cilium/cilium/pkg/labels"
	"github.com/cilium/cilium/pkg/launcher"
	"github.com/cilium/cilium/pkg/logging/logfields"
	"github.com/cilium/cilium/pkg/metrics"
	"github.com/cilium/cilium/pkg/mtu"
	"github.com/cilium/cilium/pkg/netns"
	"github.com/cilium/cilium/pkg/node"
	"github.com/cilium/cilium/pkg/option"
	"github.com/cilium/cilium/pkg/pidfile"

	"github.com/containernetworking/plugins/pkg/ns"
	"github.com/vishvananda/netlink"
	"golang.org/x/sys/unix"
)

const (
	ciliumHealth = "cilium-health"
	netNSName    = "cilium-health"
	binaryName   = "cilium-health-responder"
)

var (
	// vethName is the host-side veth link device name for cilium-health EP
	// (veth mode only).
	vethName = "lxc_health"

	// legacyVethName is the host-side cilium-health EP device name used in
	// older Cilium versions. Used for removal only.
	legacyVethName = "cilium_health"

	// epIfaceName is the endpoint-side link device name for cilium-health.
	epIfaceName = "cilium"

	// PidfilePath
	PidfilePath = "health-endpoint.pid"

	// LaunchTime is the expected time within which the health endpoint
	// should be able to be successfully run and its BPF program attached.
	LaunchTime = 30 * time.Second
)

func configureHealthRouting(netns, dev string, addressing *models.NodeAddressing, mtuConfig mtu.Configuration) error {
	routes := []route.Route{}

	if option.Config.EnableIPv4 {
		v4Routes, err := connector.IPv4Routes(addressing, mtuConfig.GetRouteMTU())
		if err == nil {
			routes = append(routes, v4Routes...)
		} else {
			log.Debugf("Couldn't get IPv4 routes for health routing")
		}
	}

	if option.Config.EnableIPv6 {
		v6Routes, err := connector.IPv6Routes(addressing, mtuConfig.GetRouteMTU())
		if err != nil {
			return fmt.Errorf("Failed to get IPv6 routes")
		}
		routes = append(routes, v6Routes...)
	}

	prog := "ip"
	args := []string{"netns", "exec", netns, "bash", "-c"}
	routeCmds := []string{}
	for _, rt := range routes {
		cmd := strings.Join(rt.ToIPCommand(dev), " ")
		log.WithField("netns", netns).WithField("command", cmd).Debug("Adding route")
		routeCmds = append(routeCmds, cmd)
	}
	cmd := strings.Join(routeCmds, " && ")
	args = append(args, cmd)

	log.Debugf("Running \"%s %+v\"", prog, args)
	out, err := exec.Command(prog, args...).CombinedOutput()
	if err == nil && len(out) > 0 {
		log.Warn(out)
	}

	return err
}

func configureHealthInterface(netNS ns.NetNS, ifName string, ip4Addr, ip6Addr *net.IPNet) error {
	return netNS.Do(func(_ ns.NetNS) error {
		link, err := netlink.LinkByName(ifName)
		if err != nil {
			return err
		}

		if ip6Addr != nil {
			if err = netlink.AddrAdd(link, &netlink.Addr{IPNet: ip6Addr}); err != nil {
				return err
			}
		}

		if ip4Addr != nil {
			if err = netlink.AddrAdd(link, &netlink.Addr{IPNet: ip4Addr}); err != nil {
				return err
			}
		}

		if err = netlink.LinkSetUp(link); err != nil {
			return err
		}

		lo, err := netlink.LinkByName("lo")
		if err != nil {
			return err
		}

		if err = netlink.LinkSetUp(lo); err != nil {
			return err
		}

		return nil
	})
}

// Client wraps a client to a specific cilium-health endpoint instance, to
// provide convenience methods such as PingEndpoint().
type Client struct {
	*probe.Client
}

// PingEndpoint attempts to make an API ping request to the local cilium-health
// endpoint, and returns whether this was successful.
func (c *Client) PingEndpoint() error {
	return c.Client.GetHello()
}

// KillEndpoint attempts to kill any existing cilium-health endpoint if it
// exists.
//
// This is intended to be invoked in multiple situations:
// * The health endpoint has never been run before
// * The health endpoint was run during a previous run of the Cilium agent
// * The health endpoint crashed during the current run of the Cilium agent
//   and needs to be cleaned up before it is restarted.
func KillEndpoint() {
	path := filepath.Join(option.Config.StateDir, PidfilePath)
	scopedLog := log.WithField(logfields.PIDFile, path)
	scopedLog.Debug("Killing old health endpoint process")
	pid, err := pidfile.Kill(path)
	if err != nil {
		scopedLog.WithError(err).Warning("Failed to kill cilium-health-responder")
	} else if pid != 0 {
		scopedLog.WithField(logfields.PID, pid).Debug("Killed endpoint process")
	}
}

// CleanupEndpoint cleans up remaining resources associated with the health
// endpoint.
//
// This is expected to be called after the process is killed and the endpoint
// is removed from the endpointmanager.
func CleanupEndpoint() {
	// Removes the interfaces used for the endpoint process, followed by the
	// deletion of the health namespace itself. The removal of the interfaces
	// is needed, because network namespace removal does not always trigger the
	// deletion of associated interfaces immediately (e.g. when a process in the
	// namespace marked for deletion has not yet been terminated).
	switch option.Config.DatapathMode {
	case option.DatapathModeVeth:
		for _, iface := range []string{legacyVethName, vethName} {
			scopedLog := log.WithField(logfields.Veth, iface)
			if link, err := netlink.LinkByName(iface); err == nil {
				err = netlink.LinkDel(link)
				if err != nil {
					scopedLog.WithError(err).Info("Couldn't delete cilium-health veth device")
				}
			} else {
				scopedLog.WithError(err).Debug("Didn't find existing device")
			}
		}
	case option.DatapathModeIpvlan:
		if err := netns.RemoveIfFromNetNSWithNameIfBothExist(netNSName, epIfaceName); err != nil {
			log.WithError(err).WithField(logfields.Ipvlan, epIfaceName).
				Info("Couldn't delete cilium-health ipvlan slave device")
		}
	}

	if err := netns.RemoveNetNSWithName(netNSName); err != nil {
		log.WithError(err).Debug("Unable to remove cilium-health namespace")
	}
}

// EndpointAdder is any type which adds an endpoint to be managed by Cilium.
type EndpointAdder interface {
	AddEndpoint(owner regeneration.Owner, ep *endpoint.Endpoint, reason string) error
}

// LaunchAsEndpoint launches the cilium-health agent in a nested network
// namespace and attaches it to Cilium the same way as any other endpoint,
// but with special reserved labels.
//
// CleanupEndpoint() must be called before calling LaunchAsEndpoint() to ensure
// cleanup of prior cilium-health endpoint instances.
func LaunchAsEndpoint(baseCtx context.Context, owner regeneration.Owner, n *node.Node, mtuConfig mtu.Configuration, epMgr EndpointAdder) (*Client, error) {
	var (
		cmd  = launcher.Launcher{}
		info = &models.EndpointChangeRequest{
			ContainerName: ciliumHealth,
			State:         models.EndpointStateWaitingForIdentity,
			Addressing:    &models.AddressPair{},
		}
		healthIP               net.IP
		ip4Address, ip6Address *net.IPNet
	)

	if n.IPv6HealthIP != nil {
		healthIP = n.IPv6HealthIP
		info.Addressing.IPV6 = healthIP.String()
		ip6Address = &net.IPNet{IP: healthIP, Mask: defaults.ContainerIPv6Mask}
	}
	if n.IPv4HealthIP != nil {
		healthIP = n.IPv4HealthIP
		info.Addressing.IPV4 = healthIP.String()
		ip4Address = &net.IPNet{IP: healthIP, Mask: defaults.ContainerIPv4Mask}
	}

	netNS, err := netns.ReplaceNetNSWithName(netNSName)
	if err != nil {
		return nil, err
	}

	switch option.Config.DatapathMode {
	case option.DatapathModeVeth:
		_, epLink, err := connector.SetupVethWithNames(vethName, epIfaceName, mtuConfig.GetDeviceMTU(), info)
		if err != nil {
			return nil, fmt.Errorf("Error while creating veth: %s", err)
		}

		if err = netlink.LinkSetNsFd(*epLink, int(netNS.Fd())); err != nil {
			return nil, fmt.Errorf("failed to move device %q to health namespace: %s", epIfaceName, err)
		}

	case option.DatapathModeIpvlan:
		mapFD, err := connector.CreateAndSetupIpvlanSlave("",
			epIfaceName, netNS, mtuConfig.GetDeviceMTU(),
			option.Config.Ipvlan.MasterDeviceIndex,
			option.Config.Ipvlan.OperationMode, info)
		if err != nil {
			if errDel := netns.RemoveNetNSWithName(netNSName); errDel != nil {
				log.WithError(errDel).WithField(logfields.NetNSName, netNSName).
					Warning("Unable to remove network namespace")
			}
			return nil, err
		}
		defer unix.Close(mapFD)

	}

	if err = configureHealthInterface(netNS, epIfaceName, ip4Address, ip6Address); err != nil {
		return nil, fmt.Errorf("failed configure health interface %q: %s", epIfaceName, err)
	}

	pidfile := filepath.Join(option.Config.StateDir, PidfilePath)
	prog := "ip"
	args := []string{"netns", "exec", netNSName, binaryName, "--pidfile", pidfile}
	cmd.SetTarget(prog)
	cmd.SetArgs(args)
	log.Infof("Spawning health endpoint with command %q %q", prog, args)
	if err := cmd.Run(); err != nil {
		return nil, err
	}

	// Create the endpoint
	ep, err := endpoint.NewEndpointFromChangeModel(owner, info)
	if err != nil {
		return nil, fmt.Errorf("Error while creating endpoint model: %s", err)
	}

	// Wait until the cilium-health endpoint is running before setting up routes
	deadline := time.Now().Add(1 * time.Minute)
	for {
		if _, err := os.Stat(pidfile); err == nil {
			log.WithField("pidfile", pidfile).Debug("cilium-health agent running")
			break
		} else if time.Now().After(deadline) {
			return nil, fmt.Errorf("Endpoint failed to run: %s", err)
		} else {
			time.Sleep(1 * time.Second)
		}
	}

	// Set up the endpoint routes
	hostAddressing := node.GetNodeAddressing()
	if err = configureHealthRouting(info.ContainerName, epIfaceName, hostAddressing, mtuConfig); err != nil {
		return nil, fmt.Errorf("Error while configuring routes: %s", err)
	}

	if err := epMgr.AddEndpoint(owner, ep, "Create cilium-health endpoint"); err != nil {
		return nil, fmt.Errorf("Error while adding endpoint: %s", err)
	}

	if err := ep.LockAlive(); err != nil {
		return nil, err
	}
	ep.PinDatapathMap()
	ep.Unlock()

	// Give the endpoint a security identity
	ctx, cancel := context.WithTimeout(baseCtx, LaunchTime)
	defer cancel()
	ep.UpdateLabels(ctx, labels.LabelHealth, nil, true)

	// Initialize the health client to talk to this instance. This is why
	// the caller must limit usage of this package to a single goroutine.
	client, err := probe.NewClient("http://" + net.JoinHostPort(healthIP.String(), fmt.Sprintf("%d", healthDefaults.HTTPPathPort)))
	if err != nil {
		return nil, fmt.Errorf("Cannot establish connection to health endpoint: %s", err)
	}
	metrics.SubprocessStart.WithLabelValues(ciliumHealth).Inc()

	return &Client{Client: client}, nil
}
<|endoftext|>


<commit_before>package images

import (
	"fmt"
	"io"
	"time"

	docker "github.com/fsouza/go-dockerclient"
	"github.com/pachyderm/pachyderm/src/server/pkg/deploy/assets"
)

// Export a tarball of the images needed by a deployment.
func Export(opts *assets.AssetOpts, out io.Writer) error {
	client, err := docker.NewClientFromEnv()
	if err != nil {
		return err
	}
	authConfigs, err := docker.NewAuthConfigurationsFromDockerCfg()
	if err != nil {
		return fmt.Errorf("error parsing auth: %s, try running `docker login`", err.Error())
	}
	if len(authConfigs.Configs) == 0 {
		return fmt.Errorf("didn't find any valud auth configurations")
	}
	images := assets.Images(opts)
	for _, image := range images {
		repository, tag := docker.ParseRepositoryTag(image)
		pulled := false
		var loopErr error
		for _, authConfig := range authConfigs.Configs {
			if err := client.PullImage(
				docker.PullImageOptions{
					Repository:        repository,
					Tag:               tag,
					InactivityTimeout: 10 * time.Millisecond,
				},
				authConfig,
			); err != nil {
				loopErr = err
				continue
			}
			pulled = true
			break
		}
		if !pulled {
			return fmt.Errorf("error pulling image: %+v", loopErr)
		}
	}
	return client.ExportImages(docker.ExportImagesOptions{
		Names:        images,
		OutputStream: out,
	})
}

// Import a tarball of the images needed by a deployment such as the one
// created by Export and push those images to the registry specific in opts.
func Import(opts *assets.AssetOpts, in io.Reader) error {
	client, err := docker.NewClientFromEnv()
	if err != nil {
		return err
	}
	authConfigs, err := docker.NewAuthConfigurationsFromDockerCfg()
	if err != nil {
		return fmt.Errorf("error parsing auth: %s, try running `docker login`", err.Error())
	}
	if len(authConfigs.Configs) == 0 {
		return fmt.Errorf("didn't find any valid auth configurations")
	}
	if err := client.LoadImage(docker.LoadImageOptions{
		InputStream: in,
	}); err != nil {
		return err
	}
	registry := opts.Registry
	opts.Registry = "" // pretend we're using default images so we can get targets to tag
	images := assets.Images(opts)
	opts.Registry = registry
	for _, image := range images {
		repository, tag := docker.ParseRepositoryTag(image)
		if err := client.TagImage(image, docker.TagImageOptions{
			Repo: assets.AddRegistry(opts, repository),
			Tag:  tag,
		},
		); err != nil {
			return fmt.Errorf("error tagging image: %v", err)
		}
		pushed := false
		var loopErr error
		for _, authConfig := range authConfigs.Configs {
			if err := client.PushImage(
				docker.PushImageOptions{
					Name:              repository,
					Tag:               tag,
					Registry:          opts.Registry,
					InactivityTimeout: 10 * time.Millisecond,
				},
				authConfig,
			); err != nil {
				loopErr = err
				continue
			}
			pushed = true
			break
		}
		if !pushed {
			return fmt.Errorf("error pushing image: %+v", loopErr)
		}
	}
	return nil
}
<commit_msg>Few bug fixes to pushing images.<commit_after>package images

import (
	"fmt"
	"io"
	"time"

	docker "github.com/fsouza/go-dockerclient"
	"github.com/pachyderm/pachyderm/src/server/pkg/deploy/assets"
)

// Export a tarball of the images needed by a deployment.
func Export(opts *assets.AssetOpts, out io.Writer) error {
	client, err := docker.NewClientFromEnv()
	if err != nil {
		return err
	}
	authConfigs, err := docker.NewAuthConfigurationsFromDockerCfg()
	if err != nil {
		return fmt.Errorf("error parsing auth: %s, try running `docker login`", err.Error())
	}
	if len(authConfigs.Configs) == 0 {
		return fmt.Errorf("didn't find any valud auth configurations")
	}
	images := assets.Images(opts)
	for _, image := range images {
		repository, tag := docker.ParseRepositoryTag(image)
		pulled := false
		var loopErr error
		for _, authConfig := range authConfigs.Configs {
			if err := client.PullImage(
				docker.PullImageOptions{
					Repository:        repository,
					Tag:               tag,
					InactivityTimeout: 10 * time.Millisecond,
				},
				authConfig,
			); err != nil {
				loopErr = err
				continue
			}
			pulled = true
			break
		}
		if !pulled {
			return fmt.Errorf("error pulling image: %+v", loopErr)
		}
	}
	return client.ExportImages(docker.ExportImagesOptions{
		Names:        images,
		OutputStream: out,
	})
}

// Import a tarball of the images needed by a deployment such as the one
// created by Export and push those images to the registry specific in opts.
func Import(opts *assets.AssetOpts, in io.Reader) error {
	client, err := docker.NewClientFromEnv()
	if err != nil {
		return err
	}
	authConfigs, err := docker.NewAuthConfigurationsFromDockerCfg()
	if err != nil {
		return fmt.Errorf("error parsing auth: %s, try running `docker login`", err.Error())
	}
	if len(authConfigs.Configs) == 0 {
		return fmt.Errorf("didn't find any valid auth configurations")
	}
	if err := client.LoadImage(docker.LoadImageOptions{
		InputStream: in,
	}); err != nil {
		return err
	}
	registry := opts.Registry
	opts.Registry = "" // pretend we're using default images so we can get targets to tag
	images := assets.Images(opts)
	opts.Registry = registry
	for _, image := range images {
		repository, tag := docker.ParseRepositoryTag(image)
		registryRepo := assets.AddRegistry(opts, repository)
		if err := client.TagImage(image, docker.TagImageOptions{
			Repo: registryRepo,
			Tag:  tag,
		},
		); err != nil {
			return fmt.Errorf("error tagging image: %v", err)
		}
		pushed := false
		var loopErr error
		for _, authConfig := range authConfigs.Configs {
			if err := client.PushImage(
				docker.PushImageOptions{
					Name:              registryRepo,
					Tag:               tag,
					Registry:          opts.Registry,
					InactivityTimeout: time.Second,
				},
				authConfig,
			); err != nil {
				loopErr = err
				continue
			}
			pushed = true
			break
		}
		if !pushed {
			return fmt.Errorf("error pushing image: %+v", loopErr)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2019 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package bar_test

import (
	"bytes"
	"net/http"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/uber-go/tally"
	testGateway "github.com/uber/zanzibar/test/lib/test_gateway"
	"github.com/uber/zanzibar/test/lib/util"
)

func TestCallMetrics(t *testing.T) {
	gateway, err := testGateway.CreateGateway(t, nil, &testGateway.Options{
		CountMetrics:      true,
		KnownHTTPBackends: []string{"bar"},
		TestBinary:        util.DefaultMainFile("example-gateway"),
		ConfigFiles:       util.DefaultConfigFiles("example-gateway"),
	})
	if !assert.NoError(t, err, "got bootstrap err") {
		return
	}
	defer gateway.Close()

	gateway.HTTPBackends()["bar"].HandleFunc(
		"POST", "/bar-path",
		func(w http.ResponseWriter, r *http.Request) {
			w.WriteHeader(200)
			if _, err := w.Write([]byte(`{
				"stringField": "stringValue",
				"intWithRange": 0,
				"intWithoutRange": 0,
				"mapIntWithRange": {},
				"mapIntWithoutRange": {},
				"binaryField": "d29ybGQ="
			}`)); err != nil {
				t.Fatal("can't write fake response")
			}
		},
	)

	numMetrics := 12
	cg := gateway.(*testGateway.ChildProcessGateway)
	cg.MetricsWaitGroup.Add(numMetrics)

	headers := make(map[string]string)
	headers["x-uuid"] = "uuid"
	headers["regionname"] = "san_francisco"
	headers["device"] = "ios"
	headers["deviceversion"] = "carbon"
	_, err = gateway.MakeRequest(
		"POST", "/bar/bar-path", headers,
		bytes.NewReader([]byte(`{
			"request":{"stringField":"foo","boolField":true,"binaryField":"aGVsbG8=","timestamp":123,"enumField":0,"longField":123}
		}`)),
	)
	if !assert.NoError(t, err, "got http error") {
		return
	}

	cg.MetricsWaitGroup.Wait()
	metrics := cg.M3Service.GetMetrics()
	cbKeys := make([]string, 0)
	for key := range metrics {
		if strings.Contains(key, "circuitbreaker") {
			cbKeys = append(cbKeys, key)
		}
	}
	assert.Equal(t, 4, len(cbKeys))
	for key := range metrics {
		if strings.HasPrefix(key, "jaeger") {
			delete(metrics, key)
		}
	}
	assert.Equal(t, numMetrics, len(metrics))

	endpointTags := map[string]string{
		"env":           "test",
		"service":       "test-gateway",
		"endpointid":    "bar",
		"handlerid":     "normal",
		"regionname":    "san_francisco",
		"device":        "ios",
		"deviceversion": "carbon",
		"dc":            "unknown",
		"protocol":      "HTTP",
	}
	statusTags := map[string]string{
		"env":           "test",
		"service":       "test-gateway",
		"status":        "200",
		"endpointid":    "bar",
		"handlerid":     "normal",
		"regionname":    "san_francisco",
		"device":        "ios",
		"deviceversion": "carbon",
		"dc":            "unknown",
		"protocol":      "HTTP",
	}
	key := tally.KeyForPrefixedStringMap("endpoint.request", endpointTags)
	assert.Contains(t, metrics, key, "expected metric: %s", key)
	key = tally.KeyForPrefixedStringMap("endpoint.latency", statusTags)
	assert.Contains(t, metrics, key, "expected metric: %s", key)

	inboundLatency := metrics[tally.KeyForPrefixedStringMap(
		"endpoint.latency", statusTags,
	)]
	value := *inboundLatency.MetricValue.Timer.I64Value
	assert.True(t, value > 1000, "expected timer to be >1000 nano seconds")
	assert.True(t, value < 1000*1000*1000, "expected timer to be <1 second")

	inboundRecvd := metrics[tally.KeyForPrefixedStringMap(
		"endpoint.request", endpointTags,
	)]
	value = *inboundRecvd.MetricValue.Count.I64Value
	assert.Equal(t, int64(1), value)

	inboundStatus := metrics[tally.KeyForPrefixedStringMap(
		"endpoint.status", statusTags,
	)]
	value = *inboundStatus.MetricValue.Count.I64Value
	assert.Equal(t, int64(1), value, "expected counter to be 1")

	httpClientNames := []string{
		"client.latency",
		"client.request",
	}
	httpClientTags := map[string]string{
		"env":            "test",
		"service":        "test-gateway",
		"clientid":       "bar",
		"clientmethod":   "Normal",
		"targetendpoint": "Bar--normal",
		"dc":             "unknown",
		"endpointid":     "bar",
		"handlerid":      "normal",
		"regionname":     "san_francisco",
		"device":         "ios",
		"deviceversion":  "carbon",
		"protocol":       "HTTP",
	}
	cStatusTags := map[string]string{
		"env":            "test",
		"service":        "test-gateway",
		"clientid":       "bar",
		"clientmethod":   "Normal",
		"targetendpoint": "Bar--normal",
		"status":         "200",
		"dc":             "unknown",
		"endpointid":     "bar",
		"handlerid":      "normal",
		"regionname":     "san_francisco",
		"device":         "ios",
		"deviceversion":  "carbon",
		"protocol":       "HTTP",
	}

	for _, name := range httpClientNames {
		key := tally.KeyForPrefixedStringMap(name, httpClientTags)
		assert.Contains(t, metrics, key, "expected metric: %s", key)
	}

	outboundLatency := metrics[tally.KeyForPrefixedStringMap(
		"client.latency", httpClientTags,
	)]
	value = *outboundLatency.MetricValue.Timer.I64Value
	assert.True(t, value > 1000, "expected timer to be >1000 nano seconds")
	assert.True(t, value < 1000*1000*1000, "expected timer to be <1 second")

	outboundSent := metrics[tally.KeyForPrefixedStringMap(
		"client.request", httpClientTags,
	)]
	value = *outboundSent.MetricValue.Count.I64Value
	assert.Equal(t, int64(1), value, "expected counter to be 1")

	statusSuccess := metrics[tally.KeyForPrefixedStringMap(
		"client.status",
		cStatusTags,
	)]
	value = *statusSuccess.MetricValue.Count.I64Value
	assert.Equal(t, int64(1), value, "expected counter to be 1")

	allLogs := gateway.AllLogs()

	logMsgs := allLogs["Finished an outgoing client HTTP request"]
	assert.Len(t, logMsgs, 1)
	logMsg := logMsgs[0]
	dynamicHeaders := []string{
		"url",
		"timestamp-finished",
		"Client-Req-Header-Uber-Trace-Id",
		"Client-Req-Header-X-Request-Uuid",
		"Client-Res-Header-Content-Length",
		"Client-Res-Header-Date",
		"timestamp-started",
		"ts",
		"hostname",
		"pid",
		"requestUUID",
	}
	for _, dynamicValue := range dynamicHeaders {
		assert.Contains(t, logMsg, dynamicValue)
		delete(logMsg, dynamicValue)
	}
	expectedValues := map[string]interface{}{
		"env":                            "test",
		"level":                          "info",
		"msg":                            "Finished an outgoing client HTTP request",
		"statusCode":                     float64(200),
		"clientID":                       "bar",
		"clientMethod":                   "Normal",
		"targetEndpoint":                 "Bar::normal",
		"clientHTTPMethod":               "POST",
		"Client-Req-Header-X-Client-Id":  "bar",
		"Client-Req-Header-Content-Type": "application/json",
		"Client-Res-Header-Content-Type": "text/plain; charset=utf-8",

		"zone":            "unknown",
		"service":         "example-gateway",
		"endpointID":      "bar",
		"endpointHandler": "normal",
		"x-uuid":          "uuid",
		"regionname":      "san_francisco",
		"device":          "ios",
		"deviceversion":   "carbon",
	}
	for actualKey, actualValue := range logMsg {
		assert.Equal(t, expectedValues[actualKey], actualValue, "unexpected field %q", actualKey)
	}
	for expectedKey, expectedValue := range expectedValues {
		assert.Equal(t, expectedValue, logMsg[expectedKey], "unexpected field %q", expectedKey)
	}
}
<commit_msg>Update test/endpoints/bar/bar_metrics_test.go<commit_after>// Copyright (c) 2019 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package bar_test

import (
	"bytes"
	"net/http"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/uber-go/tally"
	testGateway "github.com/uber/zanzibar/test/lib/test_gateway"
	"github.com/uber/zanzibar/test/lib/util"
)

func TestCallMetrics(t *testing.T) {
	gateway, err := testGateway.CreateGateway(t, nil, &testGateway.Options{
		CountMetrics:      true,
		KnownHTTPBackends: []string{"bar"},
		TestBinary:        util.DefaultMainFile("example-gateway"),
		ConfigFiles:       util.DefaultConfigFiles("example-gateway"),
	})
	if !assert.NoError(t, err, "got bootstrap err") {
		return
	}
	defer gateway.Close()

	gateway.HTTPBackends()["bar"].HandleFunc(
		"POST", "/bar-path",
		func(w http.ResponseWriter, r *http.Request) {
			w.WriteHeader(200)
			if _, err := w.Write([]byte(`{
				"stringField": "stringValue",
				"intWithRange": 0,
				"intWithoutRange": 0,
				"mapIntWithRange": {},
				"mapIntWithoutRange": {},
				"binaryField": "d29ybGQ="
			}`)); err != nil {
				t.Fatal("can't write fake response")
			}
		},
	)

	numMetrics := 12
	cg := gateway.(*testGateway.ChildProcessGateway)
	cg.MetricsWaitGroup.Add(numMetrics)

	headers := make(map[string]string)
	headers["x-uuid"] = "uuid"
	headers["regionname"] = "san_francisco"
	headers["device"] = "ios"
	headers["deviceversion"] = "carbon"
	_, err = gateway.MakeRequest(
		"POST", "/bar/bar-path", headers,
		bytes.NewReader([]byte(`{
			"request":{"stringField":"foo","boolField":true,"binaryField":"aGVsbG8=","timestamp":123,"enumField":0,"longField":123}
		}`)),
	)
	if !assert.NoError(t, err, "got http error") {
		return
	}

	cg.MetricsWaitGroup.Wait()
	metrics := cg.M3Service.GetMetrics()
	cbKeys := make([]string, 0)
	for key := range metrics {
		if strings.Contains(key, "circuitbreaker") {
			cbKeys = append(cbKeys, key)
		}
	}
	assert.Equal(t, 4, len(cbKeys))
	for key := range metrics {
		if strings.HasPrefix(key, "jaeger") {
			delete(metrics, key)
		}
	}
	assert.Equal(t, numMetrics, len(metrics))

	endpointTags := map[string]string{
		"env":           "test",
		"service":       "test-gateway",
		"endpointid":    "bar",
		"handlerid":     "normal",
		"regionname":    "san_francisco",
		"device":        "ios",
		"deviceversion": "carbon",
		"dc":            "unknown",
		"protocol":      "HTTP",
	}
	statusTags := map[string]string{
		"env":           "test",
		"service":       "test-gateway",
		"status":        "200",
		"endpointid":    "bar",
		"handlerid":     "normal",
		"regionname":    "san_francisco",
		"device":        "ios",
		"deviceversion": "carbon",
		"dc":            "unknown",
		"protocol":      "HTTP",
	}
	key := tally.KeyForPrefixedStringMap("endpoint.request", endpointTags)
	assert.Contains(t, metrics, key, "expected metric: %s", key)
	key = tally.KeyForPrefixedStringMap("endpoint.latency", statusTags)
	assert.Contains(t, metrics, key, "expected metric: %s", key)

	inboundLatency := metrics[tally.KeyForPrefixedStringMap(
		"endpoint.latency", statusTags,
	)]
	value := *inboundLatency.MetricValue.Timer.I64Value
	assert.True(t, value > 1000, "expected timer to be >1000 nano seconds")
	assert.True(t, value < 1000*1000*1000, "expected timer to be <1 second")

	inboundRecvd := metrics[tally.KeyForPrefixedStringMap(
		"endpoint.request", endpointTags,
	)]
	value = *inboundRecvd.MetricValue.Count.I64Value
	assert.Equal(t, int64(1), value)

	inboundStatus := metrics[tally.KeyForPrefixedStringMap(
		"endpoint.status", statusTags,
	)]
	value = *inboundStatus.MetricValue.Count.I64Value
	assert.Equal(t, int64(1), value, "expected counter to be 1")

	httpClientNames := []string{
		"client.latency",
		"client.request",
	}
	httpClientTags := map[string]string{
		"env":            "test",
		"service":        "test-gateway",
		"clientid":       "bar",
		"clientmethod":   "Normal",
		"targetendpoint": "Bar--normal",
		"dc":             "unknown",
		"endpointid":     "bar",
		"handlerid":      "normal",
		"regionname":     "san_francisco",
		"device":         "ios",
		"deviceversion":  "carbon",
		"protocol":       "HTTP",
	}
	cStatusTags := map[string]string{
		"env":            "test",
		"service":        "test-gateway",
		"clientid":       "bar",
		"clientmethod":   "Normal",
		"targetendpoint": "Bar--normal",
		"status":         "200",
		"dc":             "unknown",
		"endpointid":     "bar",
		"handlerid":      "normal",
		"regionname":     "san_francisco",
		"device":         "ios",
		"deviceversion":  "carbon",
		"protocol":       "HTTP",
	}

	for _, name := range httpClientNames {
		key := tally.KeyForPrefixedStringMap(name, httpClientTags)
		assert.Contains(t, metrics, key, "expected metric: %s", key)
	}

	outboundLatency := metrics[tally.KeyForPrefixedStringMap(
		"client.latency", httpClientTags,
	)]
	value = *outboundLatency.MetricValue.Timer.I64Value
	assert.True(t, value > 1000, "expected timer to be >1000 nano seconds")
	assert.True(t, value < 1000*1000*1000, "expected timer to be <1 second")

	outboundSent := metrics[tally.KeyForPrefixedStringMap(
		"client.request", httpClientTags,
	)]
	value = *outboundSent.MetricValue.Count.I64Value
	assert.Equal(t, int64(1), value, "expected counter to be 1")

	statusSuccess := metrics[tally.KeyForPrefixedStringMap(
		"client.status",
		cStatusTags,
	)]
	value = *statusSuccess.MetricValue.Count.I64Value
	assert.Equal(t, int64(1), value, "expected counter to be 1")

	allLogs := gateway.AllLogs()

	logMsgs := allLogs["Finished an outgoing client HTTP request"]
	assert.Len(t, logMsgs, 1)
	logMsg := logMsgs[0]
	dynamicHeaders := []string{
		"url",
		"timestamp-finished",
		"Client-Req-Header-Uber-Trace-Id",
		"Client-Req-Header-X-Request-Uuid",
		"Client-Res-Header-Content-Length",
		"Client-Res-Header-Date",
		"timestamp-started",
		"ts",
		"hostname",
		"pid",
		"requestUUID",
	}
	for _, dynamicValue := range dynamicHeaders {
		assert.Contains(t, logMsg, dynamicValue)
		delete(logMsg, dynamicValue)
	}
	expectedValues := map[string]interface{}{
		"env":                            "test",
		"level":                          "info",
		"msg":                            "Finished an outgoing client HTTP request",
		"statusCode":                     float64(200),
		"clientID":                       "bar",
		"clientMethod":                   "Normal",
		"clientThriftMethod":             "Bar::normal",
		"clientHTTPMethod":               "POST",
		"Client-Req-Header-X-Client-Id":  "bar",
		"Client-Req-Header-Content-Type": "application/json",
		"Client-Res-Header-Content-Type": "text/plain; charset=utf-8",

		"zone":            "unknown",
		"service":         "example-gateway",
		"endpointID":      "bar",
		"endpointHandler": "normal",
		"x-uuid":          "uuid",
		"regionname":      "san_francisco",
		"device":          "ios",
		"deviceversion":   "carbon",
	}
	for actualKey, actualValue := range logMsg {
		assert.Equal(t, expectedValues[actualKey], actualValue, "unexpected field %q", actualKey)
	}
	for expectedKey, expectedValue := range expectedValues {
		assert.Equal(t, expectedValue, logMsg[expectedKey], "unexpected field %q", expectedKey)
	}
}
<|endoftext|>