<commit_before>package container

import (
	log "github.com/Sirupsen/logrus"
	"github.com/samalba/dockerclient"

	"fmt"
	"strings"
	"time"
)

// Docker container labels
const (
	TugbotService     = "tugbot-service"
	TugbotTest        = "tugbot-test"
	TugbotEventDocker = "tugbot-event-docker"
	TugbotEventTimer  = "tugbot-event-timer"
	TugbotCreatedFrom = "tugbot-created-from"
	SwarmTaskID       = "com.docker.swarm.task.id"
)

// Docker Event Filter
const (
	// type filter: tugbot-event-docker-filter-type=container|image|daemon|network|volume|plugin
	TypeFilter = "tugbot-event-docker-filter-type"
	// action filter (depends on type), for 'container' type:
	//  - attach, commit, copy, create, destroy, detach, die, exec_create, exec_detach, exec_start, export,
	//  - health_status, kill, oom, pause, rename, resize, restart, start, stop, top, unpause, update
	ActionFilter = "tugbot-event-docker-filter-action"
	// container filter: use name, comma separated name list or RE2 regexp
	ContainerFilter = "tugbot-event-docker-filter-container"
	// image filter: use name, comma separated name list or RE2 regexp
	ImageFilter = "tugbot-event-docker-filter-image"
	// label filter: use key=value comma separated pairs
	LabelFilter = "tugbot-event-docker-filter-label"
)

// NewContainer returns a new Container instance instantiated with the
// specified ContainerInfo and ImageInfo structs.
func NewContainer(containerInfo *dockerclient.ContainerInfo, imageInfo *dockerclient.ImageInfo) *Container {
	return &Container{
		containerInfo: containerInfo,
		imageInfo:     imageInfo,
	}
}

// Container represents a running Docker container.
type Container struct {
	containerInfo *dockerclient.ContainerInfo
	imageInfo     *dockerclient.ImageInfo
}

// ID returns the Docker container ID.
func (c Container) ID() string {
	return c.containerInfo.Id
}

// Name returns the Docker container name.
func (c Container) Name() string {
	return strings.TrimPrefix(c.containerInfo.Name, "/")
}

// ImageID returns the ID of the Docker image that was used to start the
// container.
func (c Container) ImageID() string {
	return c.imageInfo.Id
}

// ImageName returns the name of the Docker image that was used to start the
// container. If the original image was specified without a particular tag, the
// "latest" tag is assumed.
func (c Container) ImageName() string {
	imageName := c.containerInfo.Config.Image

	if !strings.Contains(imageName, ":") {
		imageName = fmt.Sprintf("%s:latest", imageName)
	}

	return imageName
}

// IsTugbot returns whether or not the current container is the tugbot container itself.
// The tugbot container is identified by the presence of the "tugbot.service"
// label in the container metadata.
func (c Container) IsTugbot() bool {
	val, ok := c.containerInfo.Config.Labels[TugbotService]

	return ok && val == "true"
}

// IsTugbotCandidate returns whether or not a container is a candidate to run by tugbot.
// A candidate container is identified by the presence of "tugbot-test",
// it doesn't contain "tugbot.created.from" in the container metadata and it state is "Exited".
func (c Container) IsTugbotCandidate() bool {
	ret := false
	val, ok := c.containerInfo.Config.Labels[TugbotTest]
	if ok && val == "true" {
		if !c.IsCreatedByTugbot() {
			ret = c.containerInfo.State.StateString() == "exited"
		}
	}

	return ret
}

// IsCreatedByTugbot returns whether or not a container created by tugbot.
func (c Container) IsCreatedByTugbot() bool {
	val, ok := c.containerInfo.Config.Labels[TugbotCreatedFrom]

	return ok && val != ""
}

// IsEventListener returns whether or not a container should run when an event e is occurred.
func (c Container) IsEventListener(e *dockerclient.Event) bool {
	ret := false
	if e != nil {
		// check if container is subscribed to Docker events, i.e. 'tugbot-event-docker' label exists
		_, ret = c.containerInfo.Config.Labels[TugbotEventDocker]
		if ret {
			// filter by event type
			if typeFilter, ok := c.containerInfo.Config.Labels[TypeFilter]; ok {
				ret = sliceContains(e.Type, splitAndTrimSpaces(typeFilter, ","))
			}
			// filter by event action
			if actionFilter, ok := c.containerInfo.Config.Labels[ActionFilter]; ok {
				ret = ret && sliceContains(e.Action, splitAndTrimSpaces(actionFilter, ","))
			}
			// filter by container name or name regexp
			if containerFilter, ok := c.containerInfo.Config.Labels[ContainerFilter]; ok {
				ret = ret && inFilterOrList(e.Actor.Attributes["name"], containerFilter)
			}
		}
		// filter by event image
		if imageFilter, ok := c.containerInfo.Config.Labels[ImageFilter]; ok {
			// get image name from event.From field
			imageName := e.From
			// in case of "image" event.Type, event.ID contains image ID (name:tag) for 'pull' action and sha256:num for untag and delete
			if e.Type == "image" {
				imageName = e.ID
			}
			ret = ret && inFilterOrList(imageName, imageFilter)
		}
		// filter by event labels
		if labelFilter, ok := c.containerInfo.Config.Labels[LabelFilter]; ok {
			labels := splitAndTrimSpaces(labelFilter, ",")
			for _, label := range labels {
				ret = ret && mapContains(e.Actor.Attributes, splitAndTrimSpaces(label, "="))
			}
		}
	}

	return ret
}

// GetEventListenerTimer returns interval duration between a test container run and true
// if docker label exist and label value parsed into Duration, Otherwise false.
func (c Container) GetEventListenerInterval() (time.Duration, bool) {
	var ret time.Duration
	val, ok := c.containerInfo.Config.Labels[TugbotEventTimer]
	if ok {
		interval, err := time.ParseDuration(val)
		if err != nil {
			log.Errorf("Failed to parse %s docker label: %s into golang Duration (%v)", TugbotEventTimer, val, err)
			ok = false
		} else {
			ret = interval
		}
	}

	return ret, ok
}

// Any links in the HostConfig need to be re-written before they can be
// re-submitted to the Docker create API.
func (c Container) hostConfig() *dockerclient.HostConfig {
	hostConfig := c.containerInfo.HostConfig

	for i, link := range hostConfig.Links {
		name := link[0:strings.Index(link, ":")]
		alias := link[strings.LastIndex(link, "/"):]

		hostConfig.Links[i] = fmt.Sprintf("%s:%s", name, alias)
	}

	return hostConfig
}
<commit_msg>extra logging added temporarily<commit_after>package container

import (
	log "github.com/Sirupsen/logrus"
	"github.com/samalba/dockerclient"

	"fmt"
	"strings"
	"time"
)

// Docker container labels
const (
	TugbotService     = "tugbot-service"
	TugbotTest        = "tugbot-test"
	TugbotEventDocker = "tugbot-event-docker"
	TugbotEventTimer  = "tugbot-event-timer"
	TugbotCreatedFrom = "tugbot-created-from"
	SwarmTaskID       = "com.docker.swarm.task.id"
)

// Docker Event Filter
const (
	// type filter: tugbot-event-docker-filter-type=container|image|daemon|network|volume|plugin
	TypeFilter = "tugbot-event-docker-filter-type"
	// action filter (depends on type), for 'container' type:
	//  - attach, commit, copy, create, destroy, detach, die, exec_create, exec_detach, exec_start, export,
	//  - health_status, kill, oom, pause, rename, resize, restart, start, stop, top, unpause, update
	ActionFilter = "tugbot-event-docker-filter-action"
	// container filter: use name, comma separated name list or RE2 regexp
	ContainerFilter = "tugbot-event-docker-filter-container"
	// image filter: use name, comma separated name list or RE2 regexp
	ImageFilter = "tugbot-event-docker-filter-image"
	// label filter: use key=value comma separated pairs
	LabelFilter = "tugbot-event-docker-filter-label"
)

// NewContainer returns a new Container instance instantiated with the
// specified ContainerInfo and ImageInfo structs.
func NewContainer(containerInfo *dockerclient.ContainerInfo, imageInfo *dockerclient.ImageInfo) *Container {
	return &Container{
		containerInfo: containerInfo,
		imageInfo:     imageInfo,
	}
}

// Container represents a running Docker container.
type Container struct {
	containerInfo *dockerclient.ContainerInfo
	imageInfo     *dockerclient.ImageInfo
}

// ID returns the Docker container ID.
func (c Container) ID() string {
	return c.containerInfo.Id
}

// Name returns the Docker container name.
func (c Container) Name() string {
	return strings.TrimPrefix(c.containerInfo.Name, "/")
}

// ImageID returns the ID of the Docker image that was used to start the
// container.
func (c Container) ImageID() string {
	return c.imageInfo.Id
}

// ImageName returns the name of the Docker image that was used to start the
// container. If the original image was specified without a particular tag, the
// "latest" tag is assumed.
func (c Container) ImageName() string {
	imageName := c.containerInfo.Config.Image

	if !strings.Contains(imageName, ":") {
		imageName = fmt.Sprintf("%s:latest", imageName)
	}

	return imageName
}

// IsTugbot returns whether or not the current container is the tugbot container itself.
// The tugbot container is identified by the presence of the "tugbot.service"
// label in the container metadata.
func (c Container) IsTugbot() bool {
	val, ok := c.containerInfo.Config.Labels[TugbotService]

	return ok && val == "true"
}

// IsTugbotCandidate returns whether or not a container is a candidate to run by tugbot.
// A candidate container is identified by the presence of "tugbot-test",
// it doesn't contain "tugbot.created.from" in the container metadata and it state is "Exited".
func (c Container) IsTugbotCandidate() bool {
	ret := false
	log.Info("Checking container %s for being candidate",c.containerInfo.Name)
	val, ok := c.containerInfo.Config.Labels[TugbotTest]
	log.Info("Val is %s", val)
	if ok && val == "true" {
		if !c.IsCreatedByTugbot() {
			ret = c.containerInfo.State.StateString() == "exited"
		}
	}

	return ret
}

// IsCreatedByTugbot returns whether or not a container created by tugbot.
func (c Container) IsCreatedByTugbot() bool {
	val, ok := c.containerInfo.Config.Labels[TugbotCreatedFrom]

	return ok && val != ""
}

// IsEventListener returns whether or not a container should run when an event e is occurred.
func (c Container) IsEventListener(e *dockerclient.Event) bool {
	ret := false
	if e != nil {
		// check if container is subscribed to Docker events, i.e. 'tugbot-event-docker' label exists
		_, ret = c.containerInfo.Config.Labels[TugbotEventDocker]
		if ret {
			// filter by event type
			if typeFilter, ok := c.containerInfo.Config.Labels[TypeFilter]; ok {
				ret = sliceContains(e.Type, splitAndTrimSpaces(typeFilter, ","))
			}
			// filter by event action
			if actionFilter, ok := c.containerInfo.Config.Labels[ActionFilter]; ok {
				ret = ret && sliceContains(e.Action, splitAndTrimSpaces(actionFilter, ","))
			}
			// filter by container name or name regexp
			if containerFilter, ok := c.containerInfo.Config.Labels[ContainerFilter]; ok {
				ret = ret && inFilterOrList(e.Actor.Attributes["name"], containerFilter)
			}
		}
		// filter by event image
		if imageFilter, ok := c.containerInfo.Config.Labels[ImageFilter]; ok {
			// get image name from event.From field
			imageName := e.From
			// in case of "image" event.Type, event.ID contains image ID (name:tag) for 'pull' action and sha256:num for untag and delete
			if e.Type == "image" {
				imageName = e.ID
			}
			ret = ret && inFilterOrList(imageName, imageFilter)
		}
		// filter by event labels
		if labelFilter, ok := c.containerInfo.Config.Labels[LabelFilter]; ok {
			labels := splitAndTrimSpaces(labelFilter, ",")
			for _, label := range labels {
				ret = ret && mapContains(e.Actor.Attributes, splitAndTrimSpaces(label, "="))
			}
		}
	}

	return ret
}

// GetEventListenerTimer returns interval duration between a test container run and true
// if docker label exist and label value parsed into Duration, Otherwise false.
func (c Container) GetEventListenerInterval() (time.Duration, bool) {
	var ret time.Duration
	val, ok := c.containerInfo.Config.Labels[TugbotEventTimer]
	if ok {
		interval, err := time.ParseDuration(val)
		if err != nil {
			log.Errorf("Failed to parse %s docker label: %s into golang Duration (%v)", TugbotEventTimer, val, err)
			ok = false
		} else {
			ret = interval
		}
	}

	return ret, ok
}

// Any links in the HostConfig need to be re-written before they can be
// re-submitted to the Docker create API.
func (c Container) hostConfig() *dockerclient.HostConfig {
	hostConfig := c.containerInfo.HostConfig

	for i, link := range hostConfig.Links {
		name := link[0:strings.Index(link, ":")]
		alias := link[strings.LastIndex(link, "/"):]

		hostConfig.Links[i] = fmt.Sprintf("%s:%s", name, alias)
	}

	return hostConfig
}
<|endoftext|>


<commit_before>package jobsupervisor

import (
	"fmt"
	"path/filepath"
	"time"

	"github.com/cloudfoundry/bosh-agent/internal/github.com/pivotal-golang/clock"
	"github.com/cloudfoundry/bosh-agent/internal/github.com/pivotal/go-smtpd/smtpd"

	boshalert "github.com/cloudfoundry/bosh-agent/agent/alert"
	bosherr "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/errors"
	boshlog "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/logger"
	boshsys "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/system"
	boshmonit "github.com/cloudfoundry/bosh-agent/jobsupervisor/monit"
	boshdir "github.com/cloudfoundry/bosh-agent/settings/directories"
)

const monitJobSupervisorLogTag = "monitJobSupervisor"

type monitJobSupervisor struct {
	fs                    boshsys.FileSystem
	runner                boshsys.CmdRunner
	client                boshmonit.Client
	logger                boshlog.Logger
	dirProvider           boshdir.Provider
	jobFailuresServerPort int
	reloadOptions         MonitReloadOptions
	timeService           clock.Clock
}

type MonitReloadOptions struct {
	// Number of times `monit reload` will be executed
	MaxTries int

	// Number of times monit incarnation will be checked
	// for difference after executing `monit reload`
	MaxCheckTries int

	// Length of time between checking for incarnation difference
	DelayBetweenCheckTries time.Duration
}

func NewMonitJobSupervisor(
	fs boshsys.FileSystem,
	runner boshsys.CmdRunner,
	client boshmonit.Client,
	logger boshlog.Logger,
	dirProvider boshdir.Provider,
	jobFailuresServerPort int,
	reloadOptions MonitReloadOptions,
	timeService clock.Clock,
) JobSupervisor {
	return &monitJobSupervisor{
		fs:                    fs,
		runner:                runner,
		client:                client,
		logger:                logger,
		dirProvider:           dirProvider,
		jobFailuresServerPort: jobFailuresServerPort,
		reloadOptions:         reloadOptions,
		timeService:           timeService,
	}
}

func (m monitJobSupervisor) Reload() error {
	var currentIncarnation int

	oldIncarnation, err := m.getIncarnation()
	if err != nil {
		return bosherr.WrapError(err, "Getting monit incarnation")
	}

	// Monit process could be started in the same second as `monit reload` runs
	// so it's ideal for MaxCheckTries * DelayBetweenCheckTries to be greater than 1 sec
	// because monit incarnation id is just a timestamp with 1 sec resolution.
	for reloadI := 0; reloadI < m.reloadOptions.MaxTries; reloadI++ {
		// Exit code or output cannot be trusted
		_, _, _, err := m.runner.RunCommand("monit", "reload")
		if err != nil {
			m.logger.Error(monitJobSupervisorLogTag, "Failed to reload monit %s", err.Error())
		}

		for checkI := 0; checkI < m.reloadOptions.MaxCheckTries; checkI++ {
			currentIncarnation, err = m.getIncarnation()
			if err != nil {
				return bosherr.WrapError(err, "Getting monit incarnation")
			}

			// Incarnation id can decrease or increase because
			// monit uses time(...) and system time can be changed
			if oldIncarnation != currentIncarnation {
				return nil
			}

			m.logger.Debug(
				monitJobSupervisorLogTag,
				"Waiting for monit to reload: before=%d after=%d",
				oldIncarnation, currentIncarnation,
			)

			time.Sleep(m.reloadOptions.DelayBetweenCheckTries)
		}
	}

	return bosherr.Errorf(
		"Failed to reload monit: before=%d after=%d",
		oldIncarnation, currentIncarnation,
	)
}

func (m monitJobSupervisor) Start() error {
	services, err := m.client.ServicesInGroup("vcap")
	if err != nil {
		return bosherr.WrapError(err, "Getting vcap services")
	}

	for _, service := range services {
		m.logger.Debug(monitJobSupervisorLogTag, "Starting service %s", service)
		err = m.client.StartService(service)
		if err != nil {
			return bosherr.WrapErrorf(err, "Starting service %s", service)
		}
	}

	return nil
}

func (m monitJobSupervisor) Stop() error {
	serviceNames, err := m.client.ServicesInGroup("vcap")
	if err != nil {
		return bosherr.WrapError(err, "Getting vcap services")
	}

	for _, serviceName := range serviceNames {
		m.logger.Debug(monitJobSupervisorLogTag, "Stopping service '%s'", serviceName)
		err = m.client.StopService(serviceName)
		if err != nil {
			return bosherr.WrapErrorf(err, "Stopping service '%s'", serviceName)
		}
	}

	pending := []string{}
	timer := m.timeService.NewTimer(10 * time.Minute)

	for {
		select {
		case <-timer.C():
			return bosherr.Errorf("Timed out stopping services. Failures: %v", pending)
		default:
		}

		status, err := m.client.Status()
		if err != nil {
			return bosherr.WrapErrorf(err, "Getting monit status")
		}

		services := status.ServicesInGroup("vcap")
		pending = []string{}

		for _, service := range services {
			if service.Monitored || service.Pending {
				pending = append(pending, service.Name)
			}

			if service.Errored {
				return bosherr.Errorf("Stopping service '%s'. Status message: '%s'", service.Name, service.StatusMessage)
			}
		}

		if len(pending) == 0 {
			return nil
		}

		m.timeService.Sleep(500 * time.Millisecond)
	}
}

func (m monitJobSupervisor) Unmonitor() error {
	services, err := m.client.ServicesInGroup("vcap")
	if err != nil {
		return bosherr.WrapError(err, "Getting vcap services")
	}

	for _, service := range services {
		m.logger.Debug(monitJobSupervisorLogTag, "Unmonitoring service %s", service)
		err := m.client.UnmonitorService(service)
		if err != nil {
			return bosherr.WrapErrorf(err, "Unmonitoring service %s", service)
		}
	}

	return nil
}

func (m monitJobSupervisor) Status() (status string) {
	status = "running"
	m.logger.Debug(monitJobSupervisorLogTag, "Getting monit status")
	monitStatus, err := m.client.Status()
	if err != nil {
		status = "unknown"
		return
	}

	for _, service := range monitStatus.ServicesInGroup("vcap") {
		if service.Status == "starting" {
			return "starting"
		}
		if !service.Monitored || service.Status != "running" {
			status = "failing"
		}
	}

	return
}

func (m monitJobSupervisor) getIncarnation() (int, error) {
	monitStatus, err := m.client.Status()
	if err != nil {
		return -1, err
	}

	return monitStatus.GetIncarnation()
}

func (m monitJobSupervisor) AddJob(jobName string, jobIndex int, configPath string) error {
	targetFilename := fmt.Sprintf("%04d_%s.monitrc", jobIndex, jobName)
	targetConfigPath := filepath.Join(m.dirProvider.MonitJobsDir(), targetFilename)

	configContent, err := m.fs.ReadFile(configPath)
	if err != nil {
		return bosherr.WrapError(err, "Reading job config from file")
	}

	err = m.fs.WriteFile(targetConfigPath, configContent)
	if err != nil {
		return bosherr.WrapError(err, "Writing to job config file")
	}

	return nil
}

func (m monitJobSupervisor) RemoveAllJobs() error {
	return m.fs.RemoveAll(m.dirProvider.MonitJobsDir())
}

func (m monitJobSupervisor) MonitorJobFailures(handler JobFailureHandler) (err error) {
	alertHandler := func(smtpd.Connection, smtpd.MailAddress) (env smtpd.Envelope, err error) {
		env = &alertEnvelope{
			new(smtpd.BasicEnvelope),
			handler,
			new(boshalert.MonitAlert),
		}
		return
	}

	serv := &smtpd.Server{
		Addr:      fmt.Sprintf("127.0.0.1:%d", m.jobFailuresServerPort),
		OnNewMail: alertHandler,
	}

	err = serv.ListenAndServe()
	if err != nil {
		err = bosherr.WrapError(err, "Listen for SMTP")
	}
	return
}
<commit_msg>Rename variable for clarity.<commit_after>package jobsupervisor

import (
	"fmt"
	"path/filepath"
	"time"

	"github.com/cloudfoundry/bosh-agent/internal/github.com/pivotal-golang/clock"
	"github.com/cloudfoundry/bosh-agent/internal/github.com/pivotal/go-smtpd/smtpd"

	boshalert "github.com/cloudfoundry/bosh-agent/agent/alert"
	bosherr "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/errors"
	boshlog "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/logger"
	boshsys "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/system"
	boshmonit "github.com/cloudfoundry/bosh-agent/jobsupervisor/monit"
	boshdir "github.com/cloudfoundry/bosh-agent/settings/directories"
)

const monitJobSupervisorLogTag = "monitJobSupervisor"

type monitJobSupervisor struct {
	fs                    boshsys.FileSystem
	runner                boshsys.CmdRunner
	client                boshmonit.Client
	logger                boshlog.Logger
	dirProvider           boshdir.Provider
	jobFailuresServerPort int
	reloadOptions         MonitReloadOptions
	timeService           clock.Clock
}

type MonitReloadOptions struct {
	// Number of times `monit reload` will be executed
	MaxTries int

	// Number of times monit incarnation will be checked
	// for difference after executing `monit reload`
	MaxCheckTries int

	// Length of time between checking for incarnation difference
	DelayBetweenCheckTries time.Duration
}

func NewMonitJobSupervisor(
	fs boshsys.FileSystem,
	runner boshsys.CmdRunner,
	client boshmonit.Client,
	logger boshlog.Logger,
	dirProvider boshdir.Provider,
	jobFailuresServerPort int,
	reloadOptions MonitReloadOptions,
	timeService clock.Clock,
) JobSupervisor {
	return &monitJobSupervisor{
		fs:                    fs,
		runner:                runner,
		client:                client,
		logger:                logger,
		dirProvider:           dirProvider,
		jobFailuresServerPort: jobFailuresServerPort,
		reloadOptions:         reloadOptions,
		timeService:           timeService,
	}
}

func (m monitJobSupervisor) Reload() error {
	var currentIncarnation int

	oldIncarnation, err := m.getIncarnation()
	if err != nil {
		return bosherr.WrapError(err, "Getting monit incarnation")
	}

	// Monit process could be started in the same second as `monit reload` runs
	// so it's ideal for MaxCheckTries * DelayBetweenCheckTries to be greater than 1 sec
	// because monit incarnation id is just a timestamp with 1 sec resolution.
	for reloadI := 0; reloadI < m.reloadOptions.MaxTries; reloadI++ {
		// Exit code or output cannot be trusted
		_, _, _, err := m.runner.RunCommand("monit", "reload")
		if err != nil {
			m.logger.Error(monitJobSupervisorLogTag, "Failed to reload monit %s", err.Error())
		}

		for checkI := 0; checkI < m.reloadOptions.MaxCheckTries; checkI++ {
			currentIncarnation, err = m.getIncarnation()
			if err != nil {
				return bosherr.WrapError(err, "Getting monit incarnation")
			}

			// Incarnation id can decrease or increase because
			// monit uses time(...) and system time can be changed
			if oldIncarnation != currentIncarnation {
				return nil
			}

			m.logger.Debug(
				monitJobSupervisorLogTag,
				"Waiting for monit to reload: before=%d after=%d",
				oldIncarnation, currentIncarnation,
			)

			time.Sleep(m.reloadOptions.DelayBetweenCheckTries)
		}
	}

	return bosherr.Errorf(
		"Failed to reload monit: before=%d after=%d",
		oldIncarnation, currentIncarnation,
	)
}

func (m monitJobSupervisor) Start() error {
	services, err := m.client.ServicesInGroup("vcap")
	if err != nil {
		return bosherr.WrapError(err, "Getting vcap services")
	}

	for _, service := range services {
		m.logger.Debug(monitJobSupervisorLogTag, "Starting service %s", service)
		err = m.client.StartService(service)
		if err != nil {
			return bosherr.WrapErrorf(err, "Starting service %s", service)
		}
	}

	return nil
}

func (m monitJobSupervisor) Stop() error {
	serviceNames, err := m.client.ServicesInGroup("vcap")
	if err != nil {
		return bosherr.WrapError(err, "Getting vcap services")
	}

	for _, serviceName := range serviceNames {
		m.logger.Debug(monitJobSupervisorLogTag, "Stopping service '%s'", serviceName)
		err = m.client.StopService(serviceName)
		if err != nil {
			return bosherr.WrapErrorf(err, "Stopping service '%s'", serviceName)
		}
	}

	servicesToBeStopped := []string{}
	timer := m.timeService.NewTimer(10 * time.Minute)

	for {
		select {
		case <-timer.C():
			return bosherr.Errorf("Timed out stopping services. Failures: %v", servicesToBeStopped)
		default:
		}

		status, err := m.client.Status()
		if err != nil {
			return bosherr.WrapErrorf(err, "Getting monit status")
		}

		services := status.ServicesInGroup("vcap")
		servicesToBeStopped = []string{}

		for _, service := range services {
			if service.Monitored || service.Pending {
				servicesToBeStopped = append(servicesToBeStopped, service.Name)
			}

			if service.Errored {
				return bosherr.Errorf("Stopping service '%s'. Status message: '%s'", service.Name, service.StatusMessage)
			}
		}

		if len(servicesToBeStopped) == 0 {
			return nil
		}

		m.timeService.Sleep(500 * time.Millisecond)
	}
}

func (m monitJobSupervisor) Unmonitor() error {
	services, err := m.client.ServicesInGroup("vcap")
	if err != nil {
		return bosherr.WrapError(err, "Getting vcap services")
	}

	for _, service := range services {
		m.logger.Debug(monitJobSupervisorLogTag, "Unmonitoring service %s", service)
		err := m.client.UnmonitorService(service)
		if err != nil {
			return bosherr.WrapErrorf(err, "Unmonitoring service %s", service)
		}
	}

	return nil
}

func (m monitJobSupervisor) Status() (status string) {
	status = "running"
	m.logger.Debug(monitJobSupervisorLogTag, "Getting monit status")
	monitStatus, err := m.client.Status()
	if err != nil {
		status = "unknown"
		return
	}

	for _, service := range monitStatus.ServicesInGroup("vcap") {
		if service.Status == "starting" {
			return "starting"
		}
		if !service.Monitored || service.Status != "running" {
			status = "failing"
		}
	}

	return
}

func (m monitJobSupervisor) getIncarnation() (int, error) {
	monitStatus, err := m.client.Status()
	if err != nil {
		return -1, err
	}

	return monitStatus.GetIncarnation()
}

func (m monitJobSupervisor) AddJob(jobName string, jobIndex int, configPath string) error {
	targetFilename := fmt.Sprintf("%04d_%s.monitrc", jobIndex, jobName)
	targetConfigPath := filepath.Join(m.dirProvider.MonitJobsDir(), targetFilename)

	configContent, err := m.fs.ReadFile(configPath)
	if err != nil {
		return bosherr.WrapError(err, "Reading job config from file")
	}

	err = m.fs.WriteFile(targetConfigPath, configContent)
	if err != nil {
		return bosherr.WrapError(err, "Writing to job config file")
	}

	return nil
}

func (m monitJobSupervisor) RemoveAllJobs() error {
	return m.fs.RemoveAll(m.dirProvider.MonitJobsDir())
}

func (m monitJobSupervisor) MonitorJobFailures(handler JobFailureHandler) (err error) {
	alertHandler := func(smtpd.Connection, smtpd.MailAddress) (env smtpd.Envelope, err error) {
		env = &alertEnvelope{
			new(smtpd.BasicEnvelope),
			handler,
			new(boshalert.MonitAlert),
		}
		return
	}

	serv := &smtpd.Server{
		Addr:      fmt.Sprintf("127.0.0.1:%d", m.jobFailuresServerPort),
		OnNewMail: alertHandler,
	}

	err = serv.ListenAndServe()
	if err != nil {
		err = bosherr.WrapError(err, "Listen for SMTP")
	}
	return
}
<|endoftext|>


<commit_before>package dalga

import (
	"database/sql"
	"fmt"
	"time"
)

type table struct {
	db   *sql.DB
	name string
}

const createTableSQL = "" +
	"CREATE TABLE `%s` (" +
	"  `job`         VARCHAR(65535)  NOT NULL," +
	"  `routing_key` VARCHAR(255)    NOT NULL," +
	"  `interval`    INT UNSIGNED    NOT NULL," +
	"  `next_run`    DATETIME        NOT NULL," +
	"" +
	"  PRIMARY KEY (`job`, `routing_key`)," +
	"  KEY `idx_next_run` (`next_run`)" +
	") ENGINE=InnoDB DEFAULT CHARSET=utf8"

func (t *table) Create() error {
	sql := fmt.Sprintf(createTableSQL, t.name)
	_, err := t.db.Exec(sql)
	return err
}

// Insert job to to scheduler table.
func (t *table) Insert(j *Job) error {
	interval := j.Interval.Seconds()
	_, err := t.db.Exec("INSERT INTO "+t.name+" "+
		"(job, routing_key, `interval`, next_run) "+
		"VALUES(?, ?, ?, ?) "+
		"ON DUPLICATE KEY UPDATE "+
		"next_run=DATE_ADD(next_run, INTERVAL (? - `interval`) SECOND), "+
		"`interval`=?",
		j.Description, j.RoutingKey, interval, j.NextRun, interval, interval)
	return err
}

// Delete the job from scheduler table.
func (t *table) Delete(job, routingKey string) error {
	_, err := t.db.Exec("DELETE FROM "+t.name+" "+"WHERE job=? AND routing_key=?", job, routingKey)
	return err
}

// Front returns the next scheduled job from the table.
func (t *table) Front() (*Job, error) {
	var interval uint32
	var j Job
	row := t.db.QueryRow("SELECT job, routing_key, `interval`, next_run " +
		"FROM " + t.name + " " +
		"ORDER BY next_run ASC LIMIT 1")
	err := row.Scan(&j.Description, &j.RoutingKey, &interval, &j.NextRun)
	if err != nil {
		return nil, err
	}
	j.Interval = time.Duration(interval) * time.Second
	return &j, nil
}

// UpdateNextRun sets next_run to now+interval.
func (t *table) UpdateNextRun(j *Job) error {
	_, err := t.db.Exec("UPDATE "+t.name+" "+
		"SET next_run=? "+
		"WHERE job=? AND routing_key=?",
		time.Now().UTC().Add(j.Interval), j.Description, j.RoutingKey)
	return err
}
<commit_msg>change max job size<commit_after>package dalga

import (
	"database/sql"
	"fmt"
	"time"
)

type table struct {
	db   *sql.DB
	name string
}

const createTableSQL = "" +
	"CREATE TABLE `%s` (" +
	"  `job`         VARCHAR(20000)  NOT NULL," +
	"  `routing_key` VARCHAR(255)    NOT NULL," +
	"  `interval`    INT UNSIGNED    NOT NULL," +
	"  `next_run`    DATETIME        NOT NULL," +
	"" +
	"  PRIMARY KEY (`job`(255), `routing_key`)," +
	"  KEY `idx_next_run` (`next_run`)" +
	") ENGINE=InnoDB DEFAULT CHARSET=utf8"

func (t *table) Create() error {
	sql := fmt.Sprintf(createTableSQL, t.name)
	_, err := t.db.Exec(sql)
	return err
}

// Insert job to to scheduler table.
func (t *table) Insert(j *Job) error {
	interval := j.Interval.Seconds()
	_, err := t.db.Exec("INSERT INTO "+t.name+" "+
		"(job, routing_key, `interval`, next_run) "+
		"VALUES(?, ?, ?, ?) "+
		"ON DUPLICATE KEY UPDATE "+
		"next_run=DATE_ADD(next_run, INTERVAL (? - `interval`) SECOND), "+
		"`interval`=?",
		j.Description, j.RoutingKey, interval, j.NextRun, interval, interval)
	return err
}

// Delete the job from scheduler table.
func (t *table) Delete(job, routingKey string) error {
	_, err := t.db.Exec("DELETE FROM "+t.name+" "+"WHERE job=? AND routing_key=?", job, routingKey)
	return err
}

// Front returns the next scheduled job from the table.
func (t *table) Front() (*Job, error) {
	var interval uint32
	var j Job
	row := t.db.QueryRow("SELECT job, routing_key, `interval`, next_run " +
		"FROM " + t.name + " " +
		"ORDER BY next_run ASC LIMIT 1")
	err := row.Scan(&j.Description, &j.RoutingKey, &interval, &j.NextRun)
	if err != nil {
		return nil, err
	}
	j.Interval = time.Duration(interval) * time.Second
	return &j, nil
}

// UpdateNextRun sets next_run to now+interval.
func (t *table) UpdateNextRun(j *Job) error {
	_, err := t.db.Exec("UPDATE "+t.name+" "+
		"SET next_run=? "+
		"WHERE job=? AND routing_key=?",
		time.Now().UTC().Add(j.Interval), j.Description, j.RoutingKey)
	return err
}
<|endoftext|>


<commit_before>package bosh

import (
	"errors"
	"fmt"
	"os"
	"strings"

	yaml "gopkg.in/yaml.v2"

	"github.com/cloudfoundry/bosh-bootloader/storage"
)

var (
	osSetenv   = os.Setenv
	osUnsetenv = os.Unsetenv
)

const (
	DIRECTOR_USERNAME    = "admin"
	DIRECTOR_INTERNAL_IP = "10.0.0.6"
)

type Manager struct {
	executor    executor
	logger      logger
	socks5Proxy socks5Proxy
}

type directorVars struct {
	directorPassword       string
	directorSSLCA          string
	directorSSLCertificate string
	directorSSLPrivateKey  string
}

type deploymentVariables struct {
	DirectorName          string
	Zone                  string
	Network               string
	Subnetwork            string
	Tags                  []string
	ProjectID             string
	ExternalIP            string
	CredentialsJSON       string
	PrivateKey            string
	DefaultKeyName        string
	DefaultSecurityGroups []string
	SubnetID              string
	AZ                    string
	Region                string
	SecretAccessKey       string
	AccessKeyID           string
}

type iaasInputs struct {
	InterpolateInput InterpolateInput
}

type executor interface {
	DirectorInterpolate(InterpolateInput) (InterpolateOutput, error)
	JumpboxInterpolate(InterpolateInput) (JumpboxInterpolateOutput, error)
	CreateEnv(CreateEnvInput) (CreateEnvOutput, error)
	DeleteEnv(DeleteEnvInput) error
	Version() (string, error)
}

type logger interface {
	Step(string, ...interface{})
	Println(string)
}

type socks5Proxy interface {
	Start(string, string) error
	Addr() string
}

func NewManager(executor executor, logger logger, socks5Proxy socks5Proxy) Manager {
	return Manager{
		executor:    executor,
		logger:      logger,
		socks5Proxy: socks5Proxy,
	}
}

func (m Manager) Version() (string, error) {
	version, err := m.executor.Version()
	switch err.(type) {
	case BOSHVersionError:
		m.logger.Println("warning: BOSH version could not be parsed")
	}
	return version, err
}

func (m Manager) Create(state storage.State, terraformOutputs map[string]interface{}) (storage.State, error) {
	iaasInputs, err := m.generateIAASInputs(state)
	if err != nil {
		return storage.State{}, err
	}

	if state.Jumpbox.Enabled {
		state, err = m.createJumpbox(state, &iaasInputs, terraformOutputs)
		if err != nil {
			return storage.State{}, err
		}
	}

	state, err = m.createDirector(state, &iaasInputs, terraformOutputs)
	if err != nil {
		return storage.State{}, err
	}

	return state, nil
}

func (m Manager) Delete(state storage.State, terraformOutputs map[string]interface{}) error {
	iaasInputs, err := m.generateIAASInputs(state)
	if err != nil {
		return err
	}

	iaasInputs.InterpolateInput.DeploymentVars, err = m.GetDeploymentVars(state, terraformOutputs)
	if err != nil {
		return err //not tested
	}

	iaasInputs.InterpolateInput.OpsFile = state.BOSH.UserOpsFile

	interpolateOutputs, err := m.executor.DirectorInterpolate(iaasInputs.InterpolateInput)
	if err != nil {
		return err
	}

	err = m.executor.DeleteEnv(DeleteEnvInput{
		Manifest:  interpolateOutputs.Manifest,
		State:     state.BOSH.State,
		Variables: interpolateOutputs.Variables,
	})
	switch err.(type) {
	case DeleteEnvError:
		deErr := err.(DeleteEnvError)
		state.BOSH.State = deErr.BOSHState()
		return NewManagerDeleteError(state, err)
	case error:
		return err
	}

	return nil
}

func (m Manager) GetJumpboxDeploymentVars(state storage.State, terraformOutputs map[string]interface{}) (string, error) {
	vars := strings.Join([]string{
		"internal_cidr: 10.0.0.0/24",
		"internal_gw: 10.0.0.1",
		"internal_ip: 10.0.0.5",
		fmt.Sprintf("director_name: %s", fmt.Sprintf("bosh-%s", state.EnvID)),
		fmt.Sprintf("external_ip: %s", terraformOutputs["external_ip"]),
		fmt.Sprintf("zone: %s", state.GCP.Zone),
		fmt.Sprintf("network: %s", terraformOutputs["network_name"]),
		fmt.Sprintf("subnetwork: %s", terraformOutputs["subnetwork_name"]),
		fmt.Sprintf("tags: [%s, %s]", terraformOutputs["bosh_open_tag_name"], terraformOutputs["internal_tag_name"]),
		fmt.Sprintf("project_id: %s", state.GCP.ProjectID),
		fmt.Sprintf("gcp_credentials_json: '%s'", state.GCP.ServiceAccountKey),
	}, "\n")

	return strings.TrimSuffix(vars, "\n"), nil
}

func (m Manager) GetDeploymentVars(state storage.State, terraformOutputs map[string]interface{}) (string, error) {
	var vars string

	switch state.IAAS {
	case "gcp":
		if state.Jumpbox.Enabled {
			vars = strings.Join([]string{
				"internal_cidr: 10.0.0.0/24",
				"internal_gw: 10.0.0.1",
				fmt.Sprintf("internal_ip: %s", DIRECTOR_INTERNAL_IP),
				fmt.Sprintf("director_name: %s", fmt.Sprintf("bosh-%s", state.EnvID)),
				fmt.Sprintf("zone: %s", state.GCP.Zone),
				fmt.Sprintf("network: %s", terraformOutputs["network_name"]),
				fmt.Sprintf("subnetwork: %s", terraformOutputs["subnetwork_name"]),
				fmt.Sprintf("tags: [%s]", terraformOutputs["internal_tag_name"]),
				fmt.Sprintf("project_id: %s", state.GCP.ProjectID),
				fmt.Sprintf("gcp_credentials_json: '%s'", state.GCP.ServiceAccountKey),
			}, "\n")
		} else {
			vars = strings.Join([]string{
				"internal_cidr: 10.0.0.0/24",
				"internal_gw: 10.0.0.1",
				fmt.Sprintf("internal_ip: %s", DIRECTOR_INTERNAL_IP),
				fmt.Sprintf("director_name: %s", fmt.Sprintf("bosh-%s", state.EnvID)),
				fmt.Sprintf("external_ip: %s", terraformOutputs["external_ip"]),
				fmt.Sprintf("zone: %s", state.GCP.Zone),
				fmt.Sprintf("network: %s", terraformOutputs["network_name"]),
				fmt.Sprintf("subnetwork: %s", terraformOutputs["subnetwork_name"]),
				fmt.Sprintf("tags: [%s, %s]", terraformOutputs["bosh_open_tag_name"], terraformOutputs["internal_tag_name"]),
				fmt.Sprintf("project_id: %s", state.GCP.ProjectID),
				fmt.Sprintf("gcp_credentials_json: '%s'", state.GCP.ServiceAccountKey),
			}, "\n")
		}
	case "aws":
		vars = strings.Join([]string{
			"internal_cidr: 10.0.0.0/24",
			"internal_gw: 10.0.0.1",
			fmt.Sprintf("internal_ip: %s", DIRECTOR_INTERNAL_IP),
			fmt.Sprintf("director_name: %s", fmt.Sprintf("bosh-%s", state.EnvID)),
			fmt.Sprintf("external_ip: %s", terraformOutputs["external_ip"]),
			fmt.Sprintf("az: %s", terraformOutputs["az"]),
			fmt.Sprintf("subnet_id: %s", terraformOutputs["subnet_id"]),
			fmt.Sprintf("access_key_id: %s", terraformOutputs["access_key_id"]),
			fmt.Sprintf("secret_access_key: %s", terraformOutputs["secret_access_key"]),
			fmt.Sprintf("default_key_name: %s", state.KeyPair.Name),
			fmt.Sprintf("default_security_groups: [%s]", terraformOutputs["default_security_groups"]),
			fmt.Sprintf("region: %s", state.AWS.Region),
			fmt.Sprintf("private_key: |-\n  %s", strings.Replace(state.KeyPair.PrivateKey, "\n", "\n  ", -1)),
		}, "\n")
	}

	return strings.TrimSuffix(vars, "\n"), nil
}

func (m Manager) generateIAASInputs(state storage.State) (iaasInputs, error) {
	switch state.IAAS {
	case "gcp", "aws":
		return iaasInputs{
			InterpolateInput: InterpolateInput{
				IAAS:      state.IAAS,
				BOSHState: state.BOSH.State,
				Variables: state.BOSH.Variables,
			},
		}, nil
	default:
		return iaasInputs{}, errors.New("A valid IAAS was not provided")
	}
}

func getJumpboxPrivateKey(v string) (string, error) {
	variables := map[string]interface{}{}

	err := yaml.Unmarshal([]byte(v), &variables)
	if err != nil {
		return "", err
	}

	jumpboxMap := variables["jumpbox_ssh"].(map[interface{}]interface{})
	jumpboxSSH := map[string]string{}
	for k, v := range jumpboxMap {
		jumpboxSSH[k.(string)] = v.(string)
	}

	return jumpboxSSH["private_key"], nil
}

func getDirectorVars(v string) (directorVars, error) {
	variables := map[string]interface{}{}

	err := yaml.Unmarshal([]byte(v), &variables)
	if err != nil {
		return directorVars{}, err
	}

	directorSSLInterfaceMap := variables["director_ssl"].(map[interface{}]interface{})
	directorSSL := map[string]string{}
	for k, v := range directorSSLInterfaceMap {
		directorSSL[k.(string)] = v.(string)
	}

	return directorVars{
		directorPassword:       variables["admin_password"].(string),
		directorSSLCA:          directorSSL["ca"],
		directorSSLCertificate: directorSSL["certificate"],
		directorSSLPrivateKey:  directorSSL["private_key"],
	}, nil
}

func (m Manager) createJumpbox(state storage.State, iaasInputs *iaasInputs, terraformOutputs map[string]interface{}) (storage.State, error) {
	var err error
	m.logger.Step("creating jumpbox")
	iaasInputs.InterpolateInput.JumpboxDeploymentVars, err = m.GetJumpboxDeploymentVars(state, terraformOutputs)
	if err != nil {
		return storage.State{}, err //not tested
	}

	interpolateOutputs, err := m.executor.JumpboxInterpolate(iaasInputs.InterpolateInput)
	if err != nil {
		return storage.State{}, err
	}

	variables, err := yaml.Marshal(interpolateOutputs.Variables)
	if err != nil {
		return storage.State{}, err
	}

	osUnsetenv("BOSH_ALL_PROXY")
	createEnvOutputs, err := m.executor.CreateEnv(CreateEnvInput{
		Manifest:  interpolateOutputs.Manifest,
		State:     state.Jumpbox.State,
		Variables: string(variables),
	})
	switch err.(type) {
	case CreateEnvError:
		ceErr := err.(CreateEnvError)
		state.Jumpbox = storage.Jumpbox{
			Enabled:   true,
			Variables: interpolateOutputs.Variables,
			State:     ceErr.BOSHState(),
			Manifest:  interpolateOutputs.Manifest,
		}
		return storage.State{}, NewManagerCreateError(state, err)
	case error:
		return storage.State{}, err
	}

	state.Jumpbox = storage.Jumpbox{
		Enabled:   true,
		Variables: interpolateOutputs.Variables,
		State:     createEnvOutputs.State,
		Manifest:  interpolateOutputs.Manifest,
		URL:       terraformOutputs["jumpbox_url"].(string),
	}

	m.logger.Step("created jumpbox")

	m.logger.Step("starting socks5 proxy to jumpbox")
	jumpboxPrivateKey, err := getJumpboxPrivateKey(interpolateOutputs.Variables)
	if err != nil {
		return storage.State{}, err
	}

	err = m.socks5Proxy.Start(jumpboxPrivateKey, state.Jumpbox.URL)
	if err != nil {
		return storage.State{}, err
	}

	osSetenv("BOSH_ALL_PROXY", fmt.Sprintf("socks5://%s", m.socks5Proxy.Addr()))

	return state, nil
}

func (m Manager) createDirector(state storage.State, iaasInputs *iaasInputs, terraformOutputs map[string]interface{}) (storage.State, error) {
	var err error
	var directorAddress string

	directorAddress = terraformOutputs["director_address"].(string)

	if state.Jumpbox.Enabled {
		directorAddress = fmt.Sprintf("https://%s:25555", DIRECTOR_INTERNAL_IP)
	}

	m.logger.Step("creating bosh director")
	iaasInputs.InterpolateInput.DeploymentVars, err = m.GetDeploymentVars(state, terraformOutputs)
	if err != nil {
		return storage.State{}, err //not tested
	}

	iaasInputs.InterpolateInput.OpsFile = state.BOSH.UserOpsFile

	interpolateOutputs, err := m.executor.DirectorInterpolate(iaasInputs.InterpolateInput)
	if err != nil {
		return storage.State{}, err
	}

	createEnvOutputs, err := m.executor.CreateEnv(CreateEnvInput{
		Manifest:  interpolateOutputs.Manifest,
		State:     state.BOSH.State,
		Variables: interpolateOutputs.Variables,
	})
	switch err.(type) {
	case CreateEnvError:
		ceErr := err.(CreateEnvError)
		state.BOSH = storage.BOSH{
			Variables: interpolateOutputs.Variables,
			State:     ceErr.BOSHState(),
			Manifest:  interpolateOutputs.Manifest,
		}
		return storage.State{}, NewManagerCreateError(state, err)
	case error:
		return storage.State{}, err
	}

	directorVars, err := getDirectorVars(interpolateOutputs.Variables)
	if err != nil {
		return storage.State{}, fmt.Errorf("failed to get director outputs:\n%s", err.Error())
	}

	state.BOSH = storage.BOSH{
		DirectorName:           fmt.Sprintf("bosh-%s", state.EnvID),
		DirectorAddress:        directorAddress,
		DirectorUsername:       DIRECTOR_USERNAME,
		DirectorPassword:       directorVars.directorPassword,
		DirectorSSLCA:          directorVars.directorSSLCA,
		DirectorSSLCertificate: directorVars.directorSSLCertificate,
		DirectorSSLPrivateKey:  directorVars.directorSSLPrivateKey,
		Variables:              interpolateOutputs.Variables,
		State:                  createEnvOutputs.State,
		Manifest:               interpolateOutputs.Manifest,
	}

	m.logger.Step("created bosh director")
	return state, nil
}
<commit_msg>Unwrap IAASInputs into an InterpolateInput struct<commit_after>package bosh

import (
	"errors"
	"fmt"
	"os"
	"strings"

	yaml "gopkg.in/yaml.v2"

	"github.com/cloudfoundry/bosh-bootloader/storage"
)

var (
	osSetenv   = os.Setenv
	osUnsetenv = os.Unsetenv
)

const (
	DIRECTOR_USERNAME    = "admin"
	DIRECTOR_INTERNAL_IP = "10.0.0.6"
)

type Manager struct {
	executor    executor
	logger      logger
	socks5Proxy socks5Proxy
}

type directorVars struct {
	directorPassword       string
	directorSSLCA          string
	directorSSLCertificate string
	directorSSLPrivateKey  string
}

type deploymentVariables struct {
	DirectorName          string
	Zone                  string
	Network               string
	Subnetwork            string
	Tags                  []string
	ProjectID             string
	ExternalIP            string
	CredentialsJSON       string
	PrivateKey            string
	DefaultKeyName        string
	DefaultSecurityGroups []string
	SubnetID              string
	AZ                    string
	Region                string
	SecretAccessKey       string
	AccessKeyID           string
}

type executor interface {
	DirectorInterpolate(InterpolateInput) (InterpolateOutput, error)
	JumpboxInterpolate(InterpolateInput) (JumpboxInterpolateOutput, error)
	CreateEnv(CreateEnvInput) (CreateEnvOutput, error)
	DeleteEnv(DeleteEnvInput) error
	Version() (string, error)
}

type logger interface {
	Step(string, ...interface{})
	Println(string)
}

type socks5Proxy interface {
	Start(string, string) error
	Addr() string
}

func NewManager(executor executor, logger logger, socks5Proxy socks5Proxy) Manager {
	return Manager{
		executor:    executor,
		logger:      logger,
		socks5Proxy: socks5Proxy,
	}
}

func (m Manager) Version() (string, error) {
	version, err := m.executor.Version()
	switch err.(type) {
	case BOSHVersionError:
		m.logger.Println("warning: BOSH version could not be parsed")
	}
	return version, err
}

func (m Manager) Create(state storage.State, terraformOutputs map[string]interface{}) (storage.State, error) {
	iaasInputs, err := m.generateIAASInputs(state)
	if err != nil {
		return storage.State{}, err
	}

	if state.Jumpbox.Enabled {
		state, err = m.createJumpbox(state, &iaasInputs, terraformOutputs)
		if err != nil {
			return storage.State{}, err
		}
	}

	state, err = m.createDirector(state, &iaasInputs, terraformOutputs)
	if err != nil {
		return storage.State{}, err
	}

	return state, nil
}

func (m Manager) Delete(state storage.State, terraformOutputs map[string]interface{}) error {
	iaasInputs, err := m.generateIAASInputs(state)
	if err != nil {
		return err
	}

	iaasInputs.DeploymentVars, err = m.GetDeploymentVars(state, terraformOutputs)
	if err != nil {
		return err //not tested
	}

	iaasInputs.OpsFile = state.BOSH.UserOpsFile

	interpolateOutputs, err := m.executor.DirectorInterpolate(iaasInputs)
	if err != nil {
		return err
	}

	err = m.executor.DeleteEnv(DeleteEnvInput{
		Manifest:  interpolateOutputs.Manifest,
		State:     state.BOSH.State,
		Variables: interpolateOutputs.Variables,
	})
	switch err.(type) {
	case DeleteEnvError:
		deErr := err.(DeleteEnvError)
		state.BOSH.State = deErr.BOSHState()
		return NewManagerDeleteError(state, err)
	case error:
		return err
	}

	return nil
}

func (m Manager) GetJumpboxDeploymentVars(state storage.State, terraformOutputs map[string]interface{}) (string, error) {
	vars := strings.Join([]string{
		"internal_cidr: 10.0.0.0/24",
		"internal_gw: 10.0.0.1",
		"internal_ip: 10.0.0.5",
		fmt.Sprintf("director_name: %s", fmt.Sprintf("bosh-%s", state.EnvID)),
		fmt.Sprintf("external_ip: %s", terraformOutputs["external_ip"]),
		fmt.Sprintf("zone: %s", state.GCP.Zone),
		fmt.Sprintf("network: %s", terraformOutputs["network_name"]),
		fmt.Sprintf("subnetwork: %s", terraformOutputs["subnetwork_name"]),
		fmt.Sprintf("tags: [%s, %s]", terraformOutputs["bosh_open_tag_name"], terraformOutputs["internal_tag_name"]),
		fmt.Sprintf("project_id: %s", state.GCP.ProjectID),
		fmt.Sprintf("gcp_credentials_json: '%s'", state.GCP.ServiceAccountKey),
	}, "\n")

	return strings.TrimSuffix(vars, "\n"), nil
}

func (m Manager) GetDeploymentVars(state storage.State, terraformOutputs map[string]interface{}) (string, error) {
	var vars string

	switch state.IAAS {
	case "gcp":
		if state.Jumpbox.Enabled {
			vars = strings.Join([]string{
				"internal_cidr: 10.0.0.0/24",
				"internal_gw: 10.0.0.1",
				fmt.Sprintf("internal_ip: %s", DIRECTOR_INTERNAL_IP),
				fmt.Sprintf("director_name: %s", fmt.Sprintf("bosh-%s", state.EnvID)),
				fmt.Sprintf("zone: %s", state.GCP.Zone),
				fmt.Sprintf("network: %s", terraformOutputs["network_name"]),
				fmt.Sprintf("subnetwork: %s", terraformOutputs["subnetwork_name"]),
				fmt.Sprintf("tags: [%s]", terraformOutputs["internal_tag_name"]),
				fmt.Sprintf("project_id: %s", state.GCP.ProjectID),
				fmt.Sprintf("gcp_credentials_json: '%s'", state.GCP.ServiceAccountKey),
			}, "\n")
		} else {
			vars = strings.Join([]string{
				"internal_cidr: 10.0.0.0/24",
				"internal_gw: 10.0.0.1",
				fmt.Sprintf("internal_ip: %s", DIRECTOR_INTERNAL_IP),
				fmt.Sprintf("director_name: %s", fmt.Sprintf("bosh-%s", state.EnvID)),
				fmt.Sprintf("external_ip: %s", terraformOutputs["external_ip"]),
				fmt.Sprintf("zone: %s", state.GCP.Zone),
				fmt.Sprintf("network: %s", terraformOutputs["network_name"]),
				fmt.Sprintf("subnetwork: %s", terraformOutputs["subnetwork_name"]),
				fmt.Sprintf("tags: [%s, %s]", terraformOutputs["bosh_open_tag_name"], terraformOutputs["internal_tag_name"]),
				fmt.Sprintf("project_id: %s", state.GCP.ProjectID),
				fmt.Sprintf("gcp_credentials_json: '%s'", state.GCP.ServiceAccountKey),
			}, "\n")
		}
	case "aws":
		vars = strings.Join([]string{
			"internal_cidr: 10.0.0.0/24",
			"internal_gw: 10.0.0.1",
			fmt.Sprintf("internal_ip: %s", DIRECTOR_INTERNAL_IP),
			fmt.Sprintf("director_name: %s", fmt.Sprintf("bosh-%s", state.EnvID)),
			fmt.Sprintf("external_ip: %s", terraformOutputs["external_ip"]),
			fmt.Sprintf("az: %s", terraformOutputs["az"]),
			fmt.Sprintf("subnet_id: %s", terraformOutputs["subnet_id"]),
			fmt.Sprintf("access_key_id: %s", terraformOutputs["access_key_id"]),
			fmt.Sprintf("secret_access_key: %s", terraformOutputs["secret_access_key"]),
			fmt.Sprintf("default_key_name: %s", state.KeyPair.Name),
			fmt.Sprintf("default_security_groups: [%s]", terraformOutputs["default_security_groups"]),
			fmt.Sprintf("region: %s", state.AWS.Region),
			fmt.Sprintf("private_key: |-\n  %s", strings.Replace(state.KeyPair.PrivateKey, "\n", "\n  ", -1)),
		}, "\n")
	}

	return strings.TrimSuffix(vars, "\n"), nil
}

func (m Manager) generateIAASInputs(state storage.State) (InterpolateInput, error) {
	switch state.IAAS {
	case "gcp", "aws":
		return InterpolateInput{
			IAAS:      state.IAAS,
			BOSHState: state.BOSH.State,
			Variables: state.BOSH.Variables,
		}, nil
	default:
		return InterpolateInput{}, errors.New("A valid IAAS was not provided")
	}
}

func getJumpboxPrivateKey(v string) (string, error) {
	variables := map[string]interface{}{}

	err := yaml.Unmarshal([]byte(v), &variables)
	if err != nil {
		return "", err
	}

	jumpboxMap := variables["jumpbox_ssh"].(map[interface{}]interface{})
	jumpboxSSH := map[string]string{}
	for k, v := range jumpboxMap {
		jumpboxSSH[k.(string)] = v.(string)
	}

	return jumpboxSSH["private_key"], nil
}

func getDirectorVars(v string) (directorVars, error) {
	variables := map[string]interface{}{}

	err := yaml.Unmarshal([]byte(v), &variables)
	if err != nil {
		return directorVars{}, err
	}

	directorSSLInterfaceMap := variables["director_ssl"].(map[interface{}]interface{})
	directorSSL := map[string]string{}
	for k, v := range directorSSLInterfaceMap {
		directorSSL[k.(string)] = v.(string)
	}

	return directorVars{
		directorPassword:       variables["admin_password"].(string),
		directorSSLCA:          directorSSL["ca"],
		directorSSLCertificate: directorSSL["certificate"],
		directorSSLPrivateKey:  directorSSL["private_key"],
	}, nil
}

func (m Manager) createJumpbox(state storage.State, iaasInputs *InterpolateInput, terraformOutputs map[string]interface{}) (storage.State, error) {
	var err error
	m.logger.Step("creating jumpbox")
	iaasInputs.JumpboxDeploymentVars, err = m.GetJumpboxDeploymentVars(state, terraformOutputs)
	if err != nil {
		return storage.State{}, err //not tested
	}

	interpolateOutputs, err := m.executor.JumpboxInterpolate(*iaasInputs)
	if err != nil {
		return storage.State{}, err
	}

	variables, err := yaml.Marshal(interpolateOutputs.Variables)
	if err != nil {
		return storage.State{}, err
	}

	osUnsetenv("BOSH_ALL_PROXY")
	createEnvOutputs, err := m.executor.CreateEnv(CreateEnvInput{
		Manifest:  interpolateOutputs.Manifest,
		State:     state.Jumpbox.State,
		Variables: string(variables),
	})
	switch err.(type) {
	case CreateEnvError:
		ceErr := err.(CreateEnvError)
		state.Jumpbox = storage.Jumpbox{
			Enabled:   true,
			Variables: interpolateOutputs.Variables,
			State:     ceErr.BOSHState(),
			Manifest:  interpolateOutputs.Manifest,
		}
		return storage.State{}, NewManagerCreateError(state, err)
	case error:
		return storage.State{}, err
	}

	state.Jumpbox = storage.Jumpbox{
		Enabled:   true,
		Variables: interpolateOutputs.Variables,
		State:     createEnvOutputs.State,
		Manifest:  interpolateOutputs.Manifest,
		URL:       terraformOutputs["jumpbox_url"].(string),
	}

	m.logger.Step("created jumpbox")

	m.logger.Step("starting socks5 proxy to jumpbox")
	jumpboxPrivateKey, err := getJumpboxPrivateKey(interpolateOutputs.Variables)
	if err != nil {
		return storage.State{}, err
	}

	err = m.socks5Proxy.Start(jumpboxPrivateKey, state.Jumpbox.URL)
	if err != nil {
		return storage.State{}, err
	}

	osSetenv("BOSH_ALL_PROXY", fmt.Sprintf("socks5://%s", m.socks5Proxy.Addr()))

	return state, nil
}

func (m Manager) createDirector(state storage.State, iaasInputs *InterpolateInput, terraformOutputs map[string]interface{}) (storage.State, error) {
	var err error
	var directorAddress string

	directorAddress = terraformOutputs["director_address"].(string)

	if state.Jumpbox.Enabled {
		directorAddress = fmt.Sprintf("https://%s:25555", DIRECTOR_INTERNAL_IP)
	}

	m.logger.Step("creating bosh director")
	iaasInputs.DeploymentVars, err = m.GetDeploymentVars(state, terraformOutputs)
	if err != nil {
		return storage.State{}, err //not tested
	}

	iaasInputs.OpsFile = state.BOSH.UserOpsFile

	interpolateOutputs, err := m.executor.DirectorInterpolate(*iaasInputs)
	if err != nil {
		return storage.State{}, err
	}

	createEnvOutputs, err := m.executor.CreateEnv(CreateEnvInput{
		Manifest:  interpolateOutputs.Manifest,
		State:     state.BOSH.State,
		Variables: interpolateOutputs.Variables,
	})
	switch err.(type) {
	case CreateEnvError:
		ceErr := err.(CreateEnvError)
		state.BOSH = storage.BOSH{
			Variables: interpolateOutputs.Variables,
			State:     ceErr.BOSHState(),
			Manifest:  interpolateOutputs.Manifest,
		}
		return storage.State{}, NewManagerCreateError(state, err)
	case error:
		return storage.State{}, err
	}

	directorVars, err := getDirectorVars(interpolateOutputs.Variables)
	if err != nil {
		return storage.State{}, fmt.Errorf("failed to get director outputs:\n%s", err.Error())
	}

	state.BOSH = storage.BOSH{
		DirectorName:           fmt.Sprintf("bosh-%s", state.EnvID),
		DirectorAddress:        directorAddress,
		DirectorUsername:       DIRECTOR_USERNAME,
		DirectorPassword:       directorVars.directorPassword,
		DirectorSSLCA:          directorVars.directorSSLCA,
		DirectorSSLCertificate: directorVars.directorSSLCertificate,
		DirectorSSLPrivateKey:  directorVars.directorSSLPrivateKey,
		Variables:              interpolateOutputs.Variables,
		State:                  createEnvOutputs.State,
		Manifest:               interpolateOutputs.Manifest,
	}

	m.logger.Step("created bosh director")
	return state, nil
}
<|endoftext|>


<commit_before>package slog

import (
	"net"
	"time"
)

type _UdpNetPrinter struct {
	ch          chan *string
	blockMillis time.Duration
	conn        *net.UDPConn
}

func _NewUdpNetPrinter(addr string, maxrn int, blockMillis int) (_Printer, error) {
	udpAddr, err := net.ResolveUDPAddr("udp", addr)
	if err != nil {
		return nil, err
	}
	if blockMillis < 10 {
		blockMillis = 10
	}
	p := &_UdpNetPrinter{
		ch:          make(chan *string, maxrn),
		blockMillis: time.Millisecond * time.Duration(blockMillis),
	}
	p.conn, err = net.DialUDP("udp", nil, udpAddr)
	if err != nil {
		return nil, err
	}
	t := time.Now()
	s := t.Format("[01-02 15:04:05.999][NOTE ] ------------ start ------------\n")
	p.Print(&s)
	go p.flush()
	return p, err

}

func (this *_UdpNetPrinter) Print(s *string) {
	select {
	case this.ch <- s:
	case <-time.After(this.blockMillis):
		es := "UdpPrinter enqueue timeout:" + *s
		slogerr(&es)
	}
}

func (this *_UdpNetPrinter) flush() {
	var bs []byte
	for v := range this.ch {
		bs = []byte(*v)
		this.conn.Write(bs)
	}
}
<commit_msg>Delete udpprinter.go<commit_after><|endoftext|>


<commit_before>package device

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/ScriptRock/crypto/ssh"
)

type CiscoDevice struct {
	Username  string
	Password  string
	Enable    string
	name      string
	Hostname  string
	stdin     io.WriteCloser
	stdout    io.Reader
	session   *ssh.Session
	Echo      bool
	EnableLog bool
	Logdir    string
	Log       *os.File
	Prompt    string
}

func (d *CiscoDevice) Connect() error {
	config := &ssh.ClientConfig{
		User: d.Username,
		Auth: []ssh.AuthMethod{
			ssh.Password(d.Password),
		},
		Config: ssh.Config{
			Ciphers: ssh.AllSupportedCiphers(),
		},
	}
	client, err := ssh.Dial("tcp", d.Hostname+":22", config)
	if err != nil {
		return err
	}
	session, err := client.NewSession()
	if err != nil {
		return err
	}
	d.stdin, _ = session.StdinPipe()
	d.stdout, _ = session.StdoutPipe()
	d.Echo = true
	d.EnableLog = true
	modes := ssh.TerminalModes{
		ssh.ECHO:          0, // disable echoing
		ssh.OCRNL:         0,
		ssh.TTY_OP_ISPEED: 38400, // input speed = 14.4kbaud
		ssh.TTY_OP_OSPEED: 38400, // output speed = 14.4kbaud
	}
	session.RequestPty("vt100", 0, 2000, modes)
	session.Shell()
	if d.Logdir != "" {
		t := time.Now()
		d.Log, err = os.OpenFile(filepath.Join(d.Logdir, t.Format("200601021504")+"-"+d.Hostname), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			return err
		}
	}
	d.init()
	d.session = session
	return nil
}

func (d *CiscoDevice) Close() {
	d.session.Close()
}

func (d *CiscoDevice) Cmd(cmd string) (string, error) {
	bufstdout := bufio.NewReader(d.stdout)
	lines := strings.Split(cmd, "!")
	for _, line := range lines {
		io.WriteString(d.stdin, line+"\n")
		time.Sleep(time.Millisecond * 100)
	}
	output, err := d.readln(bufstdout)
	if err != nil {
		return "", err
	}
	output = strings.Replace(output, "\r", "", -1)
	if d.Echo == false {
		output = strings.Replace(output, lines[0], "", 1)
	}
	if d.Logdir != "" {
		return "", nil
	}
	return output, nil
}

func (d *CiscoDevice) init() {
	bufstdout := bufio.NewReader(d.stdout)
	io.WriteString(d.stdin, "enable\n")
	time.Sleep(time.Millisecond * 100)
	re := regexp.MustCompile("assword:")
	buf := make([]byte, 1000)
	loadStr := ""
	for {
		n, err := bufstdout.Read(buf)
		if err != nil {
			log.Fatal(err)
		}
		loadStr += string(buf[:n])
		if re.MatchString(loadStr) {
			io.WriteString(d.stdin, d.Enable+"\n")
			break
		} else {
			break
		}
	}
	d.Cmd("terminal length 0")
	d.Cmd("")
	prompt, _ := d.Cmd("")
	d.Prompt = strings.TrimSpace(prompt)
}

func (d *CiscoDevice) readln(r *bufio.Reader) (string, error) {
	var re *regexp.Regexp
	if d.Prompt == "" {
		re = regexp.MustCompile("[[:alnum:]]#.?$")
	} else {
		re = regexp.MustCompile(d.Prompt + ".?$")
	}
	buf := make([]byte, 10000)
	loadStr := ""
	for {
		n, err := r.Read(buf)
		if err != nil {
			return "", err
		}
		loadStr += string(buf[:n])
		// logging to file if necessary
		if d.Logdir != "" {
			if d.EnableLog {
				fmt.Fprint(d.Log, string(buf[:n]))
				loadStr=""
			}
		}
		if re.MatchString(string(buf[:n])) {
			break
		}
	}
	return loadStr, nil
}
<commit_msg>Do not clear string, pipe needs always output even when logging is enabled<commit_after>package device

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/ScriptRock/crypto/ssh"
)

type CiscoDevice struct {
	Username  string
	Password  string
	Enable    string
	name      string
	Hostname  string
	stdin     io.WriteCloser
	stdout    io.Reader
	session   *ssh.Session
	Echo      bool
	EnableLog bool
	Logdir    string
	Log       *os.File
	Prompt    string
}

func (d *CiscoDevice) Connect() error {
	config := &ssh.ClientConfig{
		User: d.Username,
		Auth: []ssh.AuthMethod{
			ssh.Password(d.Password),
		},
		Config: ssh.Config{
			Ciphers: ssh.AllSupportedCiphers(),
		},
	}
	client, err := ssh.Dial("tcp", d.Hostname+":22", config)
	if err != nil {
		return err
	}
	session, err := client.NewSession()
	if err != nil {
		return err
	}
	d.stdin, _ = session.StdinPipe()
	d.stdout, _ = session.StdoutPipe()
	d.Echo = true
	d.EnableLog = true
	modes := ssh.TerminalModes{
		ssh.ECHO:          0, // disable echoing
		ssh.OCRNL:         0,
		ssh.TTY_OP_ISPEED: 38400, // input speed = 14.4kbaud
		ssh.TTY_OP_OSPEED: 38400, // output speed = 14.4kbaud
	}
	session.RequestPty("vt100", 0, 2000, modes)
	session.Shell()
	if d.Logdir != "" {
		t := time.Now()
		d.Log, err = os.OpenFile(filepath.Join(d.Logdir, t.Format("200601021504")+"-"+d.Hostname), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			return err
		}
	}
	d.init()
	d.session = session
	return nil
}

func (d *CiscoDevice) Close() {
	d.session.Close()
}

func (d *CiscoDevice) Cmd(cmd string) (string, error) {
	bufstdout := bufio.NewReader(d.stdout)
	lines := strings.Split(cmd, "!")
	for _, line := range lines {
		io.WriteString(d.stdin, line+"\n")
		time.Sleep(time.Millisecond * 100)
	}
	output, err := d.readln(bufstdout)
	if err != nil {
		return "", err
	}
	output = strings.Replace(output, "\r", "", -1)
	if d.Echo == false {
		output = strings.Replace(output, lines[0], "", 1)
	}
	if d.Logdir != "" {
		return "", nil
	}
	return output, nil
}

func (d *CiscoDevice) init() {
	bufstdout := bufio.NewReader(d.stdout)
	io.WriteString(d.stdin, "enable\n")
	time.Sleep(time.Millisecond * 100)
	re := regexp.MustCompile("assword:")
	buf := make([]byte, 1000)
	loadStr := ""
	for {
		n, err := bufstdout.Read(buf)
		if err != nil {
			log.Fatal(err)
		}
		loadStr += string(buf[:n])
		if re.MatchString(loadStr) {
			io.WriteString(d.stdin, d.Enable+"\n")
			break
		} else {
			break
		}
	}
	d.Cmd("terminal length 0")
	d.Cmd("")
	prompt, _ := d.Cmd("")
	d.Prompt = strings.TrimSpace(prompt)
}

func (d *CiscoDevice) readln(r *bufio.Reader) (string, error) {
	var re *regexp.Regexp
	if d.Prompt == "" {
		re = regexp.MustCompile("[[:alnum:]]#.?$")
	} else {
		re = regexp.MustCompile(d.Prompt + ".?$")
	}
	buf := make([]byte, 10000)
	loadStr := ""
	for {
		n, err := r.Read(buf)
		if err != nil {
			return "", err
		}
		loadStr += string(buf[:n])
		// logging to file if necessary
		if d.Logdir != "" {
			if d.EnableLog {
				fmt.Fprint(d.Log, string(buf[:n]))
			}
		}
		if re.MatchString(string(buf[:n])) {
			break
		}
	}
	return loadStr, nil
}
<|endoftext|>


<commit_before>package device

import (
	"github.com/Comcast/webpa-common/logging"
	"github.com/spf13/viper"
)

func NewOptions(logger logging.Logger, v *viper.Viper) (o *Options, err error) {
	o = new(Options)
	if v != nil {
		err = v.Unmarshal(o)
	}

	o.Logger = logger
	return
}
<commit_msg>Comments; added a standard subkey<commit_after>package device

import (
	"github.com/Comcast/webpa-common/logging"
	"github.com/spf13/viper"
)

const (
	// DeviceManagerKey is the Viper subkey under which device.Options are typically stored
	DeviceManagerKey = "deviceManager"
)

// NewOptions unmarshals a device.Options from a Viper environment.  Listeners
// must be configured separately.
func NewOptions(logger logging.Logger, v *viper.Viper) (o *Options, err error) {
	o = new(Options)
	if v != nil {
		err = v.Unmarshal(o)
	}

	o.Logger = logger
	return
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/digitalocean/godo"
	"golang.org/x/oauth2"
	"os"
	"strconv"
	"strings"
	"text/tabwriter"
)

type TokenSource struct {
	AccessToken string
}

func (t *TokenSource) Token() (*oauth2.Token, error) {
	token := &oauth2.Token{
		AccessToken: t.AccessToken,
	}
	return token, nil
}

type DigitalOcean struct {
	config Config
	svc    *godo.Client
}

func NewDigitalOcean(config Config) DigitalOcean {
	tokenSource := &TokenSource{
		AccessToken: config.AccessToken,
	}

	oauthClient := oauth2.NewClient(oauth2.NoContext, tokenSource)
	svc := godo.NewClient(oauthClient)
	return DigitalOcean{config, svc}
}

func (d DigitalOcean) listInstances() {
	// Get droplet list
	list := []godo.Droplet{}
	opt := &godo.ListOptions{}
	for {
		droplets, resp, err := d.svc.Droplets.List(opt)
		if err != nil {
			return
		}
		for _, d := range droplets {
			list = append(list, d)
		}
		if resp.Links == nil || resp.Links.IsLastPage() {
			break
		}
		page, err := resp.Links.CurrentPage()
		if err != nil {
			return
		}
		opt.Page = page + 1
	}

	w := new(tabwriter.Writer)
	w.Init(os.Stdout, 0, 8, 0, '\t', 0)
	fmt.Fprintln(w, "DropletID\tName\tPublic IP\tPrivateIP")
	fmt.Fprintln(w, "---\t---\t---\t---")

	total := 0
	for _, drop := range list {
		public, private := getNetworks(&drop)

		// Print instance details using tabwriter
		fmt.Fprintf(
			w,
			"%d\t%s\t%s\t%s\n",
			drop.ID,
			drop.Name,
			strings.Join(public, ","),
			strings.Join(private, ","),
		)
		total++
	}
	w.Flush()
	fmt.Printf("---\nFound %d instances.\n", total)
}

func getNetworks(drop *godo.Droplet) (public, private []string) {
	// Get lists for public and private networks
	nets := drop.Networks.V4
	for _, net := range nets {
		if net.Type == "public" {
			public = append(public, net.IPAddress)
		} else if net.Type == "private" {
			private = append(private, net.IPAddress)
		}
	}

	return
}

func (d DigitalOcean) showInstance(instanceId string) {
	inst := d.getDroplet(instanceId)
	if inst != nil {
		fmt.Printf("%#v\n", inst)
	} else {
		fmt.Printf("Couldn't find %s\n", instanceId)
	}
}

func (d DigitalOcean) sshInstance(username, instanceId string) {
}

func (d DigitalOcean) removeInstances(instances []string) {
	for _, drop := range instances {
		id, err := strconv.ParseInt(drop, 10, 0)
		if err != nil {
			fmt.Println("Couldn't find", drop)
			continue
		}
		d.svc.Droplets.Delete(int(id))
	}
}

func (d DigitalOcean) getDroplet(dropletId string) *godo.Droplet {
	id, err := strconv.ParseInt(dropletId, 10, 0)
	if err != nil {
		return nil
	}

	result, _, err := d.svc.Droplets.Get(int(id))
	if err != nil {
		return nil
	}

	return result
}
<commit_msg>Implemented DigitalOcean.sshInstance()<commit_after>package main

import (
	"fmt"
	"github.com/digitalocean/godo"
	"golang.org/x/oauth2"
	"os"
	"os/exec"
	"os/user"
	"path"
	"strconv"
	"strings"
	"text/tabwriter"
)

type TokenSource struct {
	AccessToken string
}

func (t *TokenSource) Token() (*oauth2.Token, error) {
	token := &oauth2.Token{
		AccessToken: t.AccessToken,
	}
	return token, nil
}

type DigitalOcean struct {
	config Config
	svc    *godo.Client
}

func NewDigitalOcean(config Config) DigitalOcean {
	tokenSource := &TokenSource{
		AccessToken: config.AccessToken,
	}

	oauthClient := oauth2.NewClient(oauth2.NoContext, tokenSource)
	svc := godo.NewClient(oauthClient)
	return DigitalOcean{config, svc}
}

func (d DigitalOcean) listInstances() {
	// Get droplet list
	list := []godo.Droplet{}
	opt := &godo.ListOptions{}
	for {
		droplets, resp, err := d.svc.Droplets.List(opt)
		if err != nil {
			return
		}
		for _, d := range droplets {
			list = append(list, d)
		}
		if resp.Links == nil || resp.Links.IsLastPage() {
			break
		}
		page, err := resp.Links.CurrentPage()
		if err != nil {
			return
		}
		opt.Page = page + 1
	}

	w := new(tabwriter.Writer)
	w.Init(os.Stdout, 0, 8, 0, '\t', 0)
	fmt.Fprintln(w, "DropletID\tName\tPublic IP\tPrivateIP")
	fmt.Fprintln(w, "---\t---\t---\t---")

	total := 0
	for _, drop := range list {
		public, private := getNetworks(&drop)

		// Print instance details using tabwriter
		fmt.Fprintf(
			w,
			"%d\t%s\t%s\t%s\n",
			drop.ID,
			drop.Name,
			strings.Join(public, ","),
			strings.Join(private, ","),
		)
		total++
	}
	w.Flush()
	fmt.Printf("---\nFound %d instances.\n", total)
}

func getNetworks(drop *godo.Droplet) (public, private []string) {
	// Get lists for public and private networks
	nets := drop.Networks.V4
	for _, net := range nets {
		if net.Type == "public" {
			public = append(public, net.IPAddress)
		} else if net.Type == "private" {
			private = append(private, net.IPAddress)
		}
	}

	return
}

func (d DigitalOcean) showInstance(instanceId string) {
	inst := d.getDroplet(instanceId)
	if inst != nil {
		fmt.Printf("%#v\n", inst)
	} else {
		fmt.Printf("Couldn't find %s\n", instanceId)
	}
}

func (d DigitalOcean) sshInstance(username, instanceId string) {
	drop := d.getDroplet(instanceId)
	if drop != nil {
		ip, err := drop.PublicIPv4()
		if err != nil {
			fmt.Println("Can't find public IP for", instanceId)
			return
		}
		userHost := fmt.Sprintf("%s@%s", username, ip)

		me, err := user.Current()
		if err != nil {
			fmt.Println("Can't determine username details:", err)
		}
		keyFile := fmt.Sprintf(".ssh/%s", d.config.KeyName)
		keyFile = path.Join(me.HomeDir, keyFile)

		if _, err := os.Stat(keyFile); os.IsNotExist(err) {
			fmt.Println("Can't find private key:", keyFile)
			return
		}

		cmd := exec.Command("ssh", "-i", keyFile, userHost)
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout

		err = cmd.Run()
		if err != nil {
			fmt.Println(err)
		}
	} else {
		fmt.Printf("Couldn't find %s\n", instanceId)
	}
}

func (d DigitalOcean) removeInstances(instances []string) {
	for _, drop := range instances {
		id, err := strconv.ParseInt(drop, 10, 0)
		if err != nil {
			fmt.Println("Couldn't find", drop)
			continue
		}
		d.svc.Droplets.Delete(int(id))
	}
}

func (d DigitalOcean) getDroplet(dropletId string) *godo.Droplet {
	id, err := strconv.ParseInt(dropletId, 10, 0)
	if err != nil {
		return nil
	}

	result, _, err := d.svc.Droplets.Get(int(id))
	if err != nil {
		return nil
	}

	return result
}
<|endoftext|>


<commit_before>package main

import "testing"

func TestFormatYear(t *testing.T) {
	cases := []struct {
		in, want string
	}{
		{"2015", "Thu, 01 Jan 2015 00:00:00 +0000"},
		{"20151102", "Mon, 02 Nov 2015 00:00:00 +0000"},
		{"20152402", "Tue, 24 Feb 2015 00:00:00 +0000"},
		{"", ""},
		{"NOT_A_YEAR", ""},
	}
	for _, c := range cases {
		got := formatYear(c.in)
		if got != c.want {
			t.Errorf("formatYear(%q) == %q, want %q", c.in, got, c.want)
		}
	}
}

func TestFileUrl(t *testing.T) {
	cases := []struct {
		relativePath, baseUrl, want string
	}{
		{"", "", ""},
		{"c/d/e", "a/b/", "a/b/c/d/e"},
		{"c d", "a/b/", "a/b/c%20d"},
	}
	for _, c := range cases {
		got := fileUrl(c.relativePath, c.baseUrl)
		if got != c.want {
			t.Errorf("fileUrl(%q, %q) == %q, want %q", c.relativePath, c.baseUrl, got, c.want)
		}
	}
}
<commit_msg>addMeta: test add PubDate<commit_after>package main

import (
	"os"
	"testing"
)

func TestFormatYear(t *testing.T) {
	cases := []struct {
		in, want string
	}{
		{"2015", "Thu, 01 Jan 2015 00:00:00 +0000"},
		{"20151102", "Mon, 02 Nov 2015 00:00:00 +0000"},
		{"20152402", "Tue, 24 Feb 2015 00:00:00 +0000"},
		{"", ""},
		{"NOT_A_YEAR", ""},
	}
	for _, c := range cases {
		got := formatYear(c.in)
		if got != c.want {
			t.Errorf("formatYear(%q) == %q, want %q", c.in, got, c.want)
		}
	}
}

func TestFileUrl(t *testing.T) {
	cases := []struct {
		relativePath, baseUrl, want string
	}{
		{"", "", ""},
		{"c/d/e", "a/b/", "a/b/c/d/e"},
		{"c d", "a/b/", "a/b/c%20d"},
	}
	for _, c := range cases {
		got := fileUrl(c.relativePath, c.baseUrl)
		if got != c.want {
			t.Errorf("fileUrl(%q, %q) == %q, want %q", c.relativePath, c.baseUrl, got, c.want)
		}
	}
}
func TestAddMetaPubDate(t *testing.T) {
	pubDate := "Mon, 25 Nov 2013 00:00:00 +0000"
	in := Item{}
	path := "./vendor/src/github.com/mikkyang/id3-go/test.mp3"
	f, _ := os.Stat(path)

	addMeta(path, f, &in)

	if in.PubDate != pubDate {
		t.Errorf("addMeta(<test.mp3>, <FileInfo>, <Item>) Item.Title == %q want %q", in.PubDate, pubDate)
	}

}
<|endoftext|>


<commit_before>package doozer

import (
	"doozer/ack"
	"doozer/client"
	"doozer/gc"
	"doozer/lock"
	"doozer/member"
	"doozer/consensus"
	"doozer/server"
	"doozer/session"
	"doozer/store"
	"doozer/util"
	"doozer/web"
	"net"
	"os"
)

const (
	alpha           = 50
	pulseInterval   = 1e9
	timeout         = 60e9 // 60s
)

const slot = "/doozer/slot"

var slots = store.MustCompileGlob("/doozer/slot/*")

type proposer struct {
	seqns chan int64
	props chan *consensus.Prop
	st    *store.Store
}

func (p *proposer) Propose(v []byte) (e store.Event) {
	for e.Mut != string(v) {
		n := <-p.seqns
		w := p.st.Wait(n)
		println("propose", n, string(v))
		p.props <- &consensus.Prop{n, v}
		e = <-w
	}
	return
}

func Main(clusterName, attachAddr string, udpConn net.PacketConn, listener, webListener net.Listener) {
	logger := util.NewLogger("main")

	var err os.Error

	listenAddr := listener.Addr().String()

	var activateSeqn int64
	cal := make(chan bool, 1)
	useSelf := make(chan bool, 1)

	var cl *client.Client
	self := util.RandId()
	st := store.New()
	if attachAddr == "" { // we are the only node in a new cluster
		set(st, "/doozer/info/"+self+"/public-addr", listenAddr, store.Missing)
		set(st, "/doozer/info/"+self+"/hostname", os.Getenv("HOSTNAME"), store.Missing)
		set(st, "/doozer/info/"+self+"/version", Version, store.Missing)
		set(st, "/doozer/members/"+self, listenAddr, store.Missing)
		set(st, "/doozer/slot/"+"1", self, store.Missing)
		set(st, "/ping", "pong", store.Missing)

		cal <- true
		useSelf <- true

		cl = client.New("local", listenAddr) // TODO use real cluster name
	} else {
		cl = client.New("local", attachAddr) // TODO use real cluster name

		path := "/doozer/info/" + self + "/public-addr"
		_, err = cl.Set(path, store.Clobber, []byte(listenAddr))
		if err != nil {
			panic(err)
		}

		path = "/doozer/info/" + self + "/hostname"
		_, err = cl.Set(path, store.Clobber, []byte(os.Getenv("HOSTNAME")))
		if err != nil {
			panic(err)
		}

		path = "/doozer/info/" + self + "/version"
		_, err = cl.Set(path, store.Clobber, []byte(Version))
		if err != nil {
			panic(err)
		}

		joinSeqn, snap, err := cl.Join(self, listenAddr)
		if err != nil {
			panic(err)
		}

		done := make(chan int)
		st.Ops <- store.Op{1, snap}

		go advanceUntil(cl, done)

		go func() {
			st.Sync(joinSeqn + alpha)
			close(done)

			activateSeqn = activate(st, self, cl)
			cal <- true

			done = make(chan int)
			go advanceUntil(cl, done)
			st.Sync(activateSeqn + alpha)
			close(done)
			useSelf <- true
		}()

		// TODO sink needs a way to pick up missing values if there are any
		// gaps in its sequence
	}

	acker := ack.Ackify(udpConn)

	pr := &proposer{
		seqns: make(chan int64),
		props: make(chan *consensus.Prop),
		st:    st,
	}

	cmw := st.Watch(store.Any)
	in := make(chan consensus.Packet)
	out := make(chan consensus.Packet)

	consensus.NewManager(self, alpha, in, out, st.Ops, pr.seqns, pr.props, cmw)

	if attachAddr == "" {
		// Skip ahead alpha steps so that the registrar can provide a
		// meaningful cluster.
		n := <-st.Seqns
		for i := n + 1; i < n+alpha; i++ {
			st.Ops <- store.Op{i, store.Nop}
		}
	}

	live := make(chan string, 64)
	shun := make(chan string, 3) // sufficient for a cluster of 7

	go member.Clean(shun, st, pr)
	go member.Timeout(live, shun, listenAddr, timeout)

	go func() {
		<-cal
		go lock.Clean(st, pr)
		go session.Clean(st, pr)
		go gc.Pulse(self, st.Seqns, pr, pulseInterval)
		go gc.Clean(st)
	}()

	sv := &server.Server{listenAddr, st, pr, self, alpha}

	go sv.Serve(listener, useSelf)

	if webListener != nil {
		web.Store = st
		web.ClusterName = clusterName
		go web.Serve(webListener)
	}

	for {
		data, addr, err := acker.ReadFrom()
		if err == os.EINVAL {
			break
		}
		if err != nil {
			logger.Println(err)
			continue
		}

		// Update liveness time stamp for this addr
		live <- addr

		in <- consensus.Packet{addr, data}
	}
}

func activate(st *store.Store, self string, c *client.Client) (seqn int64) {
	logger := util.NewLogger("activate")
	w := store.NewWatch(st, slots)
	var err os.Error

	for _, base := range store.Getdir(st, slot) {
		p := slot + "/" + base
		v, cas := st.Get(p)
		if cas != store.Dir && v[0] == "" {
			seqn, err = c.Set(p, cas, []byte(self))
			if err != nil {
				logger.Println(err)
				continue
			}

			w.Stop()
			close(w.C)
			break
		}
	}

	for ev := range w.C {
		// TODO ev.IsEmpty()
		if ev.IsSet() && ev.Body == "" {
			seqn, err = c.Set(ev.Path, ev.Cas, []byte(self))
			if err != nil {
				logger.Println(err)
				continue
			}
			w.Stop()
			close(w.C)
		}
	}

	return seqn
}

func advanceUntil(cl *client.Client, done chan int) {
	for {
		select {
		case <-done:
			return
		default:
		}

		cl.Noop()
	}
}

func set(st *store.Store, path, body string, cas int64) {
	mut := store.MustEncodeSet(path, body, cas)
	st.Ops <- store.Op{1 + <-st.Seqns, mut}
}
<commit_msg>doozer: hook up outgoing packets to the network<commit_after>package doozer

import (
	"doozer/ack"
	"doozer/client"
	"doozer/gc"
	"doozer/lock"
	"doozer/member"
	"doozer/consensus"
	"doozer/server"
	"doozer/session"
	"doozer/store"
	"doozer/util"
	"doozer/web"
	"net"
	"os"
)

const (
	alpha           = 50
	pulseInterval   = 1e9
	timeout         = 60e9 // 60s
)

const slot = "/doozer/slot"

var slots = store.MustCompileGlob("/doozer/slot/*")

type proposer struct {
	seqns chan int64
	props chan *consensus.Prop
	st    *store.Store
}

func (p *proposer) Propose(v []byte) (e store.Event) {
	for e.Mut != string(v) {
		n := <-p.seqns
		w := p.st.Wait(n)
		println("propose", n, string(v))
		p.props <- &consensus.Prop{n, v}
		e = <-w
	}
	return
}

func Main(clusterName, attachAddr string, udpConn net.PacketConn, listener, webListener net.Listener) {
	logger := util.NewLogger("main")

	var err os.Error

	listenAddr := listener.Addr().String()

	var activateSeqn int64
	cal := make(chan bool, 1)
	useSelf := make(chan bool, 1)

	var cl *client.Client
	self := util.RandId()
	st := store.New()
	if attachAddr == "" { // we are the only node in a new cluster
		set(st, "/doozer/info/"+self+"/public-addr", listenAddr, store.Missing)
		set(st, "/doozer/info/"+self+"/hostname", os.Getenv("HOSTNAME"), store.Missing)
		set(st, "/doozer/info/"+self+"/version", Version, store.Missing)
		set(st, "/doozer/members/"+self, listenAddr, store.Missing)
		set(st, "/doozer/slot/"+"1", self, store.Missing)
		set(st, "/ping", "pong", store.Missing)

		cal <- true
		useSelf <- true

		cl = client.New("local", listenAddr) // TODO use real cluster name
	} else {
		cl = client.New("local", attachAddr) // TODO use real cluster name

		path := "/doozer/info/" + self + "/public-addr"
		_, err = cl.Set(path, store.Clobber, []byte(listenAddr))
		if err != nil {
			panic(err)
		}

		path = "/doozer/info/" + self + "/hostname"
		_, err = cl.Set(path, store.Clobber, []byte(os.Getenv("HOSTNAME")))
		if err != nil {
			panic(err)
		}

		path = "/doozer/info/" + self + "/version"
		_, err = cl.Set(path, store.Clobber, []byte(Version))
		if err != nil {
			panic(err)
		}

		joinSeqn, snap, err := cl.Join(self, listenAddr)
		if err != nil {
			panic(err)
		}

		done := make(chan int)
		st.Ops <- store.Op{1, snap}

		go advanceUntil(cl, done)

		go func() {
			st.Sync(joinSeqn + alpha)
			close(done)

			activateSeqn = activate(st, self, cl)
			cal <- true

			done = make(chan int)
			go advanceUntil(cl, done)
			st.Sync(activateSeqn + alpha)
			close(done)
			useSelf <- true
		}()

		// TODO sink needs a way to pick up missing values if there are any
		// gaps in its sequence
	}

	acker := ack.Ackify(udpConn)

	pr := &proposer{
		seqns: make(chan int64),
		props: make(chan *consensus.Prop),
		st:    st,
	}

	cmw := st.Watch(store.Any)
	in := make(chan consensus.Packet)
	out := make(chan consensus.Packet)

	consensus.NewManager(self, alpha, in, out, st.Ops, pr.seqns, pr.props, cmw)

	if attachAddr == "" {
		// Skip ahead alpha steps so that the registrar can provide a
		// meaningful cluster.
		n := <-st.Seqns
		for i := n + 1; i < n+alpha; i++ {
			st.Ops <- store.Op{i, store.Nop}
		}
	}

	live := make(chan string, 64)
	shun := make(chan string, 3) // sufficient for a cluster of 7

	go member.Clean(shun, st, pr)
	go member.Timeout(live, shun, listenAddr, timeout)

	go func() {
		<-cal
		go lock.Clean(st, pr)
		go session.Clean(st, pr)
		go gc.Pulse(self, st.Seqns, pr, pulseInterval)
		go gc.Clean(st)
	}()

	sv := &server.Server{listenAddr, st, pr, self, alpha}

	go sv.Serve(listener, useSelf)

	if webListener != nil {
		web.Store = st
		web.ClusterName = clusterName
		go web.Serve(webListener)
	}

	go func() {
		for p := range out {
			println("out packet", string(p.Data), p.Addr)
			acker.WriteTo(p.Data, p.Addr)
		}
	}()

	for {
		data, addr, err := acker.ReadFrom()
		if err == os.EINVAL {
			break
		}
		if err != nil {
			logger.Println(err)
			continue
		}

		// Update liveness time stamp for this addr
		live <- addr

		in <- consensus.Packet{addr, data}
	}
}

func activate(st *store.Store, self string, c *client.Client) (seqn int64) {
	logger := util.NewLogger("activate")
	w := store.NewWatch(st, slots)
	var err os.Error

	for _, base := range store.Getdir(st, slot) {
		p := slot + "/" + base
		v, cas := st.Get(p)
		if cas != store.Dir && v[0] == "" {
			seqn, err = c.Set(p, cas, []byte(self))
			if err != nil {
				logger.Println(err)
				continue
			}

			w.Stop()
			close(w.C)
			break
		}
	}

	for ev := range w.C {
		// TODO ev.IsEmpty()
		if ev.IsSet() && ev.Body == "" {
			seqn, err = c.Set(ev.Path, ev.Cas, []byte(self))
			if err != nil {
				logger.Println(err)
				continue
			}
			w.Stop()
			close(w.C)
		}
	}

	return seqn
}

func advanceUntil(cl *client.Client, done chan int) {
	for {
		select {
		case <-done:
			return
		default:
		}

		cl.Noop()
	}
}

func set(st *store.Store, path, body string, cas int64) {
	mut := store.MustEncodeSet(path, body, cas)
	st.Ops <- store.Op{1 + <-st.Seqns, mut}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// IP address manipulations
//
// IPv4 addresses are 4 bytes; IPv6 addresses are 16 bytes.
// An IPv4 address can be converted to an IPv6 address by
// adding a canonical prefix (10 zeros, 2 0xFFs).
// This library accepts either size of byte array but always
// returns 16-byte addresses.

package net

// IP address lengths (bytes).
const (
	IPv4len = 4
	IPv6len = 16
)

// An IP is a single IP address, an array of bytes.
// Functions in this package accept either 4-byte (IP v4)
// or 16-byte (IP v6) arrays as input.  Unless otherwise
// specified, functions in this package always return
// IP addresses in 16-byte form using the canonical
// embedding.
//
// Note that in this documentation, referring to an
// IP address as an IPv4 address or an IPv6 address
// is a semantic property of the address, not just the
// length of the byte array: a 16-byte array can still
// be an IPv4 address.
type IP []byte

// An IP mask is an IP address.
type IPMask []byte

// IPv4 returns the IP address (in 16-byte form) of the
// IPv4 address a.b.c.d.
func IPv4(a, b, c, d byte) IP {
	p := make(IP, IPv6len)
	for i := 0; i < 10; i++ {
		p[i] = 0
	}
	p[10] = 0xff
	p[11] = 0xff
	p[12] = a
	p[13] = b
	p[14] = c
	p[15] = d
	return p
}

// Well-known IPv4 addresses
var (
	IPv4bcast     = IPv4(255, 255, 255, 255) // broadcast
	IPv4allsys    = IPv4(224, 0, 0, 1)       // all systems
	IPv4allrouter = IPv4(224, 0, 0, 2)       // all routers
	IPv4zero      = IPv4(0, 0, 0, 0)         // all zeros
)

// Well-known IPv6 addresses
var (
	IPzero = make(IP, IPv6len) // all zeros
)

// Is p all zeros?
func isZeros(p IP) bool {
	for i := 0; i < len(p); i++ {
		if p[i] != 0 {
			return false
		}
	}
	return true
}

// To4 converts the IPv4 address ip to a 4-byte representation.
// If ip is not an IPv4 address, To4 returns nil.
func (ip IP) To4() IP {
	if len(ip) == IPv4len {
		return ip
	}
	if len(ip) == IPv6len &&
		isZeros(ip[0:10]) &&
		ip[10] == 0xff &&
		ip[11] == 0xff {
		return ip[12:16]
	}
	return nil
}

// To16 converts the IP address ip to a 16-byte representation.
// If ip is not an IP address (it is the wrong length), To16 returns nil.
func (ip IP) To16() IP {
	if len(ip) == IPv4len {
		return IPv4(ip[0], ip[1], ip[2], ip[3])
	}
	if len(ip) == IPv6len {
		return ip
	}
	return nil
}

// Default route masks for IPv4.
var (
	classAMask = IPMask(IPv4(0xff, 0, 0, 0))
	classBMask = IPMask(IPv4(0xff, 0xff, 0, 0))
	classCMask = IPMask(IPv4(0xff, 0xff, 0xff, 0))
)

// DefaultMask returns the default IP mask for the IP address ip.
// Only IPv4 addresses have default masks; DefaultMask returns
// nil if ip is not a valid IPv4 address.
func (ip IP) DefaultMask() IPMask {
	if ip = ip.To4(); ip == nil {
		return nil
	}
	switch true {
	case ip[0] < 0x80:
		return classAMask
	case ip[0] < 0xC0:
		return classBMask
	default:
		return classCMask
	}
	return nil // not reached
}

// Mask returns the result of masking the IP address ip with mask.
func (ip IP) Mask(mask IPMask) IP {
	n := len(ip)
	if n != len(mask) {
		return nil
	}
	out := make(IP, n)
	for i := 0; i < n; i++ {
		out[i] = ip[i] & mask[i]
	}
	return out
}

// Convert i to decimal string.
func itod(i uint) string {
	if i == 0 {
		return "0"
	}

	// Assemble decimal in reverse order.
	var b [32]byte
	bp := len(b)
	for ; i > 0; i /= 10 {
		bp--
		b[bp] = byte(i%10) + '0'
	}

	return string(b[bp:])
}

// Convert i to hexadecimal string.
func itox(i uint) string {
	if i == 0 {
		return "0"
	}

	// Assemble hexadecimal in reverse order.
	var b [32]byte
	bp := len(b)
	for ; i > 0; i /= 16 {
		bp--
		b[bp] = "0123456789abcdef"[byte(i%16)]
	}

	return string(b[bp:])
}

// String returns the string form of the IP address ip.
// If the address is an IPv4 address, the string representation
// is dotted decimal ("74.125.19.99").  Otherwise the representation
// is IPv6 ("2001:4860:0:2001::68").
func (ip IP) String() string {
	p := ip

	if len(ip) == 0 {
		return ""
	}

	// If IPv4, use dotted notation.
	if p4 := p.To4(); len(p4) == 4 {
		return itod(uint(p4[0])) + "." +
			itod(uint(p4[1])) + "." +
			itod(uint(p4[2])) + "." +
			itod(uint(p4[3]))
	}
	if len(p) != IPv6len {
		return "?"
	}

	// Find longest run of zeros.
	e0 := -1
	e1 := -1
	for i := 0; i < 16; i += 2 {
		j := i
		for j < 16 && p[j] == 0 && p[j+1] == 0 {
			j += 2
		}
		if j > i && j-i > e1-e0 {
			e0 = i
			e1 = j
		}
	}

	// Print with possible :: in place of run of zeros
	var s string
	for i := 0; i < 16; i += 2 {
		if i == e0 {
			s += "::"
			i = e1
			if i >= 16 {
				break
			}
		} else if i > 0 {
			s += ":"
		}
		s += itox((uint(p[i]) << 8) | uint(p[i+1]))
	}
	return s
}

// If mask is a sequence of 1 bits followed by 0 bits,
// return the number of 1 bits.
func simpleMaskLength(mask IPMask) int {
	var i int
	for i = 0; i < len(mask); i++ {
		if mask[i] != 0xFF {
			break
		}
	}
	n := 8 * i
	v := mask[i]
	for v&0x80 != 0 {
		n++
		v <<= 1
	}
	if v != 0 {
		return -1
	}
	for i++; i < len(mask); i++ {
		if mask[i] != 0 {
			return -1
		}
	}
	return n
}

// String returns the string representation of mask.
// If the mask is in the canonical form--ones followed by zeros--the
// string representation is just the decimal number of ones.
// If the mask is in a non-canonical form, it is formatted
// as an IP address.
func (mask IPMask) String() string {
	switch len(mask) {
	case 4:
		n := simpleMaskLength(mask)
		if n >= 0 {
			return itod(uint(n + (IPv6len-IPv4len)*8))
		}
	case 16:
		n := simpleMaskLength(mask)
		if n >= 0 {
			return itod(uint(n))
		}
	}
	return IP(mask).String()
}

// Parse IPv4 address (d.d.d.d).
func parseIPv4(s string) IP {
	var p [IPv4len]byte
	i := 0
	for j := 0; j < IPv4len; j++ {
		if i >= len(s) {
			// Missing octets.
			return nil
		}
		if j > 0 {
			if s[i] != '.' {
				return nil
			}
			i++
		}
		var (
			n  int
			ok bool
		)
		n, i, ok = dtoi(s, i)
		if !ok || n > 0xFF {
			return nil
		}
		p[j] = byte(n)
	}
	if i != len(s) {
		return nil
	}
	return IPv4(p[0], p[1], p[2], p[3])
}

// Parse IPv6 address.  Many forms.
// The basic form is a sequence of eight colon-separated
// 16-bit hex numbers separated by colons,
// as in 0123:4567:89ab:cdef:0123:4567:89ab:cdef.
// Two exceptions:
//	* A run of zeros can be replaced with "::".
//	* The last 32 bits can be in IPv4 form.
// Thus, ::ffff:1.2.3.4 is the IPv4 address 1.2.3.4.
func parseIPv6(s string) IP {
	p := make(IP, 16)
	ellipsis := -1 // position of ellipsis in p
	i := 0         // index in string s

	// Might have leading ellipsis
	if len(s) >= 2 && s[0] == ':' && s[1] == ':' {
		ellipsis = 0
		i = 2
		// Might be only ellipsis
		if i == len(s) {
			return p
		}
	}

	// Loop, parsing hex numbers followed by colon.
	j := 0
L: for j < IPv6len {
		// Hex number.
		n, i1, ok := xtoi(s, i)
		if !ok || n > 0xFFFF {
			return nil
		}

		// If followed by dot, might be in trailing IPv4.
		if i1 < len(s) && s[i1] == '.' {
			if ellipsis < 0 && j != IPv6len-IPv4len {
				// Not the right place.
				return nil
			}
			if j+IPv4len > IPv6len {
				// Not enough room.
				return nil
			}
			p4 := parseIPv4(s[i:])
			if p4 == nil {
				return nil
			}
			p[j] = p4[12]
			p[j+1] = p4[13]
			p[j+2] = p4[14]
			p[j+3] = p4[15]
			i = len(s)
			j += 4
			break
		}

		// Save this 16-bit chunk.
		p[j] = byte(n >> 8)
		p[j+1] = byte(n)
		j += 2

		// Stop at end of string.
		i = i1
		if i == len(s) {
			break
		}

		// Otherwise must be followed by colon and more.
		if s[i] != ':' && i+1 == len(s) {
			return nil
		}
		i++

		// Look for ellipsis.
		if s[i] == ':' {
			if ellipsis >= 0 { // already have one
				return nil
			}
			ellipsis = j
			if i++; i == len(s) { // can be at end
				break
			}
		}
	}

	// Must have used entire string.
	if i != len(s) {
		return nil
	}

	// If didn't parse enough, expand ellipsis.
	if j < IPv6len {
		if ellipsis < 0 {
			return nil
		}
		n := IPv6len - j
		for k := j - 1; k >= ellipsis; k-- {
			p[k+n] = p[k]
		}
		for k := ellipsis + n - 1; k >= ellipsis; k-- {
			p[k] = 0
		}
	}
	return p
}

// ParseIP parses s as an IP address, returning the result.
// The string s can be in dotted decimal ("74.125.19.99")
// or IPv6 ("2001:4860:0:2001::68") form.
// If s is not a valid textual representation of an IP address,
// ParseIP returns nil.
func ParseIP(s string) IP {
	p := parseIPv4(s)
	if p != nil {
		return p
	}
	return parseIPv6(s)
}
<commit_msg>net: fix IPMask.String not to crash on all-0xff mask<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// IP address manipulations
//
// IPv4 addresses are 4 bytes; IPv6 addresses are 16 bytes.
// An IPv4 address can be converted to an IPv6 address by
// adding a canonical prefix (10 zeros, 2 0xFFs).
// This library accepts either size of byte array but always
// returns 16-byte addresses.

package net

// IP address lengths (bytes).
const (
	IPv4len = 4
	IPv6len = 16
)

// An IP is a single IP address, an array of bytes.
// Functions in this package accept either 4-byte (IP v4)
// or 16-byte (IP v6) arrays as input.  Unless otherwise
// specified, functions in this package always return
// IP addresses in 16-byte form using the canonical
// embedding.
//
// Note that in this documentation, referring to an
// IP address as an IPv4 address or an IPv6 address
// is a semantic property of the address, not just the
// length of the byte array: a 16-byte array can still
// be an IPv4 address.
type IP []byte

// An IP mask is an IP address.
type IPMask []byte

// IPv4 returns the IP address (in 16-byte form) of the
// IPv4 address a.b.c.d.
func IPv4(a, b, c, d byte) IP {
	p := make(IP, IPv6len)
	for i := 0; i < 10; i++ {
		p[i] = 0
	}
	p[10] = 0xff
	p[11] = 0xff
	p[12] = a
	p[13] = b
	p[14] = c
	p[15] = d
	return p
}

// IPv4Mask returns the IP mask (in 16-byte form) of the
// IPv4 mask a.b.c.d.
func IPv4Mask(a, b, c, d byte) IPMask {
	p := make(IPMask, IPv6len)
	for i := 0; i < 12; i++ {
		p[i] = 0xff
	}
	p[12] = a
	p[13] = b
	p[14] = c
	p[15] = d
	return p
}

// Well-known IPv4 addresses
var (
	IPv4bcast     = IPv4(255, 255, 255, 255) // broadcast
	IPv4allsys    = IPv4(224, 0, 0, 1)       // all systems
	IPv4allrouter = IPv4(224, 0, 0, 2)       // all routers
	IPv4zero      = IPv4(0, 0, 0, 0)         // all zeros
)

// Well-known IPv6 addresses
var (
	IPzero = make(IP, IPv6len) // all zeros
)

// Is p all zeros?
func isZeros(p IP) bool {
	for i := 0; i < len(p); i++ {
		if p[i] != 0 {
			return false
		}
	}
	return true
}

// To4 converts the IPv4 address ip to a 4-byte representation.
// If ip is not an IPv4 address, To4 returns nil.
func (ip IP) To4() IP {
	if len(ip) == IPv4len {
		return ip
	}
	if len(ip) == IPv6len &&
		isZeros(ip[0:10]) &&
		ip[10] == 0xff &&
		ip[11] == 0xff {
		return ip[12:16]
	}
	return nil
}

// To16 converts the IP address ip to a 16-byte representation.
// If ip is not an IP address (it is the wrong length), To16 returns nil.
func (ip IP) To16() IP {
	if len(ip) == IPv4len {
		return IPv4(ip[0], ip[1], ip[2], ip[3])
	}
	if len(ip) == IPv6len {
		return ip
	}
	return nil
}

// Default route masks for IPv4.
var (
	classAMask = IPv4Mask(0xff, 0, 0, 0)
	classBMask = IPv4Mask(0xff, 0xff, 0, 0)
	classCMask = IPv4Mask(0xff, 0xff, 0xff, 0)
)

// DefaultMask returns the default IP mask for the IP address ip.
// Only IPv4 addresses have default masks; DefaultMask returns
// nil if ip is not a valid IPv4 address.
func (ip IP) DefaultMask() IPMask {
	if ip = ip.To4(); ip == nil {
		return nil
	}
	switch true {
	case ip[0] < 0x80:
		return classAMask
	case ip[0] < 0xC0:
		return classBMask
	default:
		return classCMask
	}
	return nil // not reached
}

// Mask returns the result of masking the IP address ip with mask.
func (ip IP) Mask(mask IPMask) IP {
	n := len(ip)
	if n != len(mask) {
		return nil
	}
	out := make(IP, n)
	for i := 0; i < n; i++ {
		out[i] = ip[i] & mask[i]
	}
	return out
}

// Convert i to decimal string.
func itod(i uint) string {
	if i == 0 {
		return "0"
	}

	// Assemble decimal in reverse order.
	var b [32]byte
	bp := len(b)
	for ; i > 0; i /= 10 {
		bp--
		b[bp] = byte(i%10) + '0'
	}

	return string(b[bp:])
}

// Convert i to hexadecimal string.
func itox(i uint) string {
	if i == 0 {
		return "0"
	}

	// Assemble hexadecimal in reverse order.
	var b [32]byte
	bp := len(b)
	for ; i > 0; i /= 16 {
		bp--
		b[bp] = "0123456789abcdef"[byte(i%16)]
	}

	return string(b[bp:])
}

// String returns the string form of the IP address ip.
// If the address is an IPv4 address, the string representation
// is dotted decimal ("74.125.19.99").  Otherwise the representation
// is IPv6 ("2001:4860:0:2001::68").
func (ip IP) String() string {
	p := ip

	if len(ip) == 0 {
		return ""
	}

	// If IPv4, use dotted notation.
	if p4 := p.To4(); len(p4) == 4 {
		return itod(uint(p4[0])) + "." +
			itod(uint(p4[1])) + "." +
			itod(uint(p4[2])) + "." +
			itod(uint(p4[3]))
	}
	if len(p) != IPv6len {
		return "?"
	}

	// Find longest run of zeros.
	e0 := -1
	e1 := -1
	for i := 0; i < 16; i += 2 {
		j := i
		for j < 16 && p[j] == 0 && p[j+1] == 0 {
			j += 2
		}
		if j > i && j-i > e1-e0 {
			e0 = i
			e1 = j
		}
	}

	// Print with possible :: in place of run of zeros
	var s string
	for i := 0; i < 16; i += 2 {
		if i == e0 {
			s += "::"
			i = e1
			if i >= 16 {
				break
			}
		} else if i > 0 {
			s += ":"
		}
		s += itox((uint(p[i]) << 8) | uint(p[i+1]))
	}
	return s
}

// If mask is a sequence of 1 bits followed by 0 bits,
// return the number of 1 bits.
func simpleMaskLength(mask IPMask) int {
	var n int
	for i, v := range mask {
		if v == 0xff {
			n += 8
			continue
		}
		// found non-ff byte
		// count 1 bits
		for v&0x80 != 0 {
			n++
			v <<= 1
		}
		// rest must be 0 bits
		if v != 0 {
			return -1
		}
		for i++; i < len(mask); i++ {
			if mask[i] != 0 {
				return -1
			}
		}
		break
	}
	return n
}

// String returns the string representation of mask.
// If the mask is in the canonical form--ones followed by zeros--the
// string representation is just the decimal number of ones.
// If the mask is in a non-canonical form, it is formatted
// as an IP address.
func (mask IPMask) String() string {
	switch len(mask) {
	case 4:
		n := simpleMaskLength(mask)
		if n >= 0 {
			return itod(uint(n + (IPv6len-IPv4len)*8))
		}
	case 16:
		n := simpleMaskLength(mask)
		if n >= 12*8 {
			return itod(uint(n - 12*8))
		}
	}
	return IP(mask).String()
}

// Parse IPv4 address (d.d.d.d).
func parseIPv4(s string) IP {
	var p [IPv4len]byte
	i := 0
	for j := 0; j < IPv4len; j++ {
		if i >= len(s) {
			// Missing octets.
			return nil
		}
		if j > 0 {
			if s[i] != '.' {
				return nil
			}
			i++
		}
		var (
			n  int
			ok bool
		)
		n, i, ok = dtoi(s, i)
		if !ok || n > 0xFF {
			return nil
		}
		p[j] = byte(n)
	}
	if i != len(s) {
		return nil
	}
	return IPv4(p[0], p[1], p[2], p[3])
}

// Parse IPv6 address.  Many forms.
// The basic form is a sequence of eight colon-separated
// 16-bit hex numbers separated by colons,
// as in 0123:4567:89ab:cdef:0123:4567:89ab:cdef.
// Two exceptions:
//	* A run of zeros can be replaced with "::".
//	* The last 32 bits can be in IPv4 form.
// Thus, ::ffff:1.2.3.4 is the IPv4 address 1.2.3.4.
func parseIPv6(s string) IP {
	p := make(IP, 16)
	ellipsis := -1 // position of ellipsis in p
	i := 0         // index in string s

	// Might have leading ellipsis
	if len(s) >= 2 && s[0] == ':' && s[1] == ':' {
		ellipsis = 0
		i = 2
		// Might be only ellipsis
		if i == len(s) {
			return p
		}
	}

	// Loop, parsing hex numbers followed by colon.
	j := 0
L: for j < IPv6len {
		// Hex number.
		n, i1, ok := xtoi(s, i)
		if !ok || n > 0xFFFF {
			return nil
		}

		// If followed by dot, might be in trailing IPv4.
		if i1 < len(s) && s[i1] == '.' {
			if ellipsis < 0 && j != IPv6len-IPv4len {
				// Not the right place.
				return nil
			}
			if j+IPv4len > IPv6len {
				// Not enough room.
				return nil
			}
			p4 := parseIPv4(s[i:])
			if p4 == nil {
				return nil
			}
			p[j] = p4[12]
			p[j+1] = p4[13]
			p[j+2] = p4[14]
			p[j+3] = p4[15]
			i = len(s)
			j += 4
			break
		}

		// Save this 16-bit chunk.
		p[j] = byte(n >> 8)
		p[j+1] = byte(n)
		j += 2

		// Stop at end of string.
		i = i1
		if i == len(s) {
			break
		}

		// Otherwise must be followed by colon and more.
		if s[i] != ':' && i+1 == len(s) {
			return nil
		}
		i++

		// Look for ellipsis.
		if s[i] == ':' {
			if ellipsis >= 0 { // already have one
				return nil
			}
			ellipsis = j
			if i++; i == len(s) { // can be at end
				break
			}
		}
	}

	// Must have used entire string.
	if i != len(s) {
		return nil
	}

	// If didn't parse enough, expand ellipsis.
	if j < IPv6len {
		if ellipsis < 0 {
			return nil
		}
		n := IPv6len - j
		for k := j - 1; k >= ellipsis; k-- {
			p[k+n] = p[k]
		}
		for k := ellipsis + n - 1; k >= ellipsis; k-- {
			p[k] = 0
		}
	}
	return p
}

// ParseIP parses s as an IP address, returning the result.
// The string s can be in dotted decimal ("74.125.19.99")
// or IPv6 ("2001:4860:0:2001::68") form.
// If s is not a valid textual representation of an IP address,
// ParseIP returns nil.
func ParseIP(s string) IP {
	p := parseIPv4(s)
	if p != nil {
		return p
	}
	return parseIPv6(s)
}
<|endoftext|>