<commit_before>package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strconv"
	"strings"
	"text/tabwriter"
	"time"

	"gopkg.in/alecthomas/kingpin.v2"

	"github.com/rackerlabs/carina/version"
	"github.com/rackerlabs/libcarina"
)

// Application is, our, well, application
type Application struct {
	*Context
	*kingpin.Application
}

// Command is a command needing a ClusterClient
type Command struct {
	*Context
	*kingpin.CmdClause
}

// Context context for the  App
type Context struct {
	ClusterClient *libcarina.ClusterClient
	TabWriter     *tabwriter.Writer
	Username      string
	APIKey        string
	Endpoint      string
}

// ClusterCommand is a Command with a ClusterName set
type ClusterCommand struct {
	*Command
	ClusterName string
}

// CredentialsCommand keeps context about the download command
type CredentialsCommand struct {
	*ClusterCommand
	Path string
}

// WaitClusterCommand is simply a ClusterCommand that waits for cluster creation
type WaitClusterCommand struct {
	*ClusterCommand
	// Whether to wait until the cluster is created (or errored)
	Wait bool
}

// CreateCommand keeps context about the create command
type CreateCommand struct {
	*WaitClusterCommand

	// Options passed along to Carina's API
	Nodes     int
	AutoScale bool

	// TODO: See if setting flavor or image makes sense, even if the API takes it
	// Flavor    string
	// Image     string
}

// GrowCommand keeps context about the number of nodes to scale by
type GrowCommand struct {
	*ClusterCommand
	Nodes int
}

// New creates a new Application
func New() *Application {

	app := kingpin.New("carina", "command line interface to launch and work with Docker Swarm clusters")
	app.Version(VersionString())

	cap := new(Application)
	ctx := new(Context)

	cap.Application = app

	cap.Context = ctx

	cap.Flag("username", "Rackspace username - can also set env var RACKSPACE_USERNAME").OverrideDefaultFromEnvar("RACKSPACE_USERNAME").StringVar(&ctx.Username)
	cap.Flag("api-key", "Rackspace API Key - can also set env var RACKSPACE_APIKEY").OverrideDefaultFromEnvar("RACKSPACE_APIKEY").PlaceHolder("RACKSPACE_APIKEY").StringVar(&ctx.APIKey)
	cap.Flag("endpoint", "Carina API endpoint").Default(libcarina.BetaEndpoint).StringVar(&ctx.Endpoint)

	writer := new(tabwriter.Writer)
	writer.Init(os.Stdout, 0, 8, 1, '\t', 0)

	// Make sure the tabwriter gets flushed at the end
	app.Terminate(func(code int) {
		// Squish any errors from flush, since we're terminating the app anyway
		_ = ctx.TabWriter.Flush()
		os.Exit(code)
	})

	ctx.TabWriter = writer

	createCommand := new(CreateCommand)
	createCommand.WaitClusterCommand = cap.NewWaitClusterCommand(ctx, "create", "Create a swarm cluster")
	createCommand.Flag("nodes", "number of nodes for the initial cluster").Default("1").IntVar(&createCommand.Nodes)
	createCommand.Flag("autoscale", "whether autoscale is on or off").BoolVar(&createCommand.AutoScale)
	createCommand.Action(createCommand.Create)

	getCommand := cap.NewClusterCommand(ctx, "get", "Get information about a swarm cluster")
	getCommand.Action(getCommand.Get)

	listCommand := cap.NewCommand(ctx, "list", "List swarm clusters")
	listCommand.Action(listCommand.List)

	credentialsCommand := new(CredentialsCommand)
	credentialsCommand.ClusterCommand = cap.NewClusterCommand(ctx, "credentials", "Download credentials for a swarm cluster")
	credentialsCommand.Flag("path", "path to write credentials out to").PlaceHolder("<cluster-name>").StringVar(&credentialsCommand.Path)
	credentialsCommand.Action(credentialsCommand.Download)

	growCommand := new(GrowCommand)
	growCommand.ClusterCommand = cap.NewClusterCommand(ctx, "grow", "Grow a cluster by the requested number of nodes")
	growCommand.Flag("nodes", "number of nodes to increase the cluster by").Required().IntVar(&growCommand.Nodes)
	growCommand.Action(growCommand.Grow)

	rebuildCommand := cap.NewWaitClusterCommand(ctx, "rebuild", "Rebuild a swarm cluster")
	rebuildCommand.Action(rebuildCommand.Rebuild)

	deleteCommand := cap.NewClusterCommand(ctx, "delete", "Delete a swarm cluster")
	deleteCommand.Action(deleteCommand.Delete)

	return cap
}

// VersionString returns the current version and commit of this binary (if set)
func VersionString() string {
	s := ""
	s += fmt.Sprintf("Version: %s\n", version.Version)
	s += fmt.Sprintf("Commit:  %s", version.Commit)
	return s
}

// NewCommand creates a command wrapped with carina.Context
func (app *Application) NewCommand(ctx *Context, name, help string) *Command {
	carina := new(Command)
	carina.Context = ctx
	carina.CmdClause = app.Command(name, help)
	carina.PreAction(carina.Auth)
	return carina
}

// NewClusterCommand is a command that uses a cluster name
func (app *Application) NewClusterCommand(ctx *Context, name, help string) *ClusterCommand {
	cc := new(ClusterCommand)
	cc.Command = app.NewCommand(ctx, name, help)
	cc.Arg("cluster-name", "name of the cluster").Required().StringVar(&cc.ClusterName)
	return cc
}

// NewWaitClusterCommand is a command that uses a cluster name and allows the
// user to wait for a cluster state
func (app *Application) NewWaitClusterCommand(ctx *Context, name, help string) *WaitClusterCommand {
	wcc := new(WaitClusterCommand)
	wcc.ClusterCommand = app.NewClusterCommand(ctx, name, help)
	wcc.Flag("wait", "wait for swarm cluster to come online (or error)").BoolVar(&wcc.Wait)
	return wcc
}

// Auth does the authentication
func (carina *Command) Auth(pc *kingpin.ParseContext) (err error) {
	carina.ClusterClient, err = libcarina.NewClusterClient(carina.Endpoint, carina.Username, carina.APIKey)
	return err
}

// List the current swarm clusters
func (carina *Command) List(pc *kingpin.ParseContext) (err error) {
	clusterList, err := carina.ClusterClient.List()
	if err != nil {
		return err
	}

	err = writeClusterHeader(carina.TabWriter)
	if err != nil {
		return err
	}

	for _, cluster := range clusterList {
		err = writeCluster(carina.TabWriter, &cluster)
		if err != nil {
			return err
		}
	}
	err = carina.TabWriter.Flush()
	return err
}

type clusterOp func(clusterName string) (*libcarina.Cluster, error)

// Does an func against a cluster then returns the new cluster representation
func (carina *ClusterCommand) clusterApply(op clusterOp) (err error) {
	cluster, err := op(carina.ClusterName)
	if err != nil {
		return err
	}

	err = writeCluster(carina.TabWriter, cluster)
	if err != nil {
		return err
	}
	return carina.TabWriter.Flush()
}

// Get an individual cluster
func (carina *ClusterCommand) Get(pc *kingpin.ParseContext) (err error) {
	return carina.clusterApply(carina.ClusterClient.Get)
}

// Delete a cluster
func (carina *ClusterCommand) Delete(pc *kingpin.ParseContext) (err error) {
	return carina.clusterApply(carina.ClusterClient.Delete)
}

// Grow increases the size of the given cluster
func (carina *GrowCommand) Grow(pc *kingpin.ParseContext) (err error) {
	return carina.clusterApply(func(clusterName string) (*libcarina.Cluster, error) {
		return carina.ClusterClient.Grow(clusterName, carina.Nodes)
	})
}

// Rebuild nukes your cluster and builds it over again
func (carina *WaitClusterCommand) Rebuild(pc *kingpin.ParseContext) (err error) {
	return carina.clusterApplyWait(carina.ClusterClient.Rebuild)
}

const startupFudgeFactor = 40 * time.Second
const waitBetween = 10 * time.Second

// Cluster status when new
const StatusNew = "new"

// Cluster status when building
const StatusBuilding = "building"

// Cluster status when rebuilding swarm
const StatusRebuildingSwarm = "rebuilding-swarm"

// Does an func against a cluster then returns the new cluster representation
func (carina *WaitClusterCommand) clusterApplyWait(op clusterOp) (err error) {
	cluster, err := op(carina.ClusterName)

	if carina.Wait {
		time.Sleep(startupFudgeFactor)
		// Transitions past point of "new" or "building" are assumed to be states we
		// can stop on.
		for cluster.Status == StatusNew || cluster.Status == StatusBuilding || cluster.Status == StatusRebuildingSwarm {
			time.Sleep(waitBetween)
			cluster, err = carina.ClusterClient.Get(carina.ClusterName)
			if err != nil {
				break
			}
		}
	}

	if err != nil {
		return err
	}

	err = writeCluster(carina.TabWriter, cluster)
	if err != nil {
		return err
	}
	return carina.TabWriter.Flush()
}

// Create a cluster
func (carina *CreateCommand) Create(pc *kingpin.ParseContext) (err error) {
	return carina.clusterApplyWait(func(clusterName string) (*libcarina.Cluster, error) {
		if carina.Nodes < 1 {
			return nil, errors.New("nodes must be >= 1")
		}
		nodes := libcarina.Number(carina.Nodes)

		c := libcarina.Cluster{
			ClusterName: carina.ClusterName,
			Nodes:       nodes,
			AutoScale:   carina.AutoScale,
		}
		return carina.ClusterClient.Create(c)
	})
}

// Download credentials for a cluster
func (carina *CredentialsCommand) Download(pc *kingpin.ParseContext) (err error) {
	credentials, err := carina.ClusterClient.GetCredentials(carina.ClusterName)
	if err != nil {
		return err
	}

	if carina.Path == "" {
		carina.Path = carina.ClusterName
	}

	p := path.Clean(carina.Path)

	if p != "." {
		err = os.MkdirAll(p, 0777)
	}

	if err != nil {
		return err
	}

	err = writeCredentials(carina.TabWriter, credentials, p)
	if err != nil {
		return err
	}

	fmt.Fprintln(os.Stdout, sourceHelpString(p))

	err = carina.TabWriter.Flush()
	return err
}

func writeCredentials(w *tabwriter.Writer, creds *libcarina.Credentials, pth string) (err error) {
	// TODO: Prompt when file already exists?
	for fname, b := range creds.Files {
		p := path.Join(pth, fname)
		err = ioutil.WriteFile(p, b, 0600)
		if err != nil {
			return err
		}
	}

	return nil
}

func writeCluster(w *tabwriter.Writer, cluster *libcarina.Cluster) (err error) {
	s := strings.Join([]string{
		cluster.ClusterName,
		cluster.Flavor,
		strconv.FormatInt(cluster.Nodes.Int64(), 10),
		strconv.FormatBool(cluster.AutoScale),
		cluster.Status,
	}, "\t")
	_, err = w.Write([]byte(s + "\n"))
	return
}

func writeClusterHeader(w *tabwriter.Writer) (err error) {
	headerFields := []string{
		"ClusterName",
		"Flavor",
		"Nodes",
		"AutoScale",
		"Status",
	}
	s := strings.Join(headerFields, "\t")

	_, err = w.Write([]byte(s + "\n"))
	return err
}

func main() {
	app := New()
	kingpin.MustParse(app.Parse(os.Args[1:]))
}
<commit_msg>Switch to CARINA_USERNAME and CARINA_APIKEY.<commit_after>package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strconv"
	"strings"
	"text/tabwriter"
	"time"

	"gopkg.in/alecthomas/kingpin.v2"

	"github.com/rackerlabs/carina/version"
	"github.com/rackerlabs/libcarina"
)

// Application is, our, well, application
type Application struct {
	*Context
	*kingpin.Application
}

// Command is a command needing a ClusterClient
type Command struct {
	*Context
	*kingpin.CmdClause
}

// Context context for the  App
type Context struct {
	ClusterClient *libcarina.ClusterClient
	TabWriter     *tabwriter.Writer
	Username      string
	APIKey        string
	Endpoint      string
}

// ClusterCommand is a Command with a ClusterName set
type ClusterCommand struct {
	*Command
	ClusterName string
}

// CredentialsCommand keeps context about the download command
type CredentialsCommand struct {
	*ClusterCommand
	Path string
}

// WaitClusterCommand is simply a ClusterCommand that waits for cluster creation
type WaitClusterCommand struct {
	*ClusterCommand
	// Whether to wait until the cluster is created (or errored)
	Wait bool
}

// CreateCommand keeps context about the create command
type CreateCommand struct {
	*WaitClusterCommand

	// Options passed along to Carina's API
	Nodes     int
	AutoScale bool

	// TODO: See if setting flavor or image makes sense, even if the API takes it
	// Flavor    string
	// Image     string
}

// GrowCommand keeps context about the number of nodes to scale by
type GrowCommand struct {
	*ClusterCommand
	Nodes int
}

// UserNameEnvKey is the name of the env var accepted for the username
const UserNameEnvVar = "CARINA_USERNAME"

// APIKeyEnvVar is the name of the env var accepted for the API key
const APIKeyEnvVar = "CARINA_APIKEY"

// New creates a new Application
func New() *Application {

	app := kingpin.New("carina", "command line interface to launch and work with Docker Swarm clusters")
	app.Version(VersionString())

	cap := new(Application)
	ctx := new(Context)

	cap.Application = app

	cap.Context = ctx

	cap.Flag("username", "Carina username - can also set env var "+UserNameEnvVar).OverrideDefaultFromEnvar(UserNameEnvVar).StringVar(&ctx.Username)
	cap.Flag("api-key", "Carina API Key - can also set env var "+APIKeyEnvVar).OverrideDefaultFromEnvar(APIKeyEnvVar).PlaceHolder(APIKeyEnvVar).StringVar(&ctx.APIKey)
	cap.Flag("endpoint", "Carina API endpoint").Default(libcarina.BetaEndpoint).StringVar(&ctx.Endpoint)

	writer := new(tabwriter.Writer)
	writer.Init(os.Stdout, 0, 8, 1, '\t', 0)

	// Make sure the tabwriter gets flushed at the end
	app.Terminate(func(code int) {
		// Squish any errors from flush, since we're terminating the app anyway
		_ = ctx.TabWriter.Flush()
		os.Exit(code)
	})

	ctx.TabWriter = writer

	createCommand := new(CreateCommand)
	createCommand.WaitClusterCommand = cap.NewWaitClusterCommand(ctx, "create", "Create a swarm cluster")
	createCommand.Flag("nodes", "number of nodes for the initial cluster").Default("1").IntVar(&createCommand.Nodes)
	createCommand.Flag("autoscale", "whether autoscale is on or off").BoolVar(&createCommand.AutoScale)
	createCommand.Action(createCommand.Create)

	getCommand := cap.NewClusterCommand(ctx, "get", "Get information about a swarm cluster")
	getCommand.Action(getCommand.Get)

	listCommand := cap.NewCommand(ctx, "list", "List swarm clusters")
	listCommand.Action(listCommand.List)

	credentialsCommand := new(CredentialsCommand)
	credentialsCommand.ClusterCommand = cap.NewClusterCommand(ctx, "credentials", "Download credentials for a swarm cluster")
	credentialsCommand.Flag("path", "path to write credentials out to").PlaceHolder("<cluster-name>").StringVar(&credentialsCommand.Path)
	credentialsCommand.Action(credentialsCommand.Download)

	growCommand := new(GrowCommand)
	growCommand.ClusterCommand = cap.NewClusterCommand(ctx, "grow", "Grow a cluster by the requested number of nodes")
	growCommand.Flag("nodes", "number of nodes to increase the cluster by").Required().IntVar(&growCommand.Nodes)
	growCommand.Action(growCommand.Grow)

	rebuildCommand := cap.NewWaitClusterCommand(ctx, "rebuild", "Rebuild a swarm cluster")
	rebuildCommand.Action(rebuildCommand.Rebuild)

	deleteCommand := cap.NewClusterCommand(ctx, "delete", "Delete a swarm cluster")
	deleteCommand.Action(deleteCommand.Delete)

	return cap
}

// VersionString returns the current version and commit of this binary (if set)
func VersionString() string {
	s := ""
	s += fmt.Sprintf("Version: %s\n", version.Version)
	s += fmt.Sprintf("Commit:  %s", version.Commit)
	return s
}

// NewCommand creates a command wrapped with carina.Context
func (app *Application) NewCommand(ctx *Context, name, help string) *Command {
	carina := new(Command)
	carina.Context = ctx
	carina.CmdClause = app.Command(name, help)
	carina.PreAction(carina.Auth)
	return carina
}

// NewClusterCommand is a command that uses a cluster name
func (app *Application) NewClusterCommand(ctx *Context, name, help string) *ClusterCommand {
	cc := new(ClusterCommand)
	cc.Command = app.NewCommand(ctx, name, help)
	cc.Arg("cluster-name", "name of the cluster").Required().StringVar(&cc.ClusterName)
	return cc
}

// NewWaitClusterCommand is a command that uses a cluster name and allows the
// user to wait for a cluster state
func (app *Application) NewWaitClusterCommand(ctx *Context, name, help string) *WaitClusterCommand {
	wcc := new(WaitClusterCommand)
	wcc.ClusterCommand = app.NewClusterCommand(ctx, name, help)
	wcc.Flag("wait", "wait for swarm cluster to come online (or error)").BoolVar(&wcc.Wait)
	return wcc
}

// Auth does the authentication
func (carina *Command) Auth(pc *kingpin.ParseContext) (err error) {
	carina.ClusterClient, err = libcarina.NewClusterClient(carina.Endpoint, carina.Username, carina.APIKey)
	return err
}

// List the current swarm clusters
func (carina *Command) List(pc *kingpin.ParseContext) (err error) {
	clusterList, err := carina.ClusterClient.List()
	if err != nil {
		return err
	}

	err = writeClusterHeader(carina.TabWriter)
	if err != nil {
		return err
	}

	for _, cluster := range clusterList {
		err = writeCluster(carina.TabWriter, &cluster)
		if err != nil {
			return err
		}
	}
	err = carina.TabWriter.Flush()
	return err
}

type clusterOp func(clusterName string) (*libcarina.Cluster, error)

// Does an func against a cluster then returns the new cluster representation
func (carina *ClusterCommand) clusterApply(op clusterOp) (err error) {
	cluster, err := op(carina.ClusterName)
	if err != nil {
		return err
	}

	err = writeCluster(carina.TabWriter, cluster)
	if err != nil {
		return err
	}
	return carina.TabWriter.Flush()
}

// Get an individual cluster
func (carina *ClusterCommand) Get(pc *kingpin.ParseContext) (err error) {
	return carina.clusterApply(carina.ClusterClient.Get)
}

// Delete a cluster
func (carina *ClusterCommand) Delete(pc *kingpin.ParseContext) (err error) {
	return carina.clusterApply(carina.ClusterClient.Delete)
}

// Grow increases the size of the given cluster
func (carina *GrowCommand) Grow(pc *kingpin.ParseContext) (err error) {
	return carina.clusterApply(func(clusterName string) (*libcarina.Cluster, error) {
		return carina.ClusterClient.Grow(clusterName, carina.Nodes)
	})
}

// Rebuild nukes your cluster and builds it over again
func (carina *WaitClusterCommand) Rebuild(pc *kingpin.ParseContext) (err error) {
	return carina.clusterApplyWait(carina.ClusterClient.Rebuild)
}

const startupFudgeFactor = 40 * time.Second
const waitBetween = 10 * time.Second

// Cluster status when new
const StatusNew = "new"

// Cluster status when building
const StatusBuilding = "building"

// Cluster status when rebuilding swarm
const StatusRebuildingSwarm = "rebuilding-swarm"

// Does an func against a cluster then returns the new cluster representation
func (carina *WaitClusterCommand) clusterApplyWait(op clusterOp) (err error) {
	cluster, err := op(carina.ClusterName)

	if carina.Wait {
		time.Sleep(startupFudgeFactor)
		// Transitions past point of "new" or "building" are assumed to be states we
		// can stop on.
		for cluster.Status == StatusNew || cluster.Status == StatusBuilding || cluster.Status == StatusRebuildingSwarm {
			time.Sleep(waitBetween)
			cluster, err = carina.ClusterClient.Get(carina.ClusterName)
			if err != nil {
				break
			}
		}
	}

	if err != nil {
		return err
	}

	err = writeCluster(carina.TabWriter, cluster)
	if err != nil {
		return err
	}
	return carina.TabWriter.Flush()
}

// Create a cluster
func (carina *CreateCommand) Create(pc *kingpin.ParseContext) (err error) {
	return carina.clusterApplyWait(func(clusterName string) (*libcarina.Cluster, error) {
		if carina.Nodes < 1 {
			return nil, errors.New("nodes must be >= 1")
		}
		nodes := libcarina.Number(carina.Nodes)

		c := libcarina.Cluster{
			ClusterName: carina.ClusterName,
			Nodes:       nodes,
			AutoScale:   carina.AutoScale,
		}
		return carina.ClusterClient.Create(c)
	})
}

// Download credentials for a cluster
func (carina *CredentialsCommand) Download(pc *kingpin.ParseContext) (err error) {
	credentials, err := carina.ClusterClient.GetCredentials(carina.ClusterName)
	if err != nil {
		return err
	}

	if carina.Path == "" {
		carina.Path = carina.ClusterName
	}

	p := path.Clean(carina.Path)

	if p != "." {
		err = os.MkdirAll(p, 0777)
	}

	if err != nil {
		return err
	}

	err = writeCredentials(carina.TabWriter, credentials, p)
	if err != nil {
		return err
	}

	fmt.Fprintln(os.Stdout, sourceHelpString(p))

	err = carina.TabWriter.Flush()
	return err
}

func writeCredentials(w *tabwriter.Writer, creds *libcarina.Credentials, pth string) (err error) {
	// TODO: Prompt when file already exists?
	for fname, b := range creds.Files {
		p := path.Join(pth, fname)
		err = ioutil.WriteFile(p, b, 0600)
		if err != nil {
			return err
		}
	}

	return nil
}

func writeCluster(w *tabwriter.Writer, cluster *libcarina.Cluster) (err error) {
	s := strings.Join([]string{
		cluster.ClusterName,
		cluster.Flavor,
		strconv.FormatInt(cluster.Nodes.Int64(), 10),
		strconv.FormatBool(cluster.AutoScale),
		cluster.Status,
	}, "\t")
	_, err = w.Write([]byte(s + "\n"))
	return
}

func writeClusterHeader(w *tabwriter.Writer) (err error) {
	headerFields := []string{
		"ClusterName",
		"Flavor",
		"Nodes",
		"AutoScale",
		"Status",
	}
	s := strings.Join(headerFields, "\t")

	_, err = w.Write([]byte(s + "\n"))
	return err
}

func main() {
	app := New()
	kingpin.MustParse(app.Parse(os.Args[1:]))
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"fmt"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/CiscoCloud/mantl-api/api"
	"github.com/CiscoCloud/mantl-api/install"
	"github.com/CiscoCloud/mantl-api/marathon"
	"github.com/CiscoCloud/mantl-api/mesos"
	"github.com/CiscoCloud/mantl-api/utils/http"
	log "github.com/Sirupsen/logrus"
	consul "github.com/hashicorp/consul/api"
	"github.com/hashicorp/go-cleanhttp"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
)

const Name = "mantl-api"
const Version = "0.2.2"

var wg sync.WaitGroup

func main() {
	rootCmd := &cobra.Command{
		Use:   "mantl-api",
		Short: "runs the mantl-api",
		Run: func(cmd *cobra.Command, args []string) {
			start()
		},
		PersistentPreRun: func(cmd *cobra.Command, args []string) {
			readConfigFile()
			setupLogging()
		},
	}

	rootCmd.PersistentFlags().String("log-level", "info", "one of debug, info, warn, error, or fatal")
	rootCmd.PersistentFlags().String("log-format", "text", "specify output (text or json)")
	rootCmd.PersistentFlags().String("consul", "http://localhost:8500", "Consul Api address")
	rootCmd.PersistentFlags().String("consul-acl-token", "", "Consul ACL token for accessing mantl-install/apps path")
	rootCmd.PersistentFlags().Bool("consul-no-verify-ssl", false, "Consul SSL verification")
	rootCmd.PersistentFlags().String("marathon", "", "Marathon Api address")
	rootCmd.PersistentFlags().String("marathon-user", "", "Marathon Api user")
	rootCmd.PersistentFlags().String("marathon-password", "", "Marathon Api password")
	rootCmd.PersistentFlags().Bool("marathon-no-verify-ssl", false, "Marathon SSL verification")
	rootCmd.PersistentFlags().String("mesos", "", "Mesos Api address")
	rootCmd.PersistentFlags().String("mesos-principal", "", "Mesos principal for framework authentication")
	rootCmd.PersistentFlags().String("mesos-secret", "", "Deprecated. Use mesos-secret-path instead")
	rootCmd.PersistentFlags().String("mesos-secret-path", "/etc/sysconfig/mantl-api", "Path to a file on host sytem that contains the mesos secret for framework authentication")
	rootCmd.PersistentFlags().Bool("mesos-no-verify-ssl", false, "Mesos SSL verification")
	rootCmd.PersistentFlags().String("listen", ":4001", "mantl-api listen address")
	rootCmd.PersistentFlags().String("zookeeper", "", "Comma-delimited list of zookeeper servers")
	rootCmd.PersistentFlags().Bool("force-sync", false, "Force a synchronization of all sources")
	rootCmd.PersistentFlags().String("config-file", "", "The path to a configuration file")
	rootCmd.PersistentFlags().Int("consul-refresh-interval", 10, "The number of seconds after which to check consul for package requests")

	for _, flags := range []*pflag.FlagSet{rootCmd.PersistentFlags()} {
		err := viper.BindPFlags(flags)
		if err != nil {
			log.WithField("error", err).Fatal("could not bind flags")
		}
	}

	viper.SetEnvPrefix("mantl_api")
	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
	viper.AutomaticEnv()

	syncCommand := &cobra.Command{
		Use:   "sync",
		Short: "Synchronize universe repositories",
		Long:  "Forces a synchronization of all configured sources",
		Run: func(cmd *cobra.Command, args []string) {
			syncRepo(nil, true)
		},
	}
	rootCmd.AddCommand(syncCommand)

	versionCommand := &cobra.Command{
		Use:   "version",
		Short: fmt.Sprintf("Print the version number of %s", Name),
		Run: func(cmd *cobra.Command, args []string) {
			fmt.Printf("%s v%s\n", Name, Version)
		},
	}
	rootCmd.AddCommand(versionCommand)

	rootCmd.Execute()
}

func start() {
	log.Infof("Starting %s v%s", Name, Version)
	client := consulClient()

	marathonUrl := viper.GetString("marathon")
	if marathonUrl == "" {
		marathonHosts := NewDiscovery(client, "marathon", "").discoveredHosts
		if len(marathonHosts) > 0 {
			marathonUrl = fmt.Sprintf("http://%s", marathonHosts[0])
		} else {
			marathonUrl = "http://localhost:8080"
		}
	}
	marathonClient, err := marathon.NewMarathon(
		marathonUrl,
		viper.GetString("marathon-user"),
		viper.GetString("marathon-password"),
		viper.GetBool("marathon-no-verify-ssl"),
	)
	if err != nil {
		log.Fatalf("Could not create marathon client: %v", err)
	}

	mesosUrl := viper.GetString("mesos")
	if mesosUrl == "" {
		mesosHosts := NewDiscovery(client, "mesos", "leader").discoveredHosts
		if len(mesosHosts) > 0 {
			mesosUrl = fmt.Sprintf("http://%s", mesosHosts[0])
		} else {
			mesosUrl = "http://locahost:5050"
		}
	}
	mesosClient, err := mesos.NewMesos(
		mesosUrl,
		viper.GetString("mesos-principal"),
		viper.GetString("mesos-secret-path"),
		viper.GetBool("mesos-no-verify-ssl"),
	)
	if err != nil {
		log.Fatalf("Could not create mesos client: %v", err)
	}

	var zkHosts []string
	zkUrls := viper.GetString("zookeeper")
	if zkUrls == "" {
		zkHosts = NewDiscovery(client, "zookeeper", "").discoveredHosts
		if len(zkHosts) == 0 {
			zkHosts = []string{"locahost:2181"}
		}
	} else {
		zkHosts = strings.Split(zkUrls, ",")
	}

	inst, err := install.NewInstall(client, marathonClient, mesosClient, zkHosts)
	if err != nil {
		log.Fatalf("Could not create install client: %v", err)
	}

	// sync sources to consul
	syncRepo(inst, viper.GetBool("force-sync"))

	wg.Add(1)
	go inst.Watch(time.Duration(viper.GetInt("consul-refresh-interval")))
	go api.NewApi(Name, viper.GetString("listen"), inst, mesosClient, wg).Start()
	wg.Wait()
}

func consulClient() *consul.Client {
	consulConfig := consul.DefaultConfig()
	scheme, address, _, err := http.ParseUrl(viper.GetString("consul"))
	if err != nil {
		log.Fatalf("Could not create consul client: %v", err)
	}
	consulConfig.Scheme = scheme
	consulConfig.Address = address

	log.Debugf("Using Consul at %s over %s", consulConfig.Address, consulConfig.Scheme)

	if viper.GetBool("consul-no-verify-ssl") {
		transport := cleanhttp.DefaultTransport()
		transport.TLSClientConfig = &tls.Config{
			InsecureSkipVerify: true,
		}
		consulConfig.HttpClient.Transport = transport
	}

	if aclToken := viper.GetString("consul-acl-token"); aclToken != "" {
		consulConfig.Token = aclToken
	}

	client, err := consul.NewClient(consulConfig)
	if err != nil {
		log.Fatalf("Could not create consul client: %v", err)
	}

	// abort if we cannot connect to consul
	err = testConsul(client)
	if err != nil {
		log.Fatalf("Could not connect to consul: %v", err)
	}

	return client
}

func testConsul(client *consul.Client) error {
	kv := client.KV()
	_, _, err := kv.Get("mantl-install", nil)
	return err
}

func syncRepo(inst *install.Install, force bool) {
	var err error
	if inst == nil {
		client := consulClient()
		inst, err = install.NewInstall(client, nil, nil, nil)
		if err != nil {
			log.Fatalf("Could not create install client: %v", err)
		}
	}

	defaultSources := []*install.Source{
		&install.Source{
			Name:       "mantl",
			Path:       "https://github.com/CiscoCloud/mantl-universe.git",
			SourceType: install.Git,
			Branch:     "version-0.7",
			Index:      0,
		},
	}

	sources := []*install.Source{}

	configuredSources := viper.GetStringMap("sources")

	if len(configuredSources) > 0 {
		for name, val := range configuredSources {
			source := &install.Source{Name: name, SourceType: install.FileSystem}
			sourceConfig := val.(map[string]interface{})

			if path, ok := sourceConfig["path"].(string); ok {
				source.Path = path
			}

			if index, ok := sourceConfig["index"].(int64); ok {
				source.Index = int(index)
			}

			if sourceType, ok := sourceConfig["type"].(string); ok {
				if strings.EqualFold(sourceType, "git") {
					source.SourceType = install.Git
				}
			}

			if branch, ok := sourceConfig["branch"].(string); ok {
				source.Branch = branch
			}

			if source.IsValid() {
				sources = append(sources, source)
			} else {
				log.Warnf("Invalid source configuration for %s", name)
			}
		}
	}

	if len(sources) == 0 {
		sources = defaultSources
	}

	if err := inst.SyncSources(sources, force); err != nil {
		log.Fatal(err)
	}
}

func readConfigFile() {
	// read configuration file if specified
	configFile := viper.GetString("config-file")
	if configFile != "" {
		configFile = os.ExpandEnv(configFile)
		if _, err := os.Stat(configFile); err == nil {
			viper.SetConfigFile(configFile)
			err = viper.ReadInConfig()
			if err != nil {
				log.Warnf("Could not read configuration file: %v", err)
			}
		} else {
			log.Warnf("Could not find configuration file: %s", configFile)
		}
	}
}

func setupLogging() {
	switch viper.GetString("log-level") {
	case "debug":
		log.SetLevel(log.DebugLevel)
	case "info":
		log.SetLevel(log.InfoLevel)
	case "warn":
		log.SetLevel(log.WarnLevel)
	case "error":
		log.SetLevel(log.ErrorLevel)
	case "fatal":
		log.SetLevel(log.FatalLevel)
	default:
		log.WithField("log-level", viper.GetString("log-level")).Warning("invalid log level. defaulting to info.")
		log.SetLevel(log.InfoLevel)
	}

	switch viper.GetString("log-format") {
	case "text":
		log.SetFormatter(new(log.TextFormatter))
	case "json":
		log.SetFormatter(new(log.JSONFormatter))
	default:
		log.WithField("log-format", viper.GetString("log-format")).Warning("invalid log format. defaulting to text.")
		log.SetFormatter(new(log.TextFormatter))
	}
}
<commit_msg>first attempt at vault integration<commit_after>package main

import (
	"crypto/tls"
	"fmt"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/CiscoCloud/mantl-api/api"
	"github.com/CiscoCloud/mantl-api/install"
	"github.com/CiscoCloud/mantl-api/marathon"
	"github.com/CiscoCloud/mantl-api/mesos"
	"github.com/CiscoCloud/mantl-api/utils/http"
	log "github.com/Sirupsen/logrus"
	consul "github.com/hashicorp/consul/api"
	"github.com/hashicorp/go-cleanhttp"
	vault "github.com/hashicorp/vault/api"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
)

const Name = "mantl-api"
const Version = "0.2.2"

var wg sync.WaitGroup

func main() {
	rootCmd := &cobra.Command{
		Use:   "mantl-api",
		Short: "runs the mantl-api",
		Run: func(cmd *cobra.Command, args []string) {
			start()
		},
		PersistentPreRun: func(cmd *cobra.Command, args []string) {
			readConfigFile()
			setupLogging()
		},
	}

	rootCmd.PersistentFlags().String("log-level", "info", "one of debug, info, warn, error, or fatal")
	rootCmd.PersistentFlags().String("log-format", "text", "specify output (text or json)")
	rootCmd.PersistentFlags().String("consul", "http://localhost:8500", "Consul Api address")
	rootCmd.PersistentFlags().String("consul-acl-token", "", "Consul ACL token for accessing mantl-install/apps path")
	rootCmd.PersistentFlags().Bool("consul-no-verify-ssl", false, "Consul SSL verification")
	rootCmd.PersistentFlags().String("marathon", "", "Marathon Api address")
	rootCmd.PersistentFlags().String("marathon-user", "", "Marathon Api user")
	rootCmd.PersistentFlags().String("marathon-password", "", "Marathon Api password")
	rootCmd.PersistentFlags().Bool("marathon-no-verify-ssl", false, "Marathon SSL verification")
	rootCmd.PersistentFlags().String("mesos", "", "Mesos Api address")
	rootCmd.PersistentFlags().String("mesos-principal", "", "Mesos principal for framework authentication")
	rootCmd.PersistentFlags().String("mesos-secret", "", "Deprecated. Use mesos-secret-path instead")
	rootCmd.PersistentFlags().String("mesos-secret-path", "/etc/sysconfig/mantl-api", "Path to a file on host sytem that contains the mesos secret for framework authentication")
	rootCmd.PersistentFlags().Bool("mesos-no-verify-ssl", false, "Mesos SSL verification")
	rootCmd.PersistentFlags().String("listen", ":4001", "mantl-api listen address")
	rootCmd.PersistentFlags().String("zookeeper", "", "Comma-delimited list of zookeeper servers")
	rootCmd.PersistentFlags().Bool("force-sync", false, "Force a synchronization of all sources")
	rootCmd.PersistentFlags().String("config-file", "", "The path to a configuration file")
	rootCmd.PersistentFlags().Int("consul-refresh-interval", 10, "The number of seconds after which to check consul for package requests")
	rootCmd.PersistentFlags().String("vault-cubbyhole-token", "", "token for retrieving secrets from vault")

	for _, flags := range []*pflag.FlagSet{rootCmd.PersistentFlags()} {
		err := viper.BindPFlags(flags)
		if err != nil {
			log.WithField("error", err).Fatal("could not bind flags")
		}
	}

	viper.SetEnvPrefix("mantl_api")
	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
	viper.AutomaticEnv()

	syncCommand := &cobra.Command{
		Use:   "sync",
		Short: "Synchronize universe repositories",
		Long:  "Forces a synchronization of all configured sources",
		Run: func(cmd *cobra.Command, args []string) {
			syncRepo(nil, true)
		},
	}
	rootCmd.AddCommand(syncCommand)

	versionCommand := &cobra.Command{
		Use:   "version",
		Short: fmt.Sprintf("Print the version number of %s", Name),
		Run: func(cmd *cobra.Command, args []string) {
			fmt.Printf("%s v%s\n", Name, Version)
		},
	}
	rootCmd.AddCommand(versionCommand)

	rootCmd.Execute()
}

func start() {
	log.Infof("Starting %s v%s", Name, Version)
	client := consulClient()

	if wrapped := viper.GetString("vault-cubbyhole-token"); wrapped != "" {
		config := vault.DefaultConfig()
		err := config.ReadEnvironment()
		if err != nil {
			log.WithError(err).Fatal("Error reading environment for Vault configuration")
		}

		client, err := vault.NewClient(config)
		if err != nil {
			log.WithError(err).Fatal("Error initializing Vault client")
		}

		token, err := client.Logical().Unwrap(wrapped)
		if err != nil {
			log.WithError(err).Fatal("Error unwrapping token")
		} else if token.WrapInfo != nil {
			log.Fatal("Secret appears to be doubly wrapped")
		} else if token.Auth == nil {
			log.Fatal("Secret contained no auth data")
		}

		client.SetToken(token.Auth.ClientToken)

		secret, err := client.Logical().Read("secret/mantl-api")
		if err != nil {
			log.WithError(err).Fatal("Error reading secret/mantl-api")
		}

		for _, secretName := range []string{
			"mesos-principal", "mesos-secret",
			"marathon-user", "marathon-password",
		} {
			secretValue, ok := secret.Data[secretName].(string)
			if ok {
				viper.Set(secretName, secretValue)
			} else {
				log.Warnf("secret/mantl-api didn't contain %s", secretName)
			}
		}
	}

	marathonUrl := viper.GetString("marathon")
	if marathonUrl == "" {
		marathonHosts := NewDiscovery(client, "marathon", "").discoveredHosts
		if len(marathonHosts) > 0 {
			marathonUrl = fmt.Sprintf("http://%s", marathonHosts[0])
		} else {
			marathonUrl = "http://localhost:8080"
		}
	}
	marathonClient, err := marathon.NewMarathon(
		marathonUrl,
		viper.GetString("marathon-user"),
		viper.GetString("marathon-password"),
		viper.GetBool("marathon-no-verify-ssl"),
	)
	if err != nil {
		log.Fatalf("Could not create marathon client: %v", err)
	}

	mesosUrl := viper.GetString("mesos")
	if mesosUrl == "" {
		mesosHosts := NewDiscovery(client, "mesos", "leader").discoveredHosts
		if len(mesosHosts) > 0 {
			mesosUrl = fmt.Sprintf("http://%s", mesosHosts[0])
		} else {
			mesosUrl = "http://locahost:5050"
		}
	}
	mesosClient, err := mesos.NewMesos(
		mesosUrl,
		viper.GetString("mesos-principal"),
		viper.GetString("mesos-secret-path"),
		viper.GetBool("mesos-no-verify-ssl"),
	)
	if err != nil {
		log.Fatalf("Could not create mesos client: %v", err)
	}

	var zkHosts []string
	zkUrls := viper.GetString("zookeeper")
	if zkUrls == "" {
		zkHosts = NewDiscovery(client, "zookeeper", "").discoveredHosts
		if len(zkHosts) == 0 {
			zkHosts = []string{"locahost:2181"}
		}
	} else {
		zkHosts = strings.Split(zkUrls, ",")
	}

	inst, err := install.NewInstall(client, marathonClient, mesosClient, zkHosts)
	if err != nil {
		log.Fatalf("Could not create install client: %v", err)
	}

	// sync sources to consul
	syncRepo(inst, viper.GetBool("force-sync"))

	wg.Add(1)
	go inst.Watch(time.Duration(viper.GetInt("consul-refresh-interval")))
	go api.NewApi(Name, viper.GetString("listen"), inst, mesosClient, wg).Start()
	wg.Wait()
}

func consulClient() *consul.Client {
	consulConfig := consul.DefaultConfig()
	scheme, address, _, err := http.ParseUrl(viper.GetString("consul"))
	if err != nil {
		log.Fatalf("Could not create consul client: %v", err)
	}
	consulConfig.Scheme = scheme
	consulConfig.Address = address

	log.Debugf("Using Consul at %s over %s", consulConfig.Address, consulConfig.Scheme)

	if viper.GetBool("consul-no-verify-ssl") {
		transport := cleanhttp.DefaultTransport()
		transport.TLSClientConfig = &tls.Config{
			InsecureSkipVerify: true,
		}
		consulConfig.HttpClient.Transport = transport
	}

	if aclToken := viper.GetString("consul-acl-token"); aclToken != "" {
		consulConfig.Token = aclToken
	}

	client, err := consul.NewClient(consulConfig)
	if err != nil {
		log.Fatalf("Could not create consul client: %v", err)
	}

	// abort if we cannot connect to consul
	err = testConsul(client)
	if err != nil {
		log.Fatalf("Could not connect to consul: %v", err)
	}

	return client
}

func testConsul(client *consul.Client) error {
	kv := client.KV()
	_, _, err := kv.Get("mantl-install", nil)
	return err
}

func syncRepo(inst *install.Install, force bool) {
	var err error
	if inst == nil {
		client := consulClient()
		inst, err = install.NewInstall(client, nil, nil, nil)
		if err != nil {
			log.Fatalf("Could not create install client: %v", err)
		}
	}

	defaultSources := []*install.Source{
		&install.Source{
			Name:       "mantl",
			Path:       "https://github.com/CiscoCloud/mantl-universe.git",
			SourceType: install.Git,
			Branch:     "version-0.7",
			Index:      0,
		},
	}

	sources := []*install.Source{}

	configuredSources := viper.GetStringMap("sources")

	if len(configuredSources) > 0 {
		for name, val := range configuredSources {
			source := &install.Source{Name: name, SourceType: install.FileSystem}
			sourceConfig := val.(map[string]interface{})

			if path, ok := sourceConfig["path"].(string); ok {
				source.Path = path
			}

			if index, ok := sourceConfig["index"].(int64); ok {
				source.Index = int(index)
			}

			if sourceType, ok := sourceConfig["type"].(string); ok {
				if strings.EqualFold(sourceType, "git") {
					source.SourceType = install.Git
				}
			}

			if branch, ok := sourceConfig["branch"].(string); ok {
				source.Branch = branch
			}

			if source.IsValid() {
				sources = append(sources, source)
			} else {
				log.Warnf("Invalid source configuration for %s", name)
			}
		}
	}

	if len(sources) == 0 {
		sources = defaultSources
	}

	if err := inst.SyncSources(sources, force); err != nil {
		log.Fatal(err)
	}
}

func readConfigFile() {
	// read configuration file if specified
	configFile := viper.GetString("config-file")
	if configFile != "" {
		configFile = os.ExpandEnv(configFile)
		if _, err := os.Stat(configFile); err == nil {
			viper.SetConfigFile(configFile)
			err = viper.ReadInConfig()
			if err != nil {
				log.Warnf("Could not read configuration file: %v", err)
			}
		} else {
			log.Warnf("Could not find configuration file: %s", configFile)
		}
	}
}

func setupLogging() {
	switch viper.GetString("log-level") {
	case "debug":
		log.SetLevel(log.DebugLevel)
	case "info":
		log.SetLevel(log.InfoLevel)
	case "warn":
		log.SetLevel(log.WarnLevel)
	case "error":
		log.SetLevel(log.ErrorLevel)
	case "fatal":
		log.SetLevel(log.FatalLevel)
	default:
		log.WithField("log-level", viper.GetString("log-level")).Warning("invalid log level. defaulting to info.")
		log.SetLevel(log.InfoLevel)
	}

	switch viper.GetString("log-format") {
	case "text":
		log.SetFormatter(new(log.TextFormatter))
	case "json":
		log.SetFormatter(new(log.JSONFormatter))
	default:
		log.WithField("log-format", viper.GetString("log-format")).Warning("invalid log format. defaulting to text.")
		log.SetFormatter(new(log.TextFormatter))
	}
}
<|endoftext|>


<commit_before>package main

import (
    "fmt"
    "log"
    "strings"
    "flag"
    "net/http"
    "html/template"
    "encoding/json"
    "sync"
    "io/ioutil"
    "github.com/acmacalister/skittles"
    "microb/conf"
    "microb/utils"
    "microb/db/rethinkdb"
)

type Page struct {
    Url string
    Title string
    Content  string
}

var Config = conf.GetConf()
var Nochangefeed = flag.Bool("nf", false, "Do not use changefeed")
var CommandFlag = flag.String("c", "", "Fire command")

func getPage(url string) *Page {
	hasSlash := strings.HasSuffix(url, "/")
	index_url := url
	found := false
	var data map[string]interface{}
	page := Page{Url:"404", Title:"Page not found", Content:"Page not found"}
	if (hasSlash == false) {
		index_url = url+"/"
	}
	// remove url mask
	index_url = strings.Replace(index_url,"/x","",-1)
	// hit db
	if (Config["db_type"] == "rethinkdb") {
		data, found = rethinkdb.GetFromDb(index_url)
		if (found == false) {
			return &page
		}
	} else {
		log.Fatal("No db configured")
	}
	fields := data["fields"].(map[string]interface{})
	content := fields["content"].(string)
	title := fields["title"].(string)
	page = Page{Url: url, Title: title, Content: content}
	return &page
}

var view = template.Must(template.New("view.html").ParseFiles("templates/view.html", "templates/routes.js"))

func renderTemplate(response http.ResponseWriter, page *Page) {
    err := view.Execute(response, page)
    if err != nil {
        http.Error(response, err.Error(), http.StatusInternalServerError)
    }
}

func viewHandler(response http.ResponseWriter, request *http.Request) {
    url := request.URL.Path
    fmt.Printf("%s Page %s\n", utils.GetTime(), url)
    page := &Page{Url: url, Title: "Page not found", Content: "Page not found"}
    renderTemplate(response, page)
}

func apiHandler(response http.ResponseWriter, request *http.Request) {
    url := request.URL.Path
    page := getPage(url)
    status := ""
    if (page.Url == "404") {
		status = "Error 404"
	}
    fmt.Printf("%s API %s %s\n", utils.GetTime(), url, skittles.BoldRed(status))
	json_bytes, _ := json.Marshal(page)
	fmt.Fprintf(response, "%s\n", json_bytes)
}

func reparseStatic() {
	utils.PrintEvent("command", "Reparsing static files")
	view = template.Must(template.New("view.html").ParseFiles("templates/view.html", "templates/routes.js"))
}

func updateRoutes(c chan bool) {
	var routestab []string
	if (Config["db_type"] == "rethinkdb") {
		routestab = rethinkdb.GetRoutes()
	} else {
		utils.PrintEvent("error", "No database configured to get routes")
	}
	var routestr string
	var route string
	for i := range(routestab) {
		route = routestab[i]
		routestr = routestr+fmt.Sprintf("page('%s', function(ctx, next) { loadPage('/x%s') } );", route, route)
	}
	utils.PrintEvent("command", "Rebuilding client side routes")
    str := []byte(routestr)
    err := ioutil.WriteFile("routes.js", str, 0644)
    if err != nil {
        panic(err)
    }
	c <- true
}

func main() {
	flag.Parse()
	// commands
	if (*CommandFlag != "") {
		valid_commands := []string{"update_routes", "reparse_templates"}
		is_valid := false
		for _, com := range(valid_commands) {
			if (com == *CommandFlag) {
				is_valid = true
				break
			}
		}
		if (is_valid == true) {
			msg := "Sending command "+skittles.BoldWhite(*CommandFlag)+" to the server"
			utils.PrintEvent("event", msg)
			var wg sync.WaitGroup
			wg.Add(1)
			go rethinkdb.SaveCommand(*CommandFlag, &wg)
			wg.Wait()

		} else {
			msg := "Unknown command: "+*CommandFlag
			utils.PrintEvent("error", msg)
		}
		return
	}
	if (*Nochangefeed == false) {
		utils.PrintEvent("info", "listening to changefeed")
	}
	// changefeed
	if (Config["db_type"] == "rethinkdb") {
		c := make(chan *rethinkdb.DataChanges)
		c2 := make(chan bool)
		comchan := make(chan *rethinkdb.Command)
		go rethinkdb.PageChangesListener(c)
		go rethinkdb.CommandsListener(comchan)
		// channel listeners
		go func() {
        	for {
                changes := <- c
				if (changes.Type == "update") {
					utils.PrintEvent("event", changes.Msg)
					go updateRoutes(c2)
				} else if (changes.Type == "delete") {
					utils.PrintEvent("event", changes.Msg)
					go updateRoutes(c2)
				} else if (changes.Type == "insert") {
					utils.PrintEvent("event", changes.Msg)
					go updateRoutes(c2)
				}
        	}
        }()
        go func() {
        	for {
                com := <- comchan
				if (com.Name != "") {
					msg := "Command "+skittles.BoldWhite(com.Name)+" received"
					utils.PrintEvent("event", msg)
					if (com.Name == "reparse_templates") {
						go reparseStatic()
					}
				}
        	}
        }()
        go func() {
			for {
				routes_done := <- c2
				if (routes_done == true) {
					//fmt.Println("[OK] Routes updated")
					go reparseStatic()
				}
			}
		}()
	}
	// http server
	http_port := Config["http_port"].(string)
	msg := "Server started on "+http_port+" ..."
	utils.PrintEvent("nil", msg)
	http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("./static/"))))
	http.Handle("/media/", http.StripPrefix("/media/", http.FileServer(http.Dir("./media/"))))
	http.HandleFunc("/x/", apiHandler)
    http.HandleFunc("/", viewHandler)
    log.Fatal(http.ListenAndServe(http_port, nil))
}
<commit_msg>Correction in routes generation<commit_after>package main

import (
    "fmt"
    "log"
    "strings"
    "flag"
    "net/http"
    "html/template"
    "encoding/json"
    "sync"
    "io/ioutil"
    "github.com/acmacalister/skittles"
    "microb/conf"
    "microb/utils"
    "microb/db/rethinkdb"
)

type Page struct {
    Url string
    Title string
    Content  string
}

var Config = conf.GetConf()
var Nochangefeed = flag.Bool("nf", false, "Do not use changefeed")
var CommandFlag = flag.String("c", "", "Fire command")

func getPage(url string) *Page {
	hasSlash := strings.HasSuffix(url, "/")
	index_url := url
	found := false
	var data map[string]interface{}
	page := Page{Url:"404", Title:"Page not found", Content:"Page not found"}
	if (hasSlash == false) {
		index_url = url+"/"
	}
	// remove url mask
	index_url = strings.Replace(index_url,"/x","",-1)
	// hit db
	if (Config["db_type"] == "rethinkdb") {
		data, found = rethinkdb.GetFromDb(index_url)
		if (found == false) {
			return &page
		}
	} else {
		log.Fatal("No db configured")
	}
	fields := data["fields"].(map[string]interface{})
	content := fields["content"].(string)
	title := fields["title"].(string)
	page = Page{Url: url, Title: title, Content: content}
	return &page
}

var view = template.Must(template.New("view.html").ParseFiles("templates/view.html", "templates/routes.js"))

func renderTemplate(response http.ResponseWriter, page *Page) {
    err := view.Execute(response, page)
    if err != nil {
        http.Error(response, err.Error(), http.StatusInternalServerError)
    }
}

func viewHandler(response http.ResponseWriter, request *http.Request) {
    url := request.URL.Path
    fmt.Printf("%s Page %s\n", utils.GetTime(), url)
    page := &Page{Url: url, Title: "Page not found", Content: "Page not found"}
    renderTemplate(response, page)
}

func apiHandler(response http.ResponseWriter, request *http.Request) {
    url := request.URL.Path
    page := getPage(url)
    status := ""
    if (page.Url == "404") {
		status = "Error 404"
	}
    fmt.Printf("%s API %s %s\n", utils.GetTime(), url, skittles.BoldRed(status))
	json_bytes, _ := json.Marshal(page)
	fmt.Fprintf(response, "%s\n", json_bytes)
}

func reparseStatic() {
	utils.PrintEvent("command", "Reparsing static files")
	view = template.Must(template.New("view.html").ParseFiles("templates/view.html", "templates/routes.js"))
}

func updateRoutes(c chan bool) {
	var routestab []string
	if (Config["db_type"] == "rethinkdb") {
		routestab = rethinkdb.GetRoutes()
	} else {
		utils.PrintEvent("error", "No database configured to get routes")
	}
	var routestr string
	var route string
	for i := range(routestab) {
		route = routestab[i]
		routestr = routestr+fmt.Sprintf("page('%s', function(ctx, next) { loadPage('/x%s') } );", route, route)
	}
	utils.PrintEvent("command", "Rebuilding client side routes")
    str := []byte(routestr)
    err := ioutil.WriteFile("templates/routes.js", str, 0644)
    if err != nil {
        panic(err)
    }
	c <- true
}

func main() {
	flag.Parse()
	// commands
	if (*CommandFlag != "") {
		valid_commands := []string{"update_routes", "reparse_templates"}
		is_valid := false
		for _, com := range(valid_commands) {
			if (com == *CommandFlag) {
				is_valid = true
				break
			}
		}
		if (is_valid == true) {
			msg := "Sending command "+skittles.BoldWhite(*CommandFlag)+" to the server"
			utils.PrintEvent("event", msg)
			var wg sync.WaitGroup
			wg.Add(1)
			go rethinkdb.SaveCommand(*CommandFlag, &wg)
			wg.Wait()

		} else {
			msg := "Unknown command: "+*CommandFlag
			utils.PrintEvent("error", msg)
		}
		return
	}
	if (*Nochangefeed == false) {
		utils.PrintEvent("info", "listening to changefeed")
	}
	// changefeed
	if (Config["db_type"] == "rethinkdb") {
		c := make(chan *rethinkdb.DataChanges)
		c2 := make(chan bool)
		comchan := make(chan *rethinkdb.Command)
		go rethinkdb.PageChangesListener(c)
		go rethinkdb.CommandsListener(comchan)
		// channel listeners
		go func() {
        	for {
                changes := <- c
				if (changes.Type == "update") {
					utils.PrintEvent("event", changes.Msg)
					go updateRoutes(c2)
				} else if (changes.Type == "delete") {
					utils.PrintEvent("event", changes.Msg)
					go updateRoutes(c2)
				} else if (changes.Type == "insert") {
					utils.PrintEvent("event", changes.Msg)
					go updateRoutes(c2)
				}
        	}
        }()
        go func() {
        	for {
                com := <- comchan
				if (com.Name != "") {
					msg := "Command "+skittles.BoldWhite(com.Name)+" received"
					utils.PrintEvent("event", msg)
					if (com.Name == "reparse_templates") {
						go reparseStatic()
					}
				}
        	}
        }()
        go func() {
			for {
				routes_done := <- c2
				if (routes_done == true) {
					//fmt.Println("[OK] Routes updated")
					go reparseStatic()
				}
			}
		}()
	}
	// http server
	http_port := Config["http_port"].(string)
	msg := "Server started on "+http_port+" ..."
	utils.PrintEvent("nil", msg)
	http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("./static/"))))
	http.Handle("/media/", http.StripPrefix("/media/", http.FileServer(http.Dir("./media/"))))
	http.HandleFunc("/x/", apiHandler)
    http.HandleFunc("/", viewHandler)
    log.Fatal(http.ListenAndServe(http_port, nil))
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/base64"
	"fmt"
	"github.com/caarlos0/env"
	_ "github.com/mattn/go-sqlite3"
	"net/http"
	"os"
	"time"
)

const pixelRaw = "R0lGODlhAQABAIAAANvf7wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw=="

type Config struct {
	Port                 string `env:"PORT"             envDefault:"8080"`
	DBFile               string `env:"DB_FILE"          envDefault:"out.db"`
	MaxConnections       int    `env:"MAX_CONNECTIONS"  envDefault:"100000"`
	WriteQueueSize       int    `env:"WRITE_QUEUE_SIZE" envDefault:"100000"`
	WriteFrequencyMillis int    `env:"WRITE_FREQUENCY"  envDefault:"3"`
}

var cfg = Config{}

func main() {
	err := env.Parse(&cfg)
	if err != nil {
		panic(err)
	}

	db := initDB()
	defer db.Close()

	eventQueue := make(chan *PageEvent, cfg.MaxConnections)
	http.HandleFunc("/a.gif", makeHandlePixel(eventQueue))
	go runEventWriter(db, eventQueue)

	if http.ListenAndServe(":"+cfg.Port, nil) != nil {
		fmt.Fprintf(os.Stderr, "Failed to start server")
	}
}

func makeHandlePixel(eventQueue chan *PageEvent) http.HandlerFunc {
	pixel, err := base64.StdEncoding.DecodeString(pixelRaw)
	if err != nil {
		panic(err)
	}

	return func(out http.ResponseWriter, req *http.Request) {
		eventQueue <- PageEventFromRequest(req)
		out.Header().Set("Content-Type", "image/gif")
		out.WriteHeader(http.StatusOK)
		out.Write(pixel)
	}
}

func runEventWriter(db *sql.DB, eventQueue chan *PageEvent) {
	queue := make([]*PageEvent, 0, cfg.WriteQueueSize)
	executeWrite := make(chan bool)

	go func() {
		for {
			select {
			case <-time.After(time.Millisecond * time.Duration(cfg.WriteFrequencyMillis)):
				executeWrite <- true
			}
		}
	}()

	for {
		select {
		case ev := <-eventQueue:
			queue = append(queue, ev)
		case <-executeWrite:
			if len(queue) > 0 {
				db.Exec("BEGIN TRANSACTION;")
				for _, ev := range queue {
					ev.InsertIntoDB(db)
				}
				db.Exec("END TRANSACTION;")
				fmt.Println("Wrote", len(queue), "records. Queue capacity is ", cap(queue))
				queue = queue[:0]
			}
		}
	}
}

func initDB() *sql.DB {
	db, err := sql.Open("sqlite3", cfg.DBFile)

	if err != nil {
		panic(err)
	}
	if db == nil {
		panic("DB is null")
	}

	_, err = db.Exec(SQLPageEventCreateTable)
	if err != nil {
		panic(err)
	}

	return db
}
<commit_msg>tuning<commit_after>package main

import (
	"database/sql"
	"encoding/base64"
	"fmt"
	"github.com/caarlos0/env"
	_ "github.com/mattn/go-sqlite3"
	"net/http"
	"os"
	"time"
)

const (
	pixelRaw   = "R0lGODlhAQABAIAAANvf7wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw=="
	pixelRoute = "a.gif"
)

type Config struct {
	Port                 string `env:"PORT"             envDefault:"8080"`
	DBFile               string `env:"DB_FILE"          envDefault:"out.db"`
	MaxConnections       int    `env:"MAX_CONNECTIONS"  envDefault:"100000"`
	WriteQueueSize       int    `env:"WRITE_QUEUE_SIZE" envDefault:"100000"`
	WriteFrequencyMillis int    `env:"WRITE_FREQUENCY"  envDefault:"8"`
}

var cfg = Config{}

func main() {
	err := env.Parse(&cfg)
	if err != nil {
		panic(err)
	}

	db := initDB()
	defer db.Close()

	eventQueue := make(chan *PageEvent, cfg.MaxConnections)
	http.HandleFunc(pixelRoute, makeHandlePixel(eventQueue))
	go runEventWriter(db, eventQueue)

	if http.ListenAndServe(":"+cfg.Port, nil) != nil {
		fmt.Fprintf(os.Stderr, "Failed to start server")
	}
}

func makeHandlePixel(eventQueue chan *PageEvent) http.HandlerFunc {
	pixel, err := base64.StdEncoding.DecodeString(pixelRaw)
	if err != nil {
		panic(err)
	}

	return func(out http.ResponseWriter, req *http.Request) {
		eventQueue <- PageEventFromRequest(req)
		out.Header().Set("Content-Type", "image/gif")
		out.WriteHeader(http.StatusOK)
		out.Write(pixel)
	}
}

func runEventWriter(db *sql.DB, eventQueue chan *PageEvent) {
	queue := make([]*PageEvent, 0, cfg.WriteQueueSize)
	executeWrite := make(chan bool)

	go func() {
		for {
			select {
			case <-time.After(time.Millisecond * time.Duration(cfg.WriteFrequencyMillis)):
				executeWrite <- true
			}
		}
	}()

	for {
		select {
		case ev := <-eventQueue:
			queue = append(queue, ev)
		case <-executeWrite:
			if len(queue) > 0 {
				db.Exec("BEGIN TRANSACTION;")
				for _, ev := range queue {
					ev.InsertIntoDB(db)
				}
				db.Exec("END TRANSACTION;")
				fmt.Println("Wrote", len(queue), "records.")
				queue = queue[:0]
			}
		}
	}
}

func initDB() *sql.DB {
	db, err := sql.Open("sqlite3", cfg.DBFile)

	if err != nil {
		panic(err)
	}
	if db == nil {
		panic("DB is null")
	}

	_, err = db.Exec(SQLPageEventCreateTable)
	if err != nil {
		panic(err)
	}

	return db
}
<|endoftext|>


<commit_before>// MsgBox Binary
//
// This is the main binary for running MsgBox.
//
// It reads in a config file at runtime and spawns:
//    - Relay
//    - Submission Agent
//    - (n) Incoming Workers
//    - (n) Outgoing Workers
//
package main

import (
	"github.com/hgfischer/goconf"
	"github.com/msgbox/relay"
	"github.com/msgbox/submission-agent"
	"github.com/msgbox/workers"
)

func main() {

	// Read Config File
	c, err := conf.ReadConfigFile("msgbox.conf")
	if err != nil {
		// Handle Error
	}

	// Startup Relay
	relay_external_port, _ := c.GetString("", "relay-external-port")
	relay_internal_port, _ := c.GetString("", "relay-internal-port")

	bindIncoming(string(relay_external_port))
	bindOutgoing(string(relay_internal_port))

	// Start Workers
	incoming_workers, _ := c.GetInt("", "incoming-workers")
	outgoing_workers, _ := c.GetInt("", "outgoing-workers")

	for i := 0; i < incoming_workers; i++ {
		createIncomingWorker(string(i))
	}

	for i := 0; i < outgoing_workers; i++ {
		createOutgoingWorker(string(i), relay_internal_port)
	}

	// Start Submission Agent
	submission_agent_port, _ := c.GetString("", "submission-port")
	createSubmissionAgent(submission_agent_port)
}

func bindIncoming(port string) {
	go relay.ListenIncoming(port)
}

func bindOutgoing(port string) {
	go relay.ListenOutgoing(port)
}

func createIncomingWorker(tag string) {
	go workers.CreateIncoming(tag)
}

func createOutgoingWorker(tag string, port string) {
	go workers.CreateOutgoing(tag, port)
}

func createSubmissionAgent(port string) {
	submission_agent.CreateAgent(port)
}
<commit_msg>update config file location<commit_after>// MsgBox Binary
//
// This is the main binary for running MsgBox.
//
// It reads in a config file at runtime and spawns:
//    - Relay
//    - Submission Agent
//    - (n) Incoming Workers
//    - (n) Outgoing Workers
//
package main

import (
	"github.com/hgfischer/goconf"
	"github.com/msgbox/relay"
	"github.com/msgbox/submission-agent"
	"github.com/msgbox/workers"
)

func main() {

	// Read Config File
	c, err := conf.ReadConfigFile("/etc/msgbox/msgbox.conf")
	if err != nil {
		// Handle Error
	}

	// Startup Relay
	relay_external_port, _ := c.GetString("", "relay-external-port")
	relay_internal_port, _ := c.GetString("", "relay-internal-port")

	bindIncoming(string(relay_external_port))
	bindOutgoing(string(relay_internal_port))

	// Start Workers
	incoming_workers, _ := c.GetInt("", "incoming-workers")
	outgoing_workers, _ := c.GetInt("", "outgoing-workers")

	for i := 0; i < incoming_workers; i++ {
		createIncomingWorker(string(i))
	}

	for i := 0; i < outgoing_workers; i++ {
		createOutgoingWorker(string(i), relay_internal_port)
	}

	// Start Submission Agent
	submission_agent_port, _ := c.GetString("", "submission-port")
	createSubmissionAgent(submission_agent_port)
}

func bindIncoming(port string) {
	go relay.ListenIncoming(port)
}

func bindOutgoing(port string) {
	go relay.ListenOutgoing(port)
}

func createIncomingWorker(tag string) {
	go workers.CreateIncoming(tag)
}

func createOutgoingWorker(tag string, port string) {
	go workers.CreateOutgoing(tag, port)
}

func createSubmissionAgent(port string) {
	submission_agent.CreateAgent(port)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"os"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/fsouza/go-dockerclient"
	"github.com/mesos/mesos-go/executor"
	mesos "github.com/mesos/mesos-go/mesosproto"
	"github.com/nitro/sidecar-executor/container"
)

type sidecarExecutor struct {
	driver *executor.MesosExecutorDriver
	client *docker.Client
}

func newSidecarExecutor(client *docker.Client) *sidecarExecutor {
	return &sidecarExecutor{
		client: client,
	}
}

const (
	TaskRunning  = 0
	TaskFinished = iota
	TaskFailed   = iota
)

func (exec *sidecarExecutor) sendStatus(status int64, taskInfo *mesos.TaskInfo) {
	var mesosStatus *mesos.TaskState
	switch status {
	case TaskRunning:
		mesosStatus = mesos.TaskState_TASK_RUNNING.Enum()
	case TaskFinished:
		mesosStatus = mesos.TaskState_TASK_FINISHED.Enum()
	case TaskFailed:
		mesosStatus = mesos.TaskState_TASK_FAILED.Enum()
	}

	update := &mesos.TaskStatus{
		TaskId: taskInfo.GetTaskId(),
		State:  mesosStatus,
	}

	if _, err := exec.driver.SendStatusUpdate(update); err != nil {
		log.Errorf("Error sending status update %s", err.Error())
		panic(err.Error())
	}
}

func (exec *sidecarExecutor) Registered(driver executor.ExecutorDriver, execInfo *mesos.ExecutorInfo, fwinfo *mesos.FrameworkInfo, slaveInfo *mesos.SlaveInfo) {
	log.Info("Registered Executor on slave ", slaveInfo.GetHostname())
}

func (exec *sidecarExecutor) Reregistered(driver executor.ExecutorDriver, slaveInfo *mesos.SlaveInfo) {
	log.Info("Re-registered Executor on slave ", slaveInfo.GetHostname())
}

func (exec *sidecarExecutor) Disconnected(driver executor.ExecutorDriver) {
	log.Info("Executor disconnected.")
}

func (exec *sidecarExecutor) LaunchTask(driver executor.ExecutorDriver, taskInfo *mesos.TaskInfo) {
	log.Infof("Launching task %s with command '%s'", taskInfo.GetName(), taskInfo.Command.GetValue())
	log.Info("Task ID ", taskInfo.GetTaskId().GetValue())

	// Store the task info we were passed so we can look at it
	info, _ := json.Marshal(taskInfo)
	ioutil.WriteFile("/tmp/taskinfo.json", info, os.ModeAppend)

	exec.sendStatus(TaskRunning, taskInfo)

	// TODO implement configurable pull timeout?
	if *taskInfo.Container.Docker.ForcePullImage {
		container.PullImage(exec.client, taskInfo)
	}

	// Configure and create the container
	containerConfig := container.ConfigForTask(taskInfo)
	container, err := exec.client.CreateContainer(*containerConfig)
	if err != nil {
		log.Error("Failed to create Docker container: %s", err.Error())
		exec.failTask(taskInfo)
		return
	}

	// Start the container
	err = exec.client.StartContainer(container.ID, containerConfig.HostConfig)
	if err != nil {
		log.Error("Failed to create Docker container: %s", err.Error())
		exec.failTask(taskInfo)
		return
	}

	// Tell Mesos and thus the framework that we're done
	exec.sendStatus(TaskFinished, taskInfo)

	log.Info("Task completed: ", taskInfo.GetName())

	// Unfortunately the status updates are sent async and we can't
	// get a handle on the channel used to send them. So we wait
	// and pray that it completes in a second.
	time.Sleep(1 * time.Second)

	// We're done with this executor, so let's stop now.
	driver.Stop()
}

func (exec *sidecarExecutor) failTask(taskInfo *mesos.TaskInfo) {
	// Tell Mesos and thus the framework that the task failed
	exec.sendStatus(TaskFailed, taskInfo)

	// Unfortunately the status updates are sent async and we can't
	// get a handle on the channel used to send them. So we wait
	time.Sleep(1 * time.Second)

	// We're done with this executor, so let's stop now.
	exec.driver.Stop()
}

func (exec *sidecarExecutor) KillTask(driver executor.ExecutorDriver, taskID *mesos.TaskID) {
	log.Info("Kill task")
}

func (exec *sidecarExecutor) FrameworkMessage(driver executor.ExecutorDriver, msg string) {
	log.Info("Got framework message: ", msg)
}

func (exec *sidecarExecutor) Shutdown(driver executor.ExecutorDriver) {
	log.Info("Shutting down the executor")
}

func (exec *sidecarExecutor) Error(driver executor.ExecutorDriver, err string) {
	log.Info("Got error message:", err)
}

func init() {
	flag.Parse()
	log.SetOutput(os.Stdout)
	log.SetLevel(log.DebugLevel)
}

func main() {
	log.Info("Starting Sidecar Executor")

	// Get a Docker client. Without one, we can't do anything.
	dockerClient, err := docker.NewClientFromEnv()
	if err != nil {
		log.Error(err.Error())
		os.Exit(1)
	}

	scExec := newSidecarExecutor(dockerClient)

	dconfig := executor.DriverConfig{
		Executor: scExec,
	}

	driver, err := executor.NewMesosExecutorDriver(dconfig)
	if err != nil || driver == nil {
		log.Info("Unable to create an ExecutorDriver ", err.Error())
	}

	// Give the executor a reference to the driver
	scExec.driver = driver

	_, err = driver.Start()
	if err != nil {
		log.Info("Got error:", err)
		return
	}

	log.Info("Executor process has started")

	_, err = driver.Join()
	if err != nil {
		log.Info("driver failed:", err)
	}

	log.Info("Sidecar Executor exiting")
}

/*
{
   "resources" : [
      {
         "type" : 1,
         "name" : "ports",
         "ranges" : {
            "range" : [
               {
                  "begin" : 31711,
                  "end" : 31711
               }
            ]
         }
      },
      {
         "name" : "cpus",
         "scalar" : {
            "value" : 0.1
         },
         "type" : 0
      },
      {
         "name" : "mem",
         "scalar" : {
            "value" : 128
         },
         "type" : 0
      }
   ],
   "labels" : {},
   "slave_id" : {
      "value" : "48647419-b03c-48f3-b938-2c2ad869eaab-S1"
   },
   "executor" : {
      "framework_id" : {
         "value" : "Singularity"
      },
      "source" : "nginx-2392676-1479746264572-1-NEW_DEPLOY-1479746261223",
      "command" : {
         "value" : "/home/kmatthias/sidecar-executor",
         "environment" : {
            "variables" : [
               {
                  "name" : "INSTANCE_NO",
                  "value" : "1"
               },
               {
                  "value" : "dev-singularity-sick-sing",
                  "name" : "TASK_HOST"
               },
               {
                  "name" : "TASK_REQUEST_ID",
                  "value" : "nginx"
               },
               {
                  "name" : "TASK_DEPLOY_ID",
                  "value" : "2392676"
               },
               {
                  "value" : "nginx-2392676-1479746266455-1-dev_singularity_sick_sing-DEFAULT",
                  "name" : "TASK_ID"
               },
               {
                  "name" : "ESTIMATED_INSTANCE_COUNT",
                  "value" : "3"
               },
               {
                  "name" : "PORT",
                  "value" : "31711"
               },
               {
                  "value" : "31711",
                  "name" : "PORT0"
               }
            ]
         }
      },
      "executor_id" : {
         "value" : "s1"
      }
   },
   "name" : "nginx",
   "task_id" : {
      "value" : "nginx-2392676-1479746266455-1-dev_singularity_sick_sing-DEFAULT"
   },
   "container" : {
      "docker" : {
         "network" : 2,
         "parameters" : [
            {
               "value" : "ServiceName=nginx",
               "key" : "label"
            },
            {
               "key" : "label",
               "value" : "ServicePort_80=11000"
            },
            {
               "value" : "HealthCheck=HttpGet",
               "key" : "label"
            },
            {
               "key" : "label",
               "value" : "HealthCheckArgs=http://{{ host }}:{{ tcp 11000 }}/"
            }
         ],
         "force_pull_image" : false,
         "privileged" : false,
         "image" : "nginx:latest",
         "port_mappings" : [
            {
               "protocol" : "tcp",
               "container_port" : 80,
               "host_port" : 31711
            }
         ]
      },
      "type" : 1
   }
}
*/
<commit_msg>Attempt to stay up while container is running<commit_after>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"io/ioutil"
	"os"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/fsouza/go-dockerclient"
	"github.com/mesos/mesos-go/executor"
	mesos "github.com/mesos/mesos-go/mesosproto"
	"github.com/nitro/sidecar-executor/container"
	"github.com/relistan/go-director"
)

type sidecarExecutor struct {
	driver *executor.MesosExecutorDriver
	client *docker.Client
}

func newSidecarExecutor(client *docker.Client) *sidecarExecutor {
	return &sidecarExecutor{
		client: client,
	}
}

const (
	TaskRunning  = 0
	TaskFinished = iota
	TaskFailed   = iota
)

const (
	KillTaskTimeout = 5 // seconds
)

func (exec *sidecarExecutor) sendStatus(status int64, taskInfo *mesos.TaskInfo) {
	var mesosStatus *mesos.TaskState
	switch status {
	case TaskRunning:
		mesosStatus = mesos.TaskState_TASK_RUNNING.Enum()
	case TaskFinished:
		mesosStatus = mesos.TaskState_TASK_FINISHED.Enum()
	case TaskFailed:
		mesosStatus = mesos.TaskState_TASK_FAILED.Enum()
	}

	update := &mesos.TaskStatus{
		TaskId: taskInfo.GetTaskId(),
		State:  mesosStatus,
	}

	if _, err := exec.driver.SendStatusUpdate(update); err != nil {
		log.Errorf("Error sending status update %s", err.Error())
		panic(err.Error())
	}
}

func (exec *sidecarExecutor) Registered(driver executor.ExecutorDriver, execInfo *mesos.ExecutorInfo, fwinfo *mesos.FrameworkInfo, slaveInfo *mesos.SlaveInfo) {
	log.Info("Registered Executor on slave ", slaveInfo.GetHostname())
}

func (exec *sidecarExecutor) Reregistered(driver executor.ExecutorDriver, slaveInfo *mesos.SlaveInfo) {
	log.Info("Re-registered Executor on slave ", slaveInfo.GetHostname())
}

func (exec *sidecarExecutor) Disconnected(driver executor.ExecutorDriver) {
	log.Info("Executor disconnected.")
}

func (exec *sidecarExecutor) LaunchTask(driver executor.ExecutorDriver, taskInfo *mesos.TaskInfo) {
	log.Infof("Launching task %s with command '%s'", taskInfo.GetName(), taskInfo.Command.GetValue())
	log.Info("Task ID ", taskInfo.GetTaskId().GetValue())

	// Store the task info we were passed so we can look at it
	info, _ := json.Marshal(taskInfo)
	ioutil.WriteFile("/tmp/taskinfo.json", info, os.ModeAppend)

	exec.sendStatus(TaskRunning, taskInfo)

	// TODO implement configurable pull timeout?
	if *taskInfo.Container.Docker.ForcePullImage {
		container.PullImage(exec.client, taskInfo)
	}

	// Configure and create the container
	containerConfig := container.ConfigForTask(taskInfo)
	container, err := exec.client.CreateContainer(*containerConfig)
	if err != nil {
		log.Error("Failed to create Docker container: %s", err.Error())
		exec.failTask(taskInfo)
		return
	}

	// Start the container
	err = exec.client.StartContainer(container.ID, containerConfig.HostConfig)
	if err != nil {
		log.Error("Failed to create Docker container: %s", err.Error())
		exec.failTask(taskInfo)
		return
	}

	// TODO may need to store the handle to the looper and stop it first
	// when killing a task.
	looper := director.NewImmediateTimedLooper(director.FOREVER, 3*time.Second, make(chan error))
	go exec.watchContainer(container, looper)

	// Block, waiting on the looper
	err = looper.Wait()
	if err != nil {
		log.Errorf("Error! %s", err.Error())
		exec.failTask(taskInfo)
		return
	}

	// Tell Mesos and thus the framework that we're done
	exec.sendStatus(TaskFinished, taskInfo)

	log.Info("Task completed: ", taskInfo.GetName())

	// Unfortunately the status updates are sent async and we can't
	// get a handle on the channel used to send them. So we wait
	// and pray that it completes in a second.
	time.Sleep(1 * time.Second)

	// We're done with this executor, so let's stop now.
	driver.Stop()
}

func (exec *sidecarExecutor) failTask(taskInfo *mesos.TaskInfo) {
	// Tell Mesos and thus the framework that the task failed
	exec.sendStatus(TaskFailed, taskInfo)

	// Unfortunately the status updates are sent async and we can't
	// get a handle on the channel used to send them. So we wait
	time.Sleep(1 * time.Second)

	// We're done with this executor, so let's stop now.
	exec.driver.Stop()
}

func (exec *sidecarExecutor) watchContainer(container *docker.Container, looper director.Looper) {
	looper.Loop(func() error {
		containers, err := exec.client.ListContainers(
			docker.ListContainersOptions{
				All: false,
			},
		)
		if err != nil {
			return err
		}

		// Loop through all the containers, looking for a running
		// container with our Id.
		ok := false
		for _, entry := range containers {
			if entry.ID == container.ID {
				ok = true
			}
		}
		if !ok {
			return errors.New("Container " + container.ID + " not running!")
		}

		return nil
	})
}

func (exec *sidecarExecutor) KillTask(driver executor.ExecutorDriver, taskID *mesos.TaskID) {
	log.Infof("Killing task: %s", *taskID.Value)
	err := exec.client.StopContainer(*taskID.Value, KillTaskTimeout)

	if err != nil {
		log.Errorf("Error! %s", err.Error())
	}

	// TODO should we be sending some kind of task status here?

	time.Sleep(1 * time.Second)
	exec.driver.Stop()
}

func (exec *sidecarExecutor) FrameworkMessage(driver executor.ExecutorDriver, msg string) {
	log.Info("Got framework message: ", msg)
}

func (exec *sidecarExecutor) Shutdown(driver executor.ExecutorDriver) {
	log.Info("Shutting down the executor")
}

func (exec *sidecarExecutor) Error(driver executor.ExecutorDriver, err string) {
	log.Info("Got error message:", err)
}

func init() {
	flag.Parse()
	log.SetOutput(os.Stdout)
	log.SetLevel(log.DebugLevel)
}

func main() {
	log.Info("Starting Sidecar Executor")

	// Get a Docker client. Without one, we can't do anything.
	dockerClient, err := docker.NewClientFromEnv()
	if err != nil {
		log.Error(err.Error())
		os.Exit(1)
	}

	scExec := newSidecarExecutor(dockerClient)

	dconfig := executor.DriverConfig{
		Executor: scExec,
	}

	driver, err := executor.NewMesosExecutorDriver(dconfig)
	if err != nil || driver == nil {
		log.Info("Unable to create an ExecutorDriver ", err.Error())
	}

	// Give the executor a reference to the driver
	scExec.driver = driver

	_, err = driver.Start()
	if err != nil {
		log.Info("Got error:", err)
		return
	}

	log.Info("Executor process has started")

	_, err = driver.Join()
	if err != nil {
		log.Info("driver failed:", err)
	}

	log.Info("Sidecar Executor exiting")
}

/*
{
   "resources" : [
      {
         "type" : 1,
         "name" : "ports",
         "ranges" : {
            "range" : [
               {
                  "begin" : 31711,
                  "end" : 31711
               }
            ]
         }
      },
      {
         "name" : "cpus",
         "scalar" : {
            "value" : 0.1
         },
         "type" : 0
      },
      {
         "name" : "mem",
         "scalar" : {
            "value" : 128
         },
         "type" : 0
      }
   ],
   "labels" : {},
   "slave_id" : {
      "value" : "48647419-b03c-48f3-b938-2c2ad869eaab-S1"
   },
   "executor" : {
      "framework_id" : {
         "value" : "Singularity"
      },
      "source" : "nginx-2392676-1479746264572-1-NEW_DEPLOY-1479746261223",
      "command" : {
         "value" : "/home/kmatthias/sidecar-executor",
         "environment" : {
            "variables" : [
               {
                  "name" : "INSTANCE_NO",
                  "value" : "1"
               },
               {
                  "value" : "dev-singularity-sick-sing",
                  "name" : "TASK_HOST"
               },
               {
                  "name" : "TASK_REQUEST_ID",
                  "value" : "nginx"
               },
               {
                  "name" : "TASK_DEPLOY_ID",
                  "value" : "2392676"
               },
               {
                  "value" : "nginx-2392676-1479746266455-1-dev_singularity_sick_sing-DEFAULT",
                  "name" : "TASK_ID"
               },
               {
                  "name" : "ESTIMATED_INSTANCE_COUNT",
                  "value" : "3"
               },
               {
                  "name" : "PORT",
                  "value" : "31711"
               },
               {
                  "value" : "31711",
                  "name" : "PORT0"
               }
            ]
         }
      },
      "executor_id" : {
         "value" : "s1"
      }
   },
   "name" : "nginx",
   "task_id" : {
      "value" : "nginx-2392676-1479746266455-1-dev_singularity_sick_sing-DEFAULT"
   },
   "container" : {
      "docker" : {
         "network" : 2,
         "parameters" : [
            {
               "value" : "ServiceName=nginx",
               "key" : "label"
            },
            {
               "key" : "label",
               "value" : "ServicePort_80=11000"
            },
            {
               "value" : "HealthCheck=HttpGet",
               "key" : "label"
            },
            {
               "key" : "label",
               "value" : "HealthCheckArgs=http://{{ host }}:{{ tcp 11000 }}/"
            }
         ],
         "force_pull_image" : false,
         "privileged" : false,
         "image" : "nginx:latest",
         "port_mappings" : [
            {
               "protocol" : "tcp",
               "container_port" : 80,
               "host_port" : 31711
            }
         ]
      },
      "type" : 1
   }
}
*/
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"

	"github.com/emgee/go-xmpp/src/xmpp"
)

func fatalOnErr(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func xmppLogin(id string, pass string) *xmpp.XMPP {
	jid, err := xmpp.ParseJID(id)
	fatalOnErr(err)

	addr, err := xmpp.HomeServerAddrs(jid)
	fatalOnErr(err)

	stream, err := xmpp.NewStream(addr[0], nil)
	fatalOnErr(err)

	client, err := xmpp.NewClientXMPP(stream, jid, pass, nil)
	fatalOnErr(err)

	client.Out <- xmpp.Presence{}

	return client
}

func main() {
	xi := os.Getenv("XMPP_ID")
	xp := os.Getenv("XMPP_PASS")

	if len(xi) < 1 || len(xp) < 1 {
		log.Fatal("XMPP_ID or XMPP_PASS not set")
	}

	xc := xmppLogin(xi, xp)

	go func() {
		for msg := range xc.In {
			log.Printf("* recv: %v\n", msg)
		}
	}()

	select {}
}
<commit_msg>xmpp working<commit_after>package main

import (
	"log"
	"os"

	"github.com/emgee/go-xmpp/src/xmpp"
)

const (
	xmppBotAnswer = "im a dumb bot"
)

// helper function for error checks
func fatalOnErr(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

// starts xmpp session, sends initial presence and returns the xmpp client
func xmppLogin(id string, pass string) *xmpp.XMPP {
	// parse jid structure
	jid, err := xmpp.ParseJID(id)
	fatalOnErr(err)

	// extract/generate address:port from jid
	addr, err := xmpp.HomeServerAddrs(jid)
	fatalOnErr(err)

	// create xml stream to address
	stream, err := xmpp.NewStream(addr[0], nil)
	fatalOnErr(err)

	// create client (login)
	client, err := xmpp.NewClientXMPP(stream, jid, pass, nil)
	fatalOnErr(err)

	// announce presence
	client.Out <- xmpp.Presence{}

	return client
}

// creates MessageBody slice suitable for xmpp.Message
func xmppBodyCreate(message string) []xmpp.MessageBody {
	return []xmpp.MessageBody{
		xmpp.MessageBody{
			Value: message,
		},
	}
}

// handles incoming stanzas
func handleXMPPStanza(in <-chan interface{}, out chan<- interface{}) {
	for stanza := range in {
		// check if stanza is a message
		message, ok := stanza.(*xmpp.Message)
		if ok && len(message.Body) > 0 {
			// send constant as answer
			out <- xmpp.Message{
				To:   message.From,
				Body: xmppBodyCreate(xmppBotAnswer),
			}
		}
	}
}

func main() {
	// get xmpp credentials from ENV
	xi := os.Getenv("XMPP_ID")
	xp := os.Getenv("XMPP_PASS")

	// check if xmpp credentials are supplied
	if len(xi) < 1 || len(xp) < 1 {
		log.Fatal("XMPP_ID or XMPP_PASS not set")
	}

	// start xmpp client
	xc := xmppLogin(xi, xp)

	// dispatch incoming stanzas to handler
	go handleXMPPStanza(xc.In, xc.Out)

	select {}
}
<|endoftext|>