<commit_before>/*
 * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"fmt"
	"net"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/minio/cli"
)

var srvConfig serverCmdConfig
var serverCmd = cli.Command{
	Name:  "server",
	Usage: "Start object storage server.",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "address",
			Value: ":9000",
			Usage: "Specify custom server \"ADDRESS:PORT\", defaults to \":9000\".",
		},
		cli.StringFlag{
			Name:  "ignore-disks",
			Usage: "Specify comma separated list of disks that are offline.",
		},
	},
	Action: serverMain,
	CustomHelpTemplate: `NAME:
  minio {{.Name}} - {{.Usage}}

USAGE:
  minio {{.Name}} [OPTIONS] PATH [PATH...]

OPTIONS:
  {{range .Flags}}{{.}}
  {{end}}
ENVIRONMENT VARIABLES:
  ACCESS:
     MINIO_ACCESS_KEY: Access key string of 5 to 20 characters in length.
     MINIO_SECRET_KEY: Secret key string of 8 to 40 characters in length.

  CACHING:
     MINIO_CACHE_SIZE: Set total cache size in NN[GB|MB|KB]. Defaults to 8GB.
     MINIO_CACHE_EXPIRY: Set cache expiration duration in NN[h|m|s]. Defaults to 72 hours.

EXAMPLES:
  1. Start minio server.
      $ minio {{.Name}} /home/shared

  2. Start minio server bound to a specific IP:PORT, when you have multiple network interfaces.
      $ minio {{.Name}} --address 192.168.1.101:9000 /home/shared

  3. Start minio server on Windows.
      $ minio {{.Name}} C:\MyShare

  4. Start minio server on 12 disks to enable erasure coded layer with 6 data and 6 parity.
      $ minio {{.Name}} /mnt/export1/backend /mnt/export2/backend /mnt/export3/backend /mnt/export4/backend \
          /mnt/export5/backend /mnt/export6/backend /mnt/export7/backend /mnt/export8/backend /mnt/export9/backend \
          /mnt/export10/backend /mnt/export11/backend /mnt/export12/backend

  5. Start minio server on 12 disks while ignoring two disks for initialization.
      $ minio {{.Name}} --ignore-disks=/mnt/export1/backend,/mnt/export2/backend /mnt/export1/backend \
          /mnt/export2/backend /mnt/export3/backend /mnt/export4/backend /mnt/export5/backend /mnt/export6/backend \
          /mnt/export7/backend /mnt/export8/backend /mnt/export9/backend /mnt/export10/backend /mnt/export11/backend \
          /mnt/export12/backend
`,
}

type serverCmdConfig struct {
	serverAddr   string
	disks        []string
	ignoredDisks []string
}

// getListenIPs - gets all the ips to listen on.
func getListenIPs(httpServerConf *http.Server) (hosts []string, port string) {
	host, port, err := net.SplitHostPort(httpServerConf.Addr)
	fatalIf(err, "Unable to parse host address.", httpServerConf.Addr)

	if host != "" {
		hosts = append(hosts, host)
		return hosts, port
	}
	addrs, err := net.InterfaceAddrs()
	fatalIf(err, "Unable to determine network interface address.")
	for _, addr := range addrs {
		if addr.Network() == "ip+net" {
			host := strings.Split(addr.String(), "/")[0]
			if ip := net.ParseIP(host); ip.To4() != nil {
				hosts = append(hosts, host)
			}
		}
	}
	return hosts, port
}

// Finalizes the endpoints based on the host list and port.
func finalizeEndpoints(tls bool, apiServer *http.Server) (endPoints []string) {
	// Get list of listen ips and port.
	hosts, port := getListenIPs(apiServer)

	// Verify current scheme.
	scheme := "http"
	if tls {
		scheme = "https"
	}

	ips := getIPsFromHosts(hosts)

	// Construct proper endpoints.
	for _, ip := range ips {
		endPoints = append(endPoints, fmt.Sprintf("%s://%s:%s", scheme, ip.String(), port))
	}

	// Success.
	return endPoints
}

// initServerConfig initialize server config.
func initServerConfig(c *cli.Context) {
	// Create certs path.
	err := createCertsPath()
	fatalIf(err, "Unable to create \"certs\" directory.")

	// Fetch max conn limit from environment variable.
	if maxConnStr := os.Getenv("MINIO_MAXCONN"); maxConnStr != "" {
		// We need to parse to its integer value.
		globalMaxConn, err = strconv.Atoi(maxConnStr)
		fatalIf(err, "Unable to convert MINIO_MAXCONN=%s environment variable into its integer value.", maxConnStr)
	}

	// Fetch max cache size from environment variable.
	if maxCacheSizeStr := os.Getenv("MINIO_CACHE_SIZE"); maxCacheSizeStr != "" {
		// We need to parse cache size to its integer value.
		globalMaxCacheSize, err = strconvBytes(maxCacheSizeStr)
		fatalIf(err, "Unable to convert MINIO_CACHE_SIZE=%s environment variable into its integer value.", maxCacheSizeStr)
	}

	// Fetch cache expiry from environment variable.
	if cacheExpiryStr := os.Getenv("MINIO_CACHE_EXPIRY"); cacheExpiryStr != "" {
		// We need to parse cache expiry to its time.Duration value.
		globalCacheExpiry, err = time.ParseDuration(cacheExpiryStr)
		fatalIf(err, "Unable to convert MINIO_CACHE_EXPIRY=%s environment variable into its time.Duration value.", cacheExpiryStr)
	}

	// Fetch access keys from environment variables if any and update the config.
	accessKey := os.Getenv("MINIO_ACCESS_KEY")
	secretKey := os.Getenv("MINIO_SECRET_KEY")

	// Validate if both keys are specified and they are valid save them.
	if accessKey != "" && secretKey != "" {
		if !isValidAccessKey.MatchString(accessKey) {
			fatalIf(errInvalidArgument, "Invalid access key.")
		}
		if !isValidSecretKey.MatchString(secretKey) {
			fatalIf(errInvalidArgument, "Invalid secret key.")
		}
		// Set new credentials.
		serverConfig.SetCredential(credential{
			AccessKeyID:     accessKey,
			SecretAccessKey: secretKey,
		})
		// Save new config.
		err = serverConfig.Save()
		fatalIf(err, "Unable to save config.")
	}

	// Set maxOpenFiles, This is necessary since default operating
	// system limits of 1024, 2048 are not enough for Minio server.
	setMaxOpenFiles()
	// Set maxMemory, This is necessary since default operating
	// system limits might be changed and we need to make sure we
	// do not crash the server so the set the maxCacheSize appropriately.
	setMaxMemory()

	// Do not fail if this is not allowed, lower limits are fine as well.
}

// Check server arguments.
func checkServerSyntax(c *cli.Context) {
	if !c.Args().Present() || c.Args().First() == "help" {
		cli.ShowCommandHelpAndExit(c, "server", 1)
	}
}

// Extract port number from address address should be of the form host:port.
func getPort(address string) int {
	_, portStr, _ := net.SplitHostPort(address)
	// If port empty, default to port '80'
	if portStr == "" {
		portStr = "80"
		// if SSL is enabled, choose port as "443" instead.
		if isSSL() {
			portStr = "443"
		}
	}

	// Return converted port number.
	portInt, err := strconv.Atoi(portStr)
	fatalIf(err, "Invalid port number.")
	return portInt
}

// serverMain handler called for 'minio server' command.
func serverMain(c *cli.Context) {
	// Check 'server' cli arguments.
	checkServerSyntax(c)

	// Initialize server config.
	initServerConfig(c)

	// If https.
	tls := isSSL()

	// Server address.
	serverAddress := c.String("address")

	// Check if requested port is available.
	port := getPort(serverAddress)
	err := checkPortAvailability(port)
	fatalIf(err, "Port unavailable %d", port)

	// Disks to be ignored in server init, to skip format healing.
	ignoredDisks := strings.Split(c.String("ignore-disks"), ",")

	// Disks to be used in server init.
	disks := c.Args()

	// Set nodes for dsync
	var isDist bool
	isDist, err = initDsyncNodes(disks, port)
	fatalIf(err, "Unable to initialize distributed locking")

	// Initialize name space lock.
	initNSLock(isDist)

	// Configure server.
	srvConfig = serverCmdConfig{
		serverAddr:   serverAddress,
		disks:        disks,
		ignoredDisks: ignoredDisks,
	}
	// Configure server.
	handler := configureServerHandler(srvConfig)

	apiServer := NewServerMux(serverAddress, handler)

	// Fetch endpoints which we are going to serve from.
	endPoints := finalizeEndpoints(tls, &apiServer.Server)

	// Register generic callbacks.
	globalShutdownCBs.AddGenericCB(func() errCode {
		// apiServer.Stop()
		return exitSuccess
	})

	// Prints the formatted startup message.
	printStartupMessage(endPoints)

	// Start server.
	// Configure TLS if certs are available.
	if tls {
		err = apiServer.ListenAndServeTLS(mustGetCertFile(), mustGetKeyFile())
	} else {
		// Fallback to http.
		err = apiServer.ListenAndServe()
	}

	fatalIf(err, "Failed to start minio server.")
}
<commit_msg>server: Add server command line for running in distributed mode<commit_after>/*
 * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"fmt"
	"net"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/minio/cli"
)

var srvConfig serverCmdConfig
var serverCmd = cli.Command{
	Name:  "server",
	Usage: "Start object storage server.",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "address",
			Value: ":9000",
			Usage: "Specify custom server \"ADDRESS:PORT\", defaults to \":9000\".",
		},
		cli.StringFlag{
			Name:  "ignore-disks",
			Usage: "Specify comma separated list of disks that are offline.",
		},
	},
	Action: serverMain,
	CustomHelpTemplate: `NAME:
  minio {{.Name}} - {{.Usage}}

USAGE:
  minio {{.Name}} [OPTIONS] PATH [PATH...]

OPTIONS:
  {{range .Flags}}{{.}}
  {{end}}
ENVIRONMENT VARIABLES:
  ACCESS:
     MINIO_ACCESS_KEY: Access key string of 5 to 20 characters in length.
     MINIO_SECRET_KEY: Secret key string of 8 to 40 characters in length.

  CACHING:
     MINIO_CACHE_SIZE: Set total cache size in NN[GB|MB|KB]. Defaults to 8GB.
     MINIO_CACHE_EXPIRY: Set cache expiration duration in NN[h|m|s]. Defaults to 72 hours.

EXAMPLES:
  1. Start minio server.
      $ minio {{.Name}} /home/shared

  2. Start minio server bound to a specific IP:PORT, when you have multiple network interfaces.
      $ minio {{.Name}} --address 192.168.1.101:9000 /home/shared

  3. Start minio server on Windows.
      $ minio {{.Name}} C:\MyShare

  4. Start minio server on 12 disks to enable erasure coded layer with 6 data and 6 parity.
      $ minio {{.Name}} /mnt/export1/backend /mnt/export2/backend /mnt/export3/backend /mnt/export4/backend \
          /mnt/export5/backend /mnt/export6/backend /mnt/export7/backend /mnt/export8/backend /mnt/export9/backend \
          /mnt/export10/backend /mnt/export11/backend /mnt/export12/backend

  5. Start minio server on 12 disks while ignoring two disks for initialization.
      $ minio {{.Name}} --ignore-disks=/mnt/export1/backend,/mnt/export2/backend /mnt/export1/backend \
          /mnt/export2/backend /mnt/export3/backend /mnt/export4/backend /mnt/export5/backend /mnt/export6/backend \
          /mnt/export7/backend /mnt/export8/backend /mnt/export9/backend /mnt/export10/backend /mnt/export11/backend \
          /mnt/export12/backend

  6. Start minio server with 4 remote disks on all remote nodes. You need to execute the same command on all nodes
     192.168.1.11, 192.168.1.12, 192.168.1.13 and 192.168.1.14.
      $ export MINIO_ACCESS_KEY=minio
      $ export MINIO_SECRET_KEY=miniostorage
      $ minio {{.Name}} 192.168.1.11:/mnt/export1/backend 192.168.1.12:/mnt/export1/backend \
          192.168.1.13:/mnt/export1/backend 192.168.1.14:/mnt/export1/backend
`,
}

type serverCmdConfig struct {
	serverAddr   string
	disks        []string
	ignoredDisks []string
}

// getListenIPs - gets all the ips to listen on.
func getListenIPs(httpServerConf *http.Server) (hosts []string, port string) {
	host, port, err := net.SplitHostPort(httpServerConf.Addr)
	fatalIf(err, "Unable to parse host address.", httpServerConf.Addr)

	if host != "" {
		hosts = append(hosts, host)
		return hosts, port
	}
	addrs, err := net.InterfaceAddrs()
	fatalIf(err, "Unable to determine network interface address.")
	for _, addr := range addrs {
		if addr.Network() == "ip+net" {
			host := strings.Split(addr.String(), "/")[0]
			if ip := net.ParseIP(host); ip.To4() != nil {
				hosts = append(hosts, host)
			}
		}
	}
	return hosts, port
}

// Finalizes the endpoints based on the host list and port.
func finalizeEndpoints(tls bool, apiServer *http.Server) (endPoints []string) {
	// Get list of listen ips and port.
	hosts, port := getListenIPs(apiServer)

	// Verify current scheme.
	scheme := "http"
	if tls {
		scheme = "https"
	}

	ips := getIPsFromHosts(hosts)

	// Construct proper endpoints.
	for _, ip := range ips {
		endPoints = append(endPoints, fmt.Sprintf("%s://%s:%s", scheme, ip.String(), port))
	}

	// Success.
	return endPoints
}

// initServerConfig initialize server config.
func initServerConfig(c *cli.Context) {
	// Create certs path.
	err := createCertsPath()
	fatalIf(err, "Unable to create \"certs\" directory.")

	// Fetch max conn limit from environment variable.
	if maxConnStr := os.Getenv("MINIO_MAXCONN"); maxConnStr != "" {
		// We need to parse to its integer value.
		globalMaxConn, err = strconv.Atoi(maxConnStr)
		fatalIf(err, "Unable to convert MINIO_MAXCONN=%s environment variable into its integer value.", maxConnStr)
	}

	// Fetch max cache size from environment variable.
	if maxCacheSizeStr := os.Getenv("MINIO_CACHE_SIZE"); maxCacheSizeStr != "" {
		// We need to parse cache size to its integer value.
		globalMaxCacheSize, err = strconvBytes(maxCacheSizeStr)
		fatalIf(err, "Unable to convert MINIO_CACHE_SIZE=%s environment variable into its integer value.", maxCacheSizeStr)
	}

	// Fetch cache expiry from environment variable.
	if cacheExpiryStr := os.Getenv("MINIO_CACHE_EXPIRY"); cacheExpiryStr != "" {
		// We need to parse cache expiry to its time.Duration value.
		globalCacheExpiry, err = time.ParseDuration(cacheExpiryStr)
		fatalIf(err, "Unable to convert MINIO_CACHE_EXPIRY=%s environment variable into its time.Duration value.", cacheExpiryStr)
	}

	// Fetch access keys from environment variables if any and update the config.
	accessKey := os.Getenv("MINIO_ACCESS_KEY")
	secretKey := os.Getenv("MINIO_SECRET_KEY")

	// Validate if both keys are specified and they are valid save them.
	if accessKey != "" && secretKey != "" {
		if !isValidAccessKey.MatchString(accessKey) {
			fatalIf(errInvalidArgument, "Invalid access key.")
		}
		if !isValidSecretKey.MatchString(secretKey) {
			fatalIf(errInvalidArgument, "Invalid secret key.")
		}
		// Set new credentials.
		serverConfig.SetCredential(credential{
			AccessKeyID:     accessKey,
			SecretAccessKey: secretKey,
		})
		// Save new config.
		err = serverConfig.Save()
		fatalIf(err, "Unable to save config.")
	}

	// Set maxOpenFiles, This is necessary since default operating
	// system limits of 1024, 2048 are not enough for Minio server.
	setMaxOpenFiles()
	// Set maxMemory, This is necessary since default operating
	// system limits might be changed and we need to make sure we
	// do not crash the server so the set the maxCacheSize appropriately.
	setMaxMemory()

	// Do not fail if this is not allowed, lower limits are fine as well.
}

// Check server arguments.
func checkServerSyntax(c *cli.Context) {
	if !c.Args().Present() || c.Args().First() == "help" {
		cli.ShowCommandHelpAndExit(c, "server", 1)
	}
}

// Extract port number from address address should be of the form host:port.
func getPort(address string) int {
	_, portStr, _ := net.SplitHostPort(address)
	// If port empty, default to port '80'
	if portStr == "" {
		portStr = "80"
		// if SSL is enabled, choose port as "443" instead.
		if isSSL() {
			portStr = "443"
		}
	}

	// Return converted port number.
	portInt, err := strconv.Atoi(portStr)
	fatalIf(err, "Invalid port number.")
	return portInt
}

// serverMain handler called for 'minio server' command.
func serverMain(c *cli.Context) {
	// Check 'server' cli arguments.
	checkServerSyntax(c)

	// Initialize server config.
	initServerConfig(c)

	// If https.
	tls := isSSL()

	// Server address.
	serverAddress := c.String("address")

	// Check if requested port is available.
	port := getPort(serverAddress)
	err := checkPortAvailability(port)
	fatalIf(err, "Port unavailable %d", port)

	// Disks to be ignored in server init, to skip format healing.
	ignoredDisks := strings.Split(c.String("ignore-disks"), ",")

	// Disks to be used in server init.
	disks := c.Args()

	// Set nodes for dsync
	var isDist bool
	isDist, err = initDsyncNodes(disks, port)
	fatalIf(err, "Unable to initialize distributed locking")

	// Initialize name space lock.
	initNSLock(isDist)

	// Configure server.
	srvConfig = serverCmdConfig{
		serverAddr:   serverAddress,
		disks:        disks,
		ignoredDisks: ignoredDisks,
	}
	// Configure server.
	handler := configureServerHandler(srvConfig)

	apiServer := NewServerMux(serverAddress, handler)

	// Fetch endpoints which we are going to serve from.
	endPoints := finalizeEndpoints(tls, &apiServer.Server)

	// Register generic callbacks.
	globalShutdownCBs.AddGenericCB(func() errCode {
		// apiServer.Stop()
		return exitSuccess
	})

	// Prints the formatted startup message.
	printStartupMessage(endPoints)

	// Start server.
	// Configure TLS if certs are available.
	if tls {
		err = apiServer.ListenAndServeTLS(mustGetCertFile(), mustGetKeyFile())
	} else {
		// Fallback to http.
		err = apiServer.ListenAndServe()
	}

	fatalIf(err, "Failed to start minio server.")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	log "github.com/Sirupsen/logrus"
	"github.com/emc-advanced-dev/unik/pkg/compilers"
)

func main() {
	os.Setenv("TMPDIR", "/Users/kohavy/tmp")
	log.SetLevel(log.DebugLevel)

	r := compilers.RunmpCompiler{
		DockerImage: "rumpcompiler-go-xen",
		CreateImage: compilers.CreateImageAws,
	}
	f, err := os.Open("a.tar")
	if err != nil {
		panic(err)
	}
	img, err := r.CompileRawImage(f, "", []string{"/yuval"})
	if err != nil {
		panic(err)
	}

	fmt.Print(img)
}
<commit_msg>virtualbox<commit_after>package main

import (
	"fmt"
	"os"

	log "github.com/Sirupsen/logrus"
	"github.com/emc-advanced-dev/unik/pkg/compilers"
)

func main() {
	os.Setenv("TMPDIR", "/Users/kohavy/tmp")
	log.SetLevel(log.DebugLevel)

	r := compilers.RunmpCompiler{
		DockerImage: "rumpcompiler-go-hw",
		CreateImage: compilers.CreateImageVirtualBox,
	}
	f, err := os.Open("a.tar")
	if err != nil {
		panic(err)
	}
	img, err := r.CompileRawImage(f, "", []string{})
	if err != nil {
		panic(err)
	}

	fmt.Print(img)
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"net"
	"net/url"
	"os"

	"github.com/cloudfoundry/dropsonde"
	"github.com/hashicorp/consul/api"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"

	"code.cloudfoundry.org/bbs"
	"code.cloudfoundry.org/clock"
	"code.cloudfoundry.org/consuladapter"
	"code.cloudfoundry.org/debugserver"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/lager/lagerflags"
	"code.cloudfoundry.org/locket"
	"code.cloudfoundry.org/runtimeschema/cc_messages"
	"code.cloudfoundry.org/runtimeschema/cc_messages/flags"
	"code.cloudfoundry.org/stager/backend"
	"code.cloudfoundry.org/stager/cc_client"
	"code.cloudfoundry.org/stager/config"
	"code.cloudfoundry.org/stager/handlers"
)

var configPath = flag.String(
	"configPath",
	"",
	"path to the stager configuration file",
)

const (
	dropsondeOrigin = "stager"
)

func main() {
	flag.Parse()

	stagerConfig, err := config.NewStagerConfig(*configPath)
	if err != nil {
		panic(err.Error())
	}
	lifecycles := flags.LifecycleMap{}
	for _, value := range stagerConfig.Lifecycles {
		if err := lifecycles.Set(value); err != nil {
			panic(err.Error())
		}
	}

	logger, reconfigurableSink := lagerflags.NewFromConfig("stager", stagerConfig.LagerConfig)

	initializeDropsonde(logger, stagerConfig)

	ccClient := cc_client.NewCcClient(stagerConfig.CCBaseUrl, stagerConfig.CCUsername, stagerConfig.CCPassword, stagerConfig.SkipCertVerify)
	backends := initializeBackends(logger, lifecycles, stagerConfig)

	handler := handlers.New(logger, ccClient, initializeBBSClient(logger, stagerConfig), backends, clock.NewClock())

	clock := clock.NewClock()
	consulClient, err := consuladapter.NewClientFromUrl(stagerConfig.ConsulCluster)
	if err != nil {
		logger.Fatal("new-client-failed", err)
	}

	_, portString, err := net.SplitHostPort(stagerConfig.ListenAddress)
	if err != nil {
		logger.Fatal("failed-invalid-listen-address", err)
	}
	portNum, err := net.LookupPort("tcp", portString)
	if err != nil {
		logger.Fatal("failed-invalid-listen-port", err)
	}

	registrationRunner := initializeRegistrationRunner(logger, consulClient, portNum, clock)

	members := grouper.Members{
		{"server", http_server.New(stagerConfig.ListenAddress, handler)},
		{"registration-runner", registrationRunner},
	}

	if dbgAddr := stagerConfig.DebugServerConfig.DebugAddress; dbgAddr != "" {
		members = append(grouper.Members{
			{"debug-server", debugserver.Runner(dbgAddr, reconfigurableSink)},
		}, members...)
	}

	logger.Info("starting")

	group := grouper.NewOrdered(os.Interrupt, members)

	process := ifrit.Invoke(sigmon.New(group))

	logger.Info("Listening for staging requests!")

	err = <-process.Wait()
	if err != nil {
		logger.Fatal("Stager exited with error", err)
	}

	logger.Info("stopped")
}

func initializeDropsonde(logger lager.Logger, stagerConfig config.StagerConfig) {
	dropsondeDestination := fmt.Sprint("localhost:", stagerConfig.DropsondePort)
	err := dropsonde.Initialize(dropsondeDestination, dropsondeOrigin)
	if err != nil {
		logger.Error("failed to initialize dropsonde: %v", err)
	}
}

func initializeBackends(logger lager.Logger, lifecycles flags.LifecycleMap, stagerConfig config.StagerConfig) map[string]backend.Backend {
	_, err := url.Parse(stagerConfig.StagingTaskCallbackURL)
	if err != nil {
		logger.Fatal("Invalid staging task callback url", err)
	}
	if stagerConfig.DockerStagingStack == "" {
		logger.Fatal("Invalid Docker staging stack", errors.New("dockerStagingStack cannot be blank"))
	}

	_, err = url.Parse(stagerConfig.ConsulCluster)
	if err != nil {
		logger.Fatal("Error parsing consul agent URL", err)
	}
	config := backend.Config{
		TaskDomain:               cc_messages.StagingTaskDomain,
		StagerURL:                stagerConfig.StagingTaskCallbackURL,
		FileServerURL:            stagerConfig.FileServerUrl,
		CCUploaderURL:            stagerConfig.CCUploaderURL,
		Lifecycles:               lifecycles,
		InsecureDockerRegistries: stagerConfig.InsecureDockerRegistries,
		ConsulCluster:            stagerConfig.ConsulCluster,
		SkipCertVerify:           stagerConfig.SkipCertVerify,
		PrivilegedContainers:     stagerConfig.PrivilegedContainers,
		Sanitizer:                backend.SanitizeErrorMessage,
		DockerStagingStack:       stagerConfig.DockerStagingStack,
	}

	return map[string]backend.Backend{
		"buildpack": backend.NewTraditionalBackend(config, logger),
		"docker":    backend.NewDockerBackend(config, logger),
	}
}

func initializeBBSClient(logger lager.Logger, stagerConfig config.StagerConfig) bbs.Client {
	bbsURL, err := url.Parse(stagerConfig.BBSAddress)
	if err != nil {
		logger.Fatal("Invalid BBS URL", err)
	}

	if bbsURL.Scheme != "https" {
		return bbs.NewClient(stagerConfig.BBSAddress)
	}

	bbsClient, err := bbs.NewSecureClient(stagerConfig.BBSAddress, stagerConfig.BBSCACert, stagerConfig.BBSClientCert, stagerConfig.BBSClientKey, stagerConfig.BBSClientSessionCacheSize, stagerConfig.BBSMaxIdleConnsPerHost)
	if err != nil {
		logger.Fatal("Failed to configure secure BBS client", err)
	}
	return bbsClient
}

func initializeRegistrationRunner(logger lager.Logger, consulClient consuladapter.Client, port int, clock clock.Clock) ifrit.Runner {
	registration := &api.AgentServiceRegistration{
		Name: "stager",
		Port: port,
		Check: &api.AgentServiceCheck{
			TTL: "20s",
		},
	}
	return locket.NewRegistrationRunner(logger, registration, consulClient, locket.RetryInterval, clock)
}
<commit_msg>update stager to use new bbs interface<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"net"
	"net/url"
	"os"

	"github.com/cloudfoundry/dropsonde"
	"github.com/hashicorp/consul/api"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"

	"code.cloudfoundry.org/bbs"
	"code.cloudfoundry.org/clock"
	"code.cloudfoundry.org/consuladapter"
	"code.cloudfoundry.org/debugserver"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/lager/lagerflags"
	"code.cloudfoundry.org/locket"
	"code.cloudfoundry.org/runtimeschema/cc_messages"
	"code.cloudfoundry.org/runtimeschema/cc_messages/flags"
	"code.cloudfoundry.org/stager/backend"
	"code.cloudfoundry.org/stager/cc_client"
	"code.cloudfoundry.org/stager/config"
	"code.cloudfoundry.org/stager/handlers"
)

var configPath = flag.String(
	"configPath",
	"",
	"path to the stager configuration file",
)

const (
	dropsondeOrigin = "stager"
)

func main() {
	flag.Parse()

	stagerConfig, err := config.NewStagerConfig(*configPath)
	if err != nil {
		panic(err.Error())
	}
	lifecycles := flags.LifecycleMap{}
	for _, value := range stagerConfig.Lifecycles {
		if err := lifecycles.Set(value); err != nil {
			panic(err.Error())
		}
	}

	logger, reconfigurableSink := lagerflags.NewFromConfig("stager", stagerConfig.LagerConfig)

	initializeDropsonde(logger, stagerConfig)

	ccClient := cc_client.NewCcClient(stagerConfig.CCBaseUrl, stagerConfig.CCUsername, stagerConfig.CCPassword, stagerConfig.SkipCertVerify)
	backends := initializeBackends(logger, lifecycles, stagerConfig)

	handler := handlers.New(logger, ccClient, initializeBBSClient(logger, stagerConfig), backends, clock.NewClock())

	clock := clock.NewClock()
	consulClient, err := consuladapter.NewClientFromUrl(stagerConfig.ConsulCluster)
	if err != nil {
		logger.Fatal("new-client-failed", err)
	}

	_, portString, err := net.SplitHostPort(stagerConfig.ListenAddress)
	if err != nil {
		logger.Fatal("failed-invalid-listen-address", err)
	}
	portNum, err := net.LookupPort("tcp", portString)
	if err != nil {
		logger.Fatal("failed-invalid-listen-port", err)
	}

	registrationRunner := initializeRegistrationRunner(logger, consulClient, portNum, clock)

	members := grouper.Members{
		{"server", http_server.New(stagerConfig.ListenAddress, handler)},
		{"registration-runner", registrationRunner},
	}

	if dbgAddr := stagerConfig.DebugServerConfig.DebugAddress; dbgAddr != "" {
		members = append(grouper.Members{
			{"debug-server", debugserver.Runner(dbgAddr, reconfigurableSink)},
		}, members...)
	}

	logger.Info("starting")

	group := grouper.NewOrdered(os.Interrupt, members)

	process := ifrit.Invoke(sigmon.New(group))

	logger.Info("Listening for staging requests!")

	err = <-process.Wait()
	if err != nil {
		logger.Fatal("Stager exited with error", err)
	}

	logger.Info("stopped")
}

func initializeDropsonde(logger lager.Logger, stagerConfig config.StagerConfig) {
	dropsondeDestination := fmt.Sprint("localhost:", stagerConfig.DropsondePort)
	err := dropsonde.Initialize(dropsondeDestination, dropsondeOrigin)
	if err != nil {
		logger.Error("failed to initialize dropsonde: %v", err)
	}
}

func initializeBackends(logger lager.Logger, lifecycles flags.LifecycleMap, stagerConfig config.StagerConfig) map[string]backend.Backend {
	_, err := url.Parse(stagerConfig.StagingTaskCallbackURL)
	if err != nil {
		logger.Fatal("Invalid staging task callback url", err)
	}
	if stagerConfig.DockerStagingStack == "" {
		logger.Fatal("Invalid Docker staging stack", errors.New("dockerStagingStack cannot be blank"))
	}

	_, err = url.Parse(stagerConfig.ConsulCluster)
	if err != nil {
		logger.Fatal("Error parsing consul agent URL", err)
	}
	config := backend.Config{
		TaskDomain:               cc_messages.StagingTaskDomain,
		StagerURL:                stagerConfig.StagingTaskCallbackURL,
		FileServerURL:            stagerConfig.FileServerUrl,
		CCUploaderURL:            stagerConfig.CCUploaderURL,
		Lifecycles:               lifecycles,
		InsecureDockerRegistries: stagerConfig.InsecureDockerRegistries,
		ConsulCluster:            stagerConfig.ConsulCluster,
		SkipCertVerify:           stagerConfig.SkipCertVerify,
		PrivilegedContainers:     stagerConfig.PrivilegedContainers,
		Sanitizer:                backend.SanitizeErrorMessage,
		DockerStagingStack:       stagerConfig.DockerStagingStack,
	}

	return map[string]backend.Backend{
		"buildpack": backend.NewTraditionalBackend(config, logger),
		"docker":    backend.NewDockerBackend(config, logger),
	}
}

func initializeBBSClient(logger lager.Logger, stagerConfig config.StagerConfig) bbs.Client {
	bbsURL, err := url.Parse(stagerConfig.BBSAddress)
	if err != nil {
		logger.Fatal("Invalid BBS URL", err)
	}

	if bbsURL.Scheme != "https" {
		var cfg bbs.ClientConfig
		cfg.URL = stagerConfig.BBSAddress
		cfg.IsTLS = false

		bbsClient, err := bbs.NewClientWithConfig(cfg)
		if err != nil {
			logger.Fatal("Failed to configure insecure BBS client", err)
		}
		return bbsClient
	}

	bbsClient, err := bbs.NewClient(stagerConfig.BBSAddress, stagerConfig.BBSCACert, stagerConfig.BBSClientCert, stagerConfig.BBSClientKey, stagerConfig.BBSClientSessionCacheSize, stagerConfig.BBSMaxIdleConnsPerHost)
	if err != nil {
		logger.Fatal("Failed to configure secure BBS client", err)
	}
	return bbsClient
}

func initializeRegistrationRunner(logger lager.Logger, consulClient consuladapter.Client, port int, clock clock.Clock) ifrit.Runner {
	registration := &api.AgentServiceRegistration{
		Name: "stager",
		Port: port,
		Check: &api.AgentServiceCheck{
			TTL: "20s",
		},
	}
	return locket.NewRegistrationRunner(logger, registration, consulClient, locket.RetryInterval, clock)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"strconv"
	"strings"

	"nelhage.com/tak/ai"
	"nelhage.com/tak/playtak"
	"nelhage.com/tak/ptn"
	"nelhage.com/tak/tak"
)

var (
	server = flag.String("server", "playtak.com:10000", "playtak.com server to connect to")
	depth  = flag.Int("depth", 5, "minimax depth")
	user   = flag.String("user", "", "username for login")
	pass   = flag.String("pass", "", "password for login")
	accept = flag.String("accept", "", "accept a game from specified user")
	once   = flag.Bool("once", false, "play a single game and exit")
)

const Client = "Takker AI"

func main() {
	flag.Parse()
	client := &client{
		debug: true,
	}
	err := client.Connect(*server)
	if err != nil {
		log.Fatal(err)
	}
	client.SendClient(Client)
	if *user != "" {
		err = client.Login(*user, *pass)
	} else {
		err = client.LoginGuest()
	}
	if err != nil {
		log.Fatal("login: ", err)
	}
	for {
		if *accept != "" {
			for line := range client.recv {
				if strings.HasPrefix(line, "Seek new") {
					bits := strings.Split(line, " ")
					if bits[3] == *accept {
						log.Printf("accepting game %s from %s", bits[2], bits[3])
						client.sendCommand("Accept", bits[2])
						break
					}
				}
			}
		} else {
			client.sendCommand("Seek", "5", "1200")
		}
		for line := range client.recv {
			if strings.HasPrefix(line, "Game Start") {
				playGame(client, line)
			}
		}
		if *once || *accept != "" {
			return
		}
	}
}

func playGame(c *client, line string) {
	log.Println("New Game", line)
	ai := ai.NewMinimax(*depth)
	ai.Debug = true
	bits := strings.Split(line, " ")
	size, _ := strconv.Atoi(bits[3])
	p := tak.New(tak.Config{Size: size})
	gameStr := fmt.Sprintf("Game#%s", bits[2])
	var color tak.Color
	switch bits[7] {
	case "white":
		color = tak.White
	case "black":
		color = tak.Black
	default:
		panic(fmt.Sprintf("bad color: %s", bits[7]))
	}
	for {
		over, _ := p.GameOver()
		if color == p.ToMove() && !over {
			move := ai.GetMove(p)
			next, err := p.Move(&move)
			if err != nil {
				log.Printf("ai returned bad move: %s: %s",
					ptn.FormatMove(&move), err)
				continue
			}
			p = next
			c.sendCommand(gameStr, playtak.FormatServer(&move))
		} else {
		theirMove:
			for line := range c.recv {
				if !strings.HasPrefix(line, gameStr) {
					continue
				}
				bits = strings.Split(line, " ")
				switch bits[1] {
				case "P", "M":
					move, err := playtak.ParseServer(strings.Join(bits[1:], " "))
					if err != nil {
						panic(err)
					}
					p, err = p.Move(&move)
					if err != nil {
						panic(err)
					}
					break theirMove
				case "Abandoned.", "Over":
					return
				}
			}
		}
	}
}
<commit_msg>bot: loop properly<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"strconv"
	"strings"

	"nelhage.com/tak/ai"
	"nelhage.com/tak/playtak"
	"nelhage.com/tak/ptn"
	"nelhage.com/tak/tak"
)

var (
	server = flag.String("server", "playtak.com:10000", "playtak.com server to connect to")
	depth  = flag.Int("depth", 5, "minimax depth")
	user   = flag.String("user", "", "username for login")
	pass   = flag.String("pass", "", "password for login")
	accept = flag.String("accept", "", "accept a game from specified user")
	once   = flag.Bool("once", false, "play a single game and exit")
)

const Client = "Takker AI"

func main() {
	flag.Parse()
	client := &client{
		debug: true,
	}
	err := client.Connect(*server)
	if err != nil {
		log.Fatal(err)
	}
	client.SendClient(Client)
	if *user != "" {
		err = client.Login(*user, *pass)
	} else {
		err = client.LoginGuest()
	}
	if err != nil {
		log.Fatal("login: ", err)
	}
	for {
		if *accept != "" {
			for line := range client.recv {
				if strings.HasPrefix(line, "Seek new") {
					bits := strings.Split(line, " ")
					if bits[3] == *accept {
						log.Printf("accepting game %s from %s", bits[2], bits[3])
						client.sendCommand("Accept", bits[2])
						break
					}
				}
			}
		} else {
			client.sendCommand("Seek", "5", "1200")
		}
		for line := range client.recv {
			if strings.HasPrefix(line, "Game Start") {
				playGame(client, line)
				break
			}
		}
		if *once || *accept != "" {
			return
		}
	}
}

func playGame(c *client, line string) {
	log.Println("New Game", line)
	ai := ai.NewMinimax(*depth)
	ai.Debug = true
	bits := strings.Split(line, " ")
	size, _ := strconv.Atoi(bits[3])
	p := tak.New(tak.Config{Size: size})
	gameStr := fmt.Sprintf("Game#%s", bits[2])
	var color tak.Color
	switch bits[7] {
	case "white":
		color = tak.White
	case "black":
		color = tak.Black
	default:
		panic(fmt.Sprintf("bad color: %s", bits[7]))
	}
	for {
		over, _ := p.GameOver()
		if color == p.ToMove() && !over {
			move := ai.GetMove(p)
			next, err := p.Move(&move)
			if err != nil {
				log.Printf("ai returned bad move: %s: %s",
					ptn.FormatMove(&move), err)
				continue
			}
			p = next
			c.sendCommand(gameStr, playtak.FormatServer(&move))
		} else {
		theirMove:
			for line := range c.recv {
				if !strings.HasPrefix(line, gameStr) {
					continue
				}
				bits = strings.Split(line, " ")
				switch bits[1] {
				case "P", "M":
					move, err := playtak.ParseServer(strings.Join(bits[1:], " "))
					if err != nil {
						panic(err)
					}
					p, err = p.Move(&move)
					if err != nil {
						panic(err)
					}
					break theirMove
				case "Abandoned.", "Over":
					return
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"sort"

	"github.com/hackebrot/turtle"
	"github.com/spf13/cobra"
)

var (
	cmdList = &cobra.Command{
		Use:   "list",
		Short: "Print a list of values from the turtle library",
		Long:  "Print a list of values from the turtle library",
	}

	cmdKeywords = &cobra.Command{
		Use:   "keywords",
		Short: "Print all keywords from the turtle library",
		Long:  "Print all keywords from the turtle library",
		Args:  cobra.NoArgs,
		RunE:  runKeywords,
	}

	cmdCategories = &cobra.Command{
		Use:   "categories",
		Short: "Print all categories from the turtle library",
		Long:  "Print all categories from the turtle library",
		Args:  cobra.NoArgs,
		RunE:  runCategories,
	}

	cmdNames = &cobra.Command{
		Use:   "names",
		Short: "Print all names from the turtle library",
		Long:  "Print all names from the turtle library",
		Args:  cobra.NoArgs,
		RunE:  runNames,
	}
)

func init() {
	cmdList.AddCommand(cmdCategories)
	cmdList.AddCommand(cmdKeywords)
	cmdList.AddCommand(cmdNames)
}

// addIfUnique adds strings to a given slice, if it
// cannot be found in the slice and then sorts the slice
func addIfUnique(r []string, sItems ...string) []string {
	for _, s := range sItems {
		i := sort.SearchStrings(r, s)

		if i >= len(r) || r[i] != s {
			r = append(r, s)
			sort.Strings(r)
		}
	}
	return r
}

func runKeywords(cmd *cobra.Command, args []string) error {
	var keywords []string

	for _, e := range turtle.Emojis {
		keywords = addIfUnique(keywords, e.Keywords...)
	}

	j, err := NewJSONWriter(os.Stdout, WithIndent(prefix, indent))
	if err != nil {
		return fmt.Errorf("error creating JSONWriter: %v", err)
	}

	return j.Write(keywords)
}

func runCategories(cmd *cobra.Command, args []string) error {
	var categories []string

	for _, e := range turtle.Emojis {
		categories = addIfUnique(categories, e.Category)
	}

	j, err := NewJSONWriter(os.Stdout, WithIndent(prefix, indent))
	if err != nil {
		return fmt.Errorf("error creating JSONWriter: %v", err)
	}

	return j.Write(categories)
}

func runNames(cmd *cobra.Command, args []string) error {
	var names []string

	for _, e := range turtle.Emojis {
		names = addIfUnique(names, e.Name)
	}

	j, err := NewJSONWriter(os.Stdout, WithIndent(prefix, indent))
	if err != nil {
		return fmt.Errorf("error creating JSONWriter: %v", err)
	}

	return j.Write(names)
}
<commit_msg>Sort given slice in addIfUnique first<commit_after>package main

import (
	"fmt"
	"os"
	"sort"

	"github.com/hackebrot/turtle"
	"github.com/spf13/cobra"
)

var (
	cmdList = &cobra.Command{
		Use:   "list",
		Short: "Print a list of values from the turtle library",
		Long:  "Print a list of values from the turtle library",
	}

	cmdKeywords = &cobra.Command{
		Use:   "keywords",
		Short: "Print all keywords from the turtle library",
		Long:  "Print all keywords from the turtle library",
		Args:  cobra.NoArgs,
		RunE:  runKeywords,
	}

	cmdCategories = &cobra.Command{
		Use:   "categories",
		Short: "Print all categories from the turtle library",
		Long:  "Print all categories from the turtle library",
		Args:  cobra.NoArgs,
		RunE:  runCategories,
	}

	cmdNames = &cobra.Command{
		Use:   "names",
		Short: "Print all names from the turtle library",
		Long:  "Print all names from the turtle library",
		Args:  cobra.NoArgs,
		RunE:  runNames,
	}
)

func init() {
	cmdList.AddCommand(cmdCategories)
	cmdList.AddCommand(cmdKeywords)
	cmdList.AddCommand(cmdNames)
}

// addIfUnique adds strings to a given slice, if it
// cannot be found in the slice and then sorts the slice
func addIfUnique(r []string, sItems ...string) []string {

	// The slice must be sorted in ascending order
	// before we use it in sort.SearchStrings
	sort.Strings(r)

	for _, s := range sItems {
		i := sort.SearchStrings(r, s)

		if i >= len(r) || r[i] != s {
			// r does not contain s, add it and sort
			r = append(r, s)
			sort.Strings(r)
		}
	}

	return r
}

func runKeywords(cmd *cobra.Command, args []string) error {
	var keywords []string

	for _, e := range turtle.Emojis {
		keywords = addIfUnique(keywords, e.Keywords...)
	}

	j, err := NewJSONWriter(os.Stdout, WithIndent(prefix, indent))
	if err != nil {
		return fmt.Errorf("error creating JSONWriter: %v", err)
	}

	return j.Write(keywords)
}

func runCategories(cmd *cobra.Command, args []string) error {
	var categories []string

	for _, e := range turtle.Emojis {
		categories = addIfUnique(categories, e.Category)
	}

	j, err := NewJSONWriter(os.Stdout, WithIndent(prefix, indent))
	if err != nil {
		return fmt.Errorf("error creating JSONWriter: %v", err)
	}

	return j.Write(categories)
}

func runNames(cmd *cobra.Command, args []string) error {
	var names []string

	for _, e := range turtle.Emojis {
		names = addIfUnique(names, e.Name)
	}

	j, err := NewJSONWriter(os.Stdout, WithIndent(prefix, indent))
	if err != nil {
		return fmt.Errorf("error creating JSONWriter: %v", err)
	}

	return j.Write(names)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"sort"
	"strings"

	"github.com/emicklei/melrose/core"

	"github.com/emicklei/melrose/dsl"
)

// see Makefile how to run this

func grammar() {
	data, err := ioutil.ReadFile(os.Args[2])
	if err != nil {
		log.Fatal(err)
	}
	content := string(data)

	varstore := dsl.NewVariableStore()
	ctx := core.PlayContext{
		VariableStorage: varstore,
		LoopControl:     core.NoLooper,
	}
	// collect and sort by length , descending
	keywords := []string{}
	for k := range dsl.EvalFunctions(ctx) {
		keywords = append(keywords, k)
	}
	sort.SliceStable(keywords, func(i, j int) bool {
		return len(keywords[i]) > len(keywords[j])
	})
	var buffer bytes.Buffer
	for _, k := range keywords {
		if buffer.Len() > 0 {
			fmt.Fprintf(&buffer, "|")
		}
		fmt.Fprintf(&buffer, "%s", k)
	}
	content = strings.Replace(content, "$Keywords", buffer.String(), -1)
	if err := ioutil.WriteFile(os.Args[3], []byte(content), os.ModePerm); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>add alias to grammar<commit_after>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"sort"
	"strings"

	"github.com/emicklei/melrose/core"

	"github.com/emicklei/melrose/dsl"
)

// see Makefile how to run this

func grammar() {
	data, err := ioutil.ReadFile(os.Args[2])
	if err != nil {
		log.Fatal(err)
	}
	content := string(data)

	varstore := dsl.NewVariableStore()
	ctx := core.PlayContext{
		VariableStorage: varstore,
		LoopControl:     core.NoLooper,
	}
	// collect and sort by length , descending
	keywords := []string{}
	for k, f := range dsl.EvalFunctions(ctx) {
		keywords = append(keywords, k)
		if len(f.Alias) > 0 {
			keywords = append(keywords, f.Alias)
		}
	}
	sort.SliceStable(keywords, func(i, j int) bool {
		return len(keywords[i]) > len(keywords[j])
	})
	var buffer bytes.Buffer
	for _, k := range keywords {
		if buffer.Len() > 0 {
			fmt.Fprintf(&buffer, "|")
		}
		fmt.Fprintf(&buffer, "%s", k)
	}
	content = strings.Replace(content, "$Keywords", buffer.String(), -1)
	if err := ioutil.WriteFile(os.Args[3], []byte(content), os.ModePerm); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>/*
   Copyright 2014 CoreOS, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"os"
	"sync"
	"time"

	"github.com/coreos/coreos-cloudinit/config"
	"github.com/coreos/coreos-cloudinit/datasource"
	"github.com/coreos/coreos-cloudinit/datasource/configdrive"
	"github.com/coreos/coreos-cloudinit/datasource/file"
	"github.com/coreos/coreos-cloudinit/datasource/metadata/cloudsigma"
	"github.com/coreos/coreos-cloudinit/datasource/metadata/digitalocean"
	"github.com/coreos/coreos-cloudinit/datasource/metadata/ec2"
	"github.com/coreos/coreos-cloudinit/datasource/proc_cmdline"
	"github.com/coreos/coreos-cloudinit/datasource/url"
	"github.com/coreos/coreos-cloudinit/datasource/waagent"
	"github.com/coreos/coreos-cloudinit/initialize"
	"github.com/coreos/coreos-cloudinit/pkg"
	"github.com/coreos/coreos-cloudinit/system"
)

const (
	version               = "0.10.4+git"
	datasourceInterval    = 100 * time.Millisecond
	datasourceMaxInterval = 30 * time.Second
	datasourceTimeout     = 5 * time.Minute
)

var (
	flags = struct {
		printVersion  bool
		ignoreFailure bool
		sources       struct {
			file                        string
			configDrive                 string
			waagent                     string
			metadataService             bool
			ec2MetadataService          string
			cloudSigmaMetadataService   bool
			digitalOceanMetadataService string
			url                         string
			procCmdLine                 bool
		}
		convertNetconf string
		workspace      string
		sshKeyName     string
		oem            string
	}{}
)

func init() {
	flag.BoolVar(&flags.printVersion, "version", false, "Print the version and exit")
	flag.BoolVar(&flags.ignoreFailure, "ignore-failure", false, "Exits with 0 status in the event of malformed input from user-data")
	flag.StringVar(&flags.sources.file, "from-file", "", "Read user-data from provided file")
	flag.StringVar(&flags.sources.configDrive, "from-configdrive", "", "Read data from provided cloud-drive directory")
	flag.StringVar(&flags.sources.waagent, "from-waagent", "", "Read data from provided waagent directory")
	flag.BoolVar(&flags.sources.metadataService, "from-metadata-service", false, "[DEPRECATED - Use -from-ec2-metadata] Download data from metadata service")
	flag.StringVar(&flags.sources.ec2MetadataService, "from-ec2-metadata", "", "Download EC2 data from the provided url")
	flag.BoolVar(&flags.sources.cloudSigmaMetadataService, "from-cloudsigma-metadata", false, "Download data from CloudSigma server context")
	flag.StringVar(&flags.sources.digitalOceanMetadataService, "from-digitalocean-metadata", "", "Download DigitalOcean data from the provided url")
	flag.StringVar(&flags.sources.url, "from-url", "", "Download user-data from provided url")
	flag.BoolVar(&flags.sources.procCmdLine, "from-proc-cmdline", false, fmt.Sprintf("Parse %s for '%s=<url>', using the cloud-config served by an HTTP GET to <url>", proc_cmdline.ProcCmdlineLocation, proc_cmdline.ProcCmdlineCloudConfigFlag))
	flag.StringVar(&flags.oem, "oem", "", "Use the settings specific to the provided OEM")
	flag.StringVar(&flags.convertNetconf, "convert-netconf", "", "Read the network config provided in cloud-drive and translate it from the specified format into networkd unit files")
	flag.StringVar(&flags.workspace, "workspace", "/var/lib/coreos-cloudinit", "Base directory coreos-cloudinit should use to store data")
	flag.StringVar(&flags.sshKeyName, "ssh-key-name", initialize.DefaultSSHKeyName, "Add SSH keys to the system with the given name")
}

type oemConfig map[string]string

var (
	oemConfigs = map[string]oemConfig{
		"digitalocean": oemConfig{
			"from-digitalocean-metadata": "http://169.254.169.254/",
			"convert-netconf":            "digitalocean",
		},
		"ec2-compat": oemConfig{
			"from-ec2-metadata": "http://169.254.169.254/",
			"from-configdrive":  "/media/configdrive",
		},
		"rackspace-onmetal": oemConfig{
			"from-configdrive": "/media/configdrive",
			"convert-netconf":  "debian",
		},
		"azure": oemConfig{
			"from-waagent": "/var/lib/waagent",
		},
	}
)

func main() {
	failure := false

	flag.Parse()

	if c, ok := oemConfigs[flags.oem]; ok {
		for k, v := range c {
			flag.Set(k, v)
		}
	} else if flags.oem != "" {
		oems := make([]string, 0, len(oemConfigs))
		for k := range oemConfigs {
			oems = append(oems, k)
		}
		fmt.Printf("Invalid option to --oem: %q. Supported options: %q\n", flags.oem, oems)
		os.Exit(2)
	}

	if flags.printVersion == true {
		fmt.Printf("coreos-cloudinit version %s\n", version)
		os.Exit(0)
	}

	switch flags.convertNetconf {
	case "":
	case "debian":
	case "digitalocean":
	default:
		fmt.Printf("Invalid option to -convert-netconf: '%s'. Supported options: 'debian, digitalocean'\n", flags.convertNetconf)
		os.Exit(2)
	}

	dss := getDatasources()
	if len(dss) == 0 {
		fmt.Println("Provide at least one of --from-file, --from-configdrive, --from-ec2-metadata, --from-cloudsigma-metadata, --from-url or --from-proc-cmdline")
		os.Exit(2)
	}

	ds := selectDatasource(dss)
	if ds == nil {
		fmt.Println("No datasources available in time")
		os.Exit(1)
	}

	fmt.Printf("Fetching user-data from datasource of type %q\n", ds.Type())
	userdataBytes, err := ds.FetchUserdata()
	if err != nil {
		fmt.Printf("Failed fetching user-data from datasource: %v\nContinuing...\n", err)
		failure = true
	}

	fmt.Printf("Fetching meta-data from datasource of type %q\n", ds.Type())
	metadataBytes, err := ds.FetchMetadata()
	if err != nil {
		fmt.Printf("Failed fetching meta-data from datasource: %v\n", err)
		os.Exit(1)
	}

	// Extract IPv4 addresses from metadata if possible
	var subs map[string]string
	if len(metadataBytes) > 0 {
		subs, err = initialize.ExtractIPsFromMetadata(metadataBytes)
		if err != nil {
			fmt.Printf("Failed extracting IPs from meta-data: %v\n", err)
			os.Exit(1)
		}
	}

	// Apply environment to user-data
	env := initialize.NewEnvironment("/", ds.ConfigRoot(), flags.workspace, flags.convertNetconf, flags.sshKeyName, subs)
	userdata := env.Apply(string(userdataBytes))

	var ccm, ccu *config.CloudConfig
	var script *config.Script
	if ccm, err = initialize.ParseMetaData(string(metadataBytes)); err != nil {
		fmt.Printf("Failed to parse meta-data: %v\n", err)
		os.Exit(1)
	}

	if ccm != nil && flags.convertNetconf != "" {
		fmt.Printf("Fetching network config from datasource of type %q\n", ds.Type())
		netconfBytes, err := ds.FetchNetworkConfig(ccm.NetworkConfigPath)
		if err != nil {
			fmt.Printf("Failed fetching network config from datasource: %v\n", err)
			os.Exit(1)
		}
		ccm.NetworkConfig = string(netconfBytes)
	}

	if ud, err := initialize.ParseUserData(userdata); err != nil {
		fmt.Printf("Failed to parse user-data: %v\nContinuing...\n", err)
		failure = true
	} else {
		switch t := ud.(type) {
		case *config.CloudConfig:
			ccu = t
		case config.Script:
			script = &t
		}
	}

	var cc *config.CloudConfig
	if ccm != nil && ccu != nil {
		fmt.Println("Merging cloud-config from meta-data and user-data")
		merged := mergeCloudConfig(*ccm, *ccu)
		cc = &merged
	} else if ccm != nil && ccu == nil {
		fmt.Println("Processing cloud-config from meta-data")
		cc = ccm
	} else if ccm == nil && ccu != nil {
		fmt.Println("Processing cloud-config from user-data")
		cc = ccu
	} else {
		fmt.Println("No cloud-config data to handle.")
	}

	if cc != nil {
		if err = initialize.Apply(*cc, env); err != nil {
			fmt.Printf("Failed to apply cloud-config: %v\n", err)
			os.Exit(1)
		}
	}

	if script != nil {
		if err = runScript(*script, env); err != nil {
			fmt.Printf("Failed to run script: %v\n", err)
			os.Exit(1)
		}
	}

	if failure && !flags.ignoreFailure {
		os.Exit(1)
	}
}

// mergeCloudConfig merges certain options from mdcc (a CloudConfig derived from
// meta-data) onto udcc (a CloudConfig derived from user-data), if they are
// not already set on udcc (i.e. user-data always takes precedence)
// NB: This needs to be kept in sync with ParseMetadata so that it tracks all
// elements of a CloudConfig which that function can populate.
func mergeCloudConfig(mdcc, udcc config.CloudConfig) (cc config.CloudConfig) {
	if mdcc.Hostname != "" {
		if udcc.Hostname != "" {
			fmt.Printf("Warning: user-data hostname (%s) overrides metadata hostname (%s)\n", udcc.Hostname, mdcc.Hostname)
		} else {
			udcc.Hostname = mdcc.Hostname
		}

	}
	for _, key := range mdcc.SSHAuthorizedKeys {
		udcc.SSHAuthorizedKeys = append(udcc.SSHAuthorizedKeys, key)
	}
	if mdcc.NetworkConfigPath != "" {
		if udcc.NetworkConfigPath != "" {
			fmt.Printf("Warning: user-data NetworkConfigPath %s overrides metadata NetworkConfigPath %s\n", udcc.NetworkConfigPath, mdcc.NetworkConfigPath)
		} else {
			udcc.NetworkConfigPath = mdcc.NetworkConfigPath
		}
	}
	if mdcc.NetworkConfig != "" {
		if udcc.NetworkConfig != "" {
			fmt.Printf("Warning: user-data NetworkConfig %s overrides metadata NetworkConfig %s\n", udcc.NetworkConfig, mdcc.NetworkConfig)
		} else {
			udcc.NetworkConfig = mdcc.NetworkConfig
		}
	}
	return udcc
}

// getDatasources creates a slice of possible Datasources for cloudinit based
// on the different source command-line flags.
func getDatasources() []datasource.Datasource {
	dss := make([]datasource.Datasource, 0, 5)
	if flags.sources.file != "" {
		dss = append(dss, file.NewDatasource(flags.sources.file))
	}
	if flags.sources.url != "" {
		dss = append(dss, url.NewDatasource(flags.sources.url))
	}
	if flags.sources.configDrive != "" {
		dss = append(dss, configdrive.NewDatasource(flags.sources.configDrive))
	}
	if flags.sources.metadataService {
		dss = append(dss, ec2.NewDatasource(ec2.DefaultAddress))
	}
	if flags.sources.ec2MetadataService != "" {
		dss = append(dss, ec2.NewDatasource(flags.sources.ec2MetadataService))
	}
	if flags.sources.cloudSigmaMetadataService {
		dss = append(dss, cloudsigma.NewServerContextService())
	}
	if flags.sources.digitalOceanMetadataService != "" {
		dss = append(dss, digitalocean.NewDatasource(flags.sources.digitalOceanMetadataService))
	}
	if flags.sources.waagent != "" {
		dss = append(dss, waagent.NewDatasource(flags.sources.waagent))
	}
	if flags.sources.procCmdLine {
		dss = append(dss, proc_cmdline.NewDatasource())
	}
	return dss
}

// selectDatasource attempts to choose a valid Datasource to use based on its
// current availability. The first Datasource to report to be available is
// returned. Datasources will be retried if possible if they are not
// immediately available. If all Datasources are permanently unavailable or
// datasourceTimeout is reached before one becomes available, nil is returned.
func selectDatasource(sources []datasource.Datasource) datasource.Datasource {
	ds := make(chan datasource.Datasource)
	stop := make(chan struct{})
	var wg sync.WaitGroup

	for _, s := range sources {
		wg.Add(1)
		go func(s datasource.Datasource) {
			defer wg.Done()

			duration := datasourceInterval
			for {
				fmt.Printf("Checking availability of %q\n", s.Type())
				if s.IsAvailable() {
					ds <- s
					return
				} else if !s.AvailabilityChanges() {
					return
				}
				select {
				case <-stop:
					return
				case <-time.After(duration):
					duration = pkg.ExpBackoff(duration, datasourceMaxInterval)
				}
			}
		}(s)
	}

	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()

	var s datasource.Datasource
	select {
	case s = <-ds:
	case <-done:
	case <-time.After(datasourceTimeout):
	}

	close(stop)
	return s
}

// TODO(jonboulle): this should probably be refactored and moved into a different module
func runScript(script config.Script, env *initialize.Environment) error {
	err := initialize.PrepWorkspace(env.Workspace())
	if err != nil {
		fmt.Printf("Failed preparing workspace: %v\n", err)
		return err
	}
	path, err := initialize.PersistScriptInWorkspace(script, env.Workspace())
	if err == nil {
		var name string
		name, err = system.ExecuteScript(path)
		initialize.PersistUnitNameInWorkspace(name, env.Workspace())
	}
	return err
}
<commit_msg>flags: add validate flag<commit_after>/*
   Copyright 2014 CoreOS, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"os"
	"sync"
	"time"

	"github.com/coreos/coreos-cloudinit/config"
	"github.com/coreos/coreos-cloudinit/config/validate"
	"github.com/coreos/coreos-cloudinit/datasource"
	"github.com/coreos/coreos-cloudinit/datasource/configdrive"
	"github.com/coreos/coreos-cloudinit/datasource/file"
	"github.com/coreos/coreos-cloudinit/datasource/metadata/cloudsigma"
	"github.com/coreos/coreos-cloudinit/datasource/metadata/digitalocean"
	"github.com/coreos/coreos-cloudinit/datasource/metadata/ec2"
	"github.com/coreos/coreos-cloudinit/datasource/proc_cmdline"
	"github.com/coreos/coreos-cloudinit/datasource/url"
	"github.com/coreos/coreos-cloudinit/datasource/waagent"
	"github.com/coreos/coreos-cloudinit/initialize"
	"github.com/coreos/coreos-cloudinit/pkg"
	"github.com/coreos/coreos-cloudinit/system"
)

const (
	version               = "0.10.4+git"
	datasourceInterval    = 100 * time.Millisecond
	datasourceMaxInterval = 30 * time.Second
	datasourceTimeout     = 5 * time.Minute
)

var (
	flags = struct {
		printVersion  bool
		ignoreFailure bool
		sources       struct {
			file                        string
			configDrive                 string
			waagent                     string
			metadataService             bool
			ec2MetadataService          string
			cloudSigmaMetadataService   bool
			digitalOceanMetadataService string
			url                         string
			procCmdLine                 bool
		}
		convertNetconf string
		workspace      string
		sshKeyName     string
		oem            string
		validate       bool
	}{}
)

func init() {
	flag.BoolVar(&flags.printVersion, "version", false, "Print the version and exit")
	flag.BoolVar(&flags.ignoreFailure, "ignore-failure", false, "Exits with 0 status in the event of malformed input from user-data")
	flag.StringVar(&flags.sources.file, "from-file", "", "Read user-data from provided file")
	flag.StringVar(&flags.sources.configDrive, "from-configdrive", "", "Read data from provided cloud-drive directory")
	flag.StringVar(&flags.sources.waagent, "from-waagent", "", "Read data from provided waagent directory")
	flag.BoolVar(&flags.sources.metadataService, "from-metadata-service", false, "[DEPRECATED - Use -from-ec2-metadata] Download data from metadata service")
	flag.StringVar(&flags.sources.ec2MetadataService, "from-ec2-metadata", "", "Download EC2 data from the provided url")
	flag.BoolVar(&flags.sources.cloudSigmaMetadataService, "from-cloudsigma-metadata", false, "Download data from CloudSigma server context")
	flag.StringVar(&flags.sources.digitalOceanMetadataService, "from-digitalocean-metadata", "", "Download DigitalOcean data from the provided url")
	flag.StringVar(&flags.sources.url, "from-url", "", "Download user-data from provided url")
	flag.BoolVar(&flags.sources.procCmdLine, "from-proc-cmdline", false, fmt.Sprintf("Parse %s for '%s=<url>', using the cloud-config served by an HTTP GET to <url>", proc_cmdline.ProcCmdlineLocation, proc_cmdline.ProcCmdlineCloudConfigFlag))
	flag.StringVar(&flags.oem, "oem", "", "Use the settings specific to the provided OEM")
	flag.StringVar(&flags.convertNetconf, "convert-netconf", "", "Read the network config provided in cloud-drive and translate it from the specified format into networkd unit files")
	flag.StringVar(&flags.workspace, "workspace", "/var/lib/coreos-cloudinit", "Base directory coreos-cloudinit should use to store data")
	flag.StringVar(&flags.sshKeyName, "ssh-key-name", initialize.DefaultSSHKeyName, "Add SSH keys to the system with the given name")
	flag.BoolVar(&flags.validate, "validate", false, "[EXPERIMENTAL] Validate the user-data but do not apply it to the system")
}

type oemConfig map[string]string

var (
	oemConfigs = map[string]oemConfig{
		"digitalocean": oemConfig{
			"from-digitalocean-metadata": "http://169.254.169.254/",
			"convert-netconf":            "digitalocean",
		},
		"ec2-compat": oemConfig{
			"from-ec2-metadata": "http://169.254.169.254/",
			"from-configdrive":  "/media/configdrive",
		},
		"rackspace-onmetal": oemConfig{
			"from-configdrive": "/media/configdrive",
			"convert-netconf":  "debian",
		},
		"azure": oemConfig{
			"from-waagent": "/var/lib/waagent",
		},
	}
)

func main() {
	failure := false

	flag.Parse()

	if c, ok := oemConfigs[flags.oem]; ok {
		for k, v := range c {
			flag.Set(k, v)
		}
	} else if flags.oem != "" {
		oems := make([]string, 0, len(oemConfigs))
		for k := range oemConfigs {
			oems = append(oems, k)
		}
		fmt.Printf("Invalid option to --oem: %q. Supported options: %q\n", flags.oem, oems)
		os.Exit(2)
	}

	if flags.printVersion == true {
		fmt.Printf("coreos-cloudinit version %s\n", version)
		os.Exit(0)
	}

	switch flags.convertNetconf {
	case "":
	case "debian":
	case "digitalocean":
	default:
		fmt.Printf("Invalid option to -convert-netconf: '%s'. Supported options: 'debian, digitalocean'\n", flags.convertNetconf)
		os.Exit(2)
	}

	dss := getDatasources()
	if len(dss) == 0 {
		fmt.Println("Provide at least one of --from-file, --from-configdrive, --from-ec2-metadata, --from-cloudsigma-metadata, --from-url or --from-proc-cmdline")
		os.Exit(2)
	}

	ds := selectDatasource(dss)
	if ds == nil {
		fmt.Println("No datasources available in time")
		os.Exit(1)
	}

	fmt.Printf("Fetching user-data from datasource of type %q\n", ds.Type())
	userdataBytes, err := ds.FetchUserdata()
	if err != nil {
		fmt.Printf("Failed fetching user-data from datasource: %v\nContinuing...\n", err)
		failure = true
	}

	if report, err := validate.Validate(userdataBytes); err == nil {
		ret := 0
		for _, e := range report.Entries() {
			fmt.Println(e)
			ret = 1
		}
		if flags.validate {
			os.Exit(ret)
		}
	} else {
		fmt.Printf("Failed while validating user_data (%q)\n", err)
		if flags.validate {
			os.Exit(1)
		}
	}

	fmt.Printf("Fetching meta-data from datasource of type %q\n", ds.Type())
	metadataBytes, err := ds.FetchMetadata()
	if err != nil {
		fmt.Printf("Failed fetching meta-data from datasource: %v\n", err)
		os.Exit(1)
	}

	// Extract IPv4 addresses from metadata if possible
	var subs map[string]string
	if len(metadataBytes) > 0 {
		subs, err = initialize.ExtractIPsFromMetadata(metadataBytes)
		if err != nil {
			fmt.Printf("Failed extracting IPs from meta-data: %v\n", err)
			os.Exit(1)
		}
	}

	// Apply environment to user-data
	env := initialize.NewEnvironment("/", ds.ConfigRoot(), flags.workspace, flags.convertNetconf, flags.sshKeyName, subs)
	userdata := env.Apply(string(userdataBytes))

	var ccm, ccu *config.CloudConfig
	var script *config.Script
	if ccm, err = initialize.ParseMetaData(string(metadataBytes)); err != nil {
		fmt.Printf("Failed to parse meta-data: %v\n", err)
		os.Exit(1)
	}

	if ccm != nil && flags.convertNetconf != "" {
		fmt.Printf("Fetching network config from datasource of type %q\n", ds.Type())
		netconfBytes, err := ds.FetchNetworkConfig(ccm.NetworkConfigPath)
		if err != nil {
			fmt.Printf("Failed fetching network config from datasource: %v\n", err)
			os.Exit(1)
		}
		ccm.NetworkConfig = string(netconfBytes)
	}

	if ud, err := initialize.ParseUserData(userdata); err != nil {
		fmt.Printf("Failed to parse user-data: %v\nContinuing...\n", err)
		failure = true
	} else {
		switch t := ud.(type) {
		case *config.CloudConfig:
			ccu = t
		case config.Script:
			script = &t
		}
	}

	var cc *config.CloudConfig
	if ccm != nil && ccu != nil {
		fmt.Println("Merging cloud-config from meta-data and user-data")
		merged := mergeCloudConfig(*ccm, *ccu)
		cc = &merged
	} else if ccm != nil && ccu == nil {
		fmt.Println("Processing cloud-config from meta-data")
		cc = ccm
	} else if ccm == nil && ccu != nil {
		fmt.Println("Processing cloud-config from user-data")
		cc = ccu
	} else {
		fmt.Println("No cloud-config data to handle.")
	}

	if cc != nil {
		if err = initialize.Apply(*cc, env); err != nil {
			fmt.Printf("Failed to apply cloud-config: %v\n", err)
			os.Exit(1)
		}
	}

	if script != nil {
		if err = runScript(*script, env); err != nil {
			fmt.Printf("Failed to run script: %v\n", err)
			os.Exit(1)
		}
	}

	if failure && !flags.ignoreFailure {
		os.Exit(1)
	}
}

// mergeCloudConfig merges certain options from mdcc (a CloudConfig derived from
// meta-data) onto udcc (a CloudConfig derived from user-data), if they are
// not already set on udcc (i.e. user-data always takes precedence)
// NB: This needs to be kept in sync with ParseMetadata so that it tracks all
// elements of a CloudConfig which that function can populate.
func mergeCloudConfig(mdcc, udcc config.CloudConfig) (cc config.CloudConfig) {
	if mdcc.Hostname != "" {
		if udcc.Hostname != "" {
			fmt.Printf("Warning: user-data hostname (%s) overrides metadata hostname (%s)\n", udcc.Hostname, mdcc.Hostname)
		} else {
			udcc.Hostname = mdcc.Hostname
		}

	}
	for _, key := range mdcc.SSHAuthorizedKeys {
		udcc.SSHAuthorizedKeys = append(udcc.SSHAuthorizedKeys, key)
	}
	if mdcc.NetworkConfigPath != "" {
		if udcc.NetworkConfigPath != "" {
			fmt.Printf("Warning: user-data NetworkConfigPath %s overrides metadata NetworkConfigPath %s\n", udcc.NetworkConfigPath, mdcc.NetworkConfigPath)
		} else {
			udcc.NetworkConfigPath = mdcc.NetworkConfigPath
		}
	}
	if mdcc.NetworkConfig != "" {
		if udcc.NetworkConfig != "" {
			fmt.Printf("Warning: user-data NetworkConfig %s overrides metadata NetworkConfig %s\n", udcc.NetworkConfig, mdcc.NetworkConfig)
		} else {
			udcc.NetworkConfig = mdcc.NetworkConfig
		}
	}
	return udcc
}

// getDatasources creates a slice of possible Datasources for cloudinit based
// on the different source command-line flags.
func getDatasources() []datasource.Datasource {
	dss := make([]datasource.Datasource, 0, 5)
	if flags.sources.file != "" {
		dss = append(dss, file.NewDatasource(flags.sources.file))
	}
	if flags.sources.url != "" {
		dss = append(dss, url.NewDatasource(flags.sources.url))
	}
	if flags.sources.configDrive != "" {
		dss = append(dss, configdrive.NewDatasource(flags.sources.configDrive))
	}
	if flags.sources.metadataService {
		dss = append(dss, ec2.NewDatasource(ec2.DefaultAddress))
	}
	if flags.sources.ec2MetadataService != "" {
		dss = append(dss, ec2.NewDatasource(flags.sources.ec2MetadataService))
	}
	if flags.sources.cloudSigmaMetadataService {
		dss = append(dss, cloudsigma.NewServerContextService())
	}
	if flags.sources.digitalOceanMetadataService != "" {
		dss = append(dss, digitalocean.NewDatasource(flags.sources.digitalOceanMetadataService))
	}
	if flags.sources.waagent != "" {
		dss = append(dss, waagent.NewDatasource(flags.sources.waagent))
	}
	if flags.sources.procCmdLine {
		dss = append(dss, proc_cmdline.NewDatasource())
	}
	return dss
}

// selectDatasource attempts to choose a valid Datasource to use based on its
// current availability. The first Datasource to report to be available is
// returned. Datasources will be retried if possible if they are not
// immediately available. If all Datasources are permanently unavailable or
// datasourceTimeout is reached before one becomes available, nil is returned.
func selectDatasource(sources []datasource.Datasource) datasource.Datasource {
	ds := make(chan datasource.Datasource)
	stop := make(chan struct{})
	var wg sync.WaitGroup

	for _, s := range sources {
		wg.Add(1)
		go func(s datasource.Datasource) {
			defer wg.Done()

			duration := datasourceInterval
			for {
				fmt.Printf("Checking availability of %q\n", s.Type())
				if s.IsAvailable() {
					ds <- s
					return
				} else if !s.AvailabilityChanges() {
					return
				}
				select {
				case <-stop:
					return
				case <-time.After(duration):
					duration = pkg.ExpBackoff(duration, datasourceMaxInterval)
				}
			}
		}(s)
	}

	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()

	var s datasource.Datasource
	select {
	case s = <-ds:
	case <-done:
	case <-time.After(datasourceTimeout):
	}

	close(stop)
	return s
}

// TODO(jonboulle): this should probably be refactored and moved into a different module
func runScript(script config.Script, env *initialize.Environment) error {
	err := initialize.PrepWorkspace(env.Workspace())
	if err != nil {
		fmt.Printf("Failed preparing workspace: %v\n", err)
		return err
	}
	path, err := initialize.PersistScriptInWorkspace(script, env.Workspace())
	if err == nil {
		var name string
		name, err = system.ExecuteScript(path)
		initialize.PersistUnitNameInWorkspace(name, env.Workspace())
	}
	return err
}
<|endoftext|>


<commit_before>package api

import (
	"html/template"

	log "github.com/alecthomas/log4go"
	"github.com/gin-gonic/gin"
	"github.com/torkelo/grafana-pro/backend/components"
	"github.com/torkelo/grafana-pro/backend/stores"
)

type HttpServer struct {
	port     string
	shutdown chan bool
	store    stores.Store
	renderer *components.PhantomRenderer
	router   *gin.Engine
}

func NewHttpServer(port string, store stores.Store) *HttpServer {
	self := &HttpServer{}
	self.port = port
	self.store = store
	self.renderer = &components.PhantomRenderer{ImagesDir: "data/png", PhantomDir: "_vendor/phantomjs"}

	return self
}

func CacheHeadersMiddleware() gin.HandlerFunc {
	return func(c *gin.Context) {
		c.Writer.Header().Add("Cache-Control", "max-age=0, public, must-revalidate, proxy-revalidate")
	}
}

func (self *HttpServer) ListenAndServe() {
	log.Info("Starting Http Listener on port %v", self.port)
	defer func() { self.shutdown <- true }()

	self.router = gin.Default()
	self.router.Use(CacheHeadersMiddleware())

	// register & parse templates
	templates := template.New("templates")
	templates.Delims("[[", "]]")
	templates.ParseFiles("./views/index.html")
	self.router.SetHTMLTemplate(templates)

	// register default route
	self.router.GET("/", self.index)
	for _, fn := range routeHandlers {
		fn(self)
	}

	self.router.Static("/public", "./public")
	self.router.Static("/app", "./public/app")
	self.router.Static("/img", "./public/img")

	self.router.Run(":" + self.port)
}

func (self *HttpServer) index(c *gin.Context) {
	c.HTML(200, "index.html", &indexViewModel{title: "hello from go"})
}

// Api Handler Registration
var routeHandlers = make([]routeHandlerRegisterFn, 0)

type routeHandlerRegisterFn func(self *HttpServer)

func addRoutes(fn routeHandlerRegisterFn) {
	routeHandlers = append(routeHandlers, fn)
}
<commit_msg>updated<commit_after>package api

import (
	"html/template"

	log "github.com/alecthomas/log4go"
	"github.com/gin-gonic/gin"
	"github.com/gorilla/sessions"
	"github.com/torkelo/grafana-pro/backend/components"
	"github.com/torkelo/grafana-pro/backend/stores"
)

type HttpServer struct {
	port     string
	shutdown chan bool
	store    stores.Store
	renderer *components.PhantomRenderer
	router   *gin.Engine
}

var sessionStore = sessions.NewCookieStore([]byte("something-very-secret"))

// var hashKey = []byte("very-secret")
// var blockKey = []byte("a-lot-secret")
// var s = securecookie.New(hashKey, blockKey)

func NewHttpServer(port string, store stores.Store) *HttpServer {
	self := &HttpServer{}
	self.port = port
	self.store = store
	self.renderer = &components.PhantomRenderer{ImagesDir: "data/png", PhantomDir: "_vendor/phantomjs"}

	return self
}

func (self *HttpServer) ListenAndServe() {
	log.Info("Starting Http Listener on port %v", self.port)
	defer func() { self.shutdown <- true }()

	self.router = gin.Default()
	self.router.Use(CacheHeadersMiddleware())
	self.router.Use(self.AuthMiddleware())

	// register & parse templates
	templates := template.New("templates")
	templates.Delims("[[", "]]")
	templates.ParseFiles("./views/index.html")
	self.router.SetHTMLTemplate(templates)

	// register default route
	self.router.GET("/", self.index)
	for _, fn := range routeHandlers {
		fn(self)
	}

	self.router.Static("/public", "./public")
	self.router.Static("/app", "./public/app")
	self.router.Static("/img", "./public/img")

	self.router.Run(":" + self.port)
}

func (self *HttpServer) index(c *gin.Context) {
	c.HTML(200, "index.html", &indexViewModel{title: "hello from go"})
}

func (self *HttpServer) AuthMiddleware() gin.HandlerFunc {
	return func(c *gin.Context) {
		session, _ := sessionStore.Get(c.Request, "grafana-session")
		session.Values["asd"] = 1
		session.Save(c.Request, c.Writer)
	}
}

func CacheHeadersMiddleware() gin.HandlerFunc {
	return func(c *gin.Context) {
		c.Writer.Header().Add("Cache-Control", "max-age=0, public, must-revalidate, proxy-revalidate")
	}
}

// Api Handler Registration
var routeHandlers = make([]routeHandlerRegisterFn, 0)

type routeHandlerRegisterFn func(self *HttpServer)

func addRoutes(fn routeHandlerRegisterFn) {
	routeHandlers = append(routeHandlers, fn)
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"fmt"
	"path"
	"strings"
	"path/filepath"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
)

// Given a SRC and DST URL - copy the file
//  this is a useful helper
func copyFile(config *Config, src, dst *FileURI, ensure_directory bool) error {
    if config.Verbose {
        fmt.Printf("Copy %s -> %s\n", src.String(), dst.String())
    }
    if config.DryRun {
        return nil
    }

    switch src.Scheme + "->" + dst.Scheme {
    case "file->file":
        return fmt.Errorf("cp should not be doing local files")
    case "s3->s3":
        return copyOnS3(config, src, dst)
    case "s3->file":
        return copyToLocal(config, src, dst, ensure_directory)
    case "file->s3":
        return copyToS3(config, src, dst)
    }
    return nil
}

// Copy from S3 to local file
func copyToLocal(config *Config, src, dst *FileURI, ensure_directory bool) error {
    svc := SessionForBucket(SessionNew(config), src.Bucket)
    downloader := s3manager.NewDownloaderWithClient(svc)

    params := &s3.GetObjectInput{
        Bucket: aws.String(src.Bucket),
        Key:    src.Key(),
    }

    dst_path := dst.Path

    // if the destination is a directory then copy to a file in the directory
    sinfo, err := os.Stat(dst_path)
    if err == nil && sinfo.IsDir() {
        dst_path = path.Join(dst_path, filepath.Base(src.Path))
    }

    if ensure_directory {
    }

    fd, err := os.Create(dst_path)
    if err != nil {
        fmt.Println(err)
        return err
    }
    defer fd.Close()

    _, err = downloader.Download(fd, params)
    if err != nil {
        return err
    }

    return nil
}

// Copy from local file to S3
func copyToS3(config *Config, src, dst *FileURI) error {
    svc := SessionForBucket(SessionNew(config), dst.Bucket)
    uploader := s3manager.NewUploaderWithClient(svc)

    fd, err := os.Open(src.Path)
    if err != nil {
        return err
    }
    defer fd.Close()

    params := &s3manager.UploadInput{
        Bucket:     aws.String(dst.Bucket), // Required
        Key:        cleanBucketDestPath(src.Path, dst.Path),
        Body:       fd,
    }

    _, err = uploader.Upload(params)
    if err != nil {
        return err
    }

    return nil
}

// Copy from S3 to S3
//  -- if src and dst are the same it effects a "touch"
func copyOnS3(config *Config, src, dst *FileURI) error {
    svc := SessionForBucket(SessionNew(config), dst.Bucket)

    if strings.HasSuffix(src.Path, "/") {
        return fmt.Errorf("Invalid source for bucket to bucket copy path ends in '/'")
    }

    params := &s3.CopyObjectInput{
        Bucket:         aws.String(dst.Bucket),
        CopySource:     aws.String(fmt.Sprintf("/%s/%s", src.Bucket, src.Path[1:])),
        Key:            cleanBucketDestPath(src.Path, dst.Path),
    }

    // if this is an overwrite - note that
    if src.Bucket == dst.Bucket && *params.CopySource == fmt.Sprintf("/%s/%s", dst.Bucket, *params.Key) {
        params.MetadataDirective = aws.String("REPLACE")
    }

    _, err := svc.CopyObject(params)
    if err != nil {
        return err
    }

    return nil
}

// Take a src and dst and make a valid destination path for the bucket
//  if the dst ends in "/" add the basename of the source to the object
//  make sure the leading "/" is stripped off
func cleanBucketDestPath(src, dst string) *string {
    if strings.HasSuffix(dst, "/") {
        dst += filepath.Base(src)
    }
    if strings.HasPrefix(dst, "/") {
        dst = dst[1:]
    }
    return &dst
}
<commit_msg>make the directory<commit_after>package main

import (
	"os"
	"fmt"
	"path"
	"strings"
	"path/filepath"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
)

// Given a SRC and DST URL - copy the file
//  this is a useful helper
func copyFile(config *Config, src, dst *FileURI, ensure_directory bool) error {
    if config.Verbose {
        fmt.Printf("Copy %s -> %s\n", src.String(), dst.String())
    }
    if config.DryRun {
        return nil
    }

    switch src.Scheme + "->" + dst.Scheme {
    case "file->file":
        return fmt.Errorf("cp should not be doing local files")
    case "s3->s3":
        return copyOnS3(config, src, dst)
    case "s3->file":
        return copyToLocal(config, src, dst, ensure_directory)
    case "file->s3":
        return copyToS3(config, src, dst)
    }
    return nil
}

// Copy from S3 to local file
func copyToLocal(config *Config, src, dst *FileURI, ensure_directory bool) error {
    svc := SessionForBucket(SessionNew(config), src.Bucket)
    downloader := s3manager.NewDownloaderWithClient(svc)

    params := &s3.GetObjectInput{
        Bucket: aws.String(src.Bucket),
        Key:    src.Key(),
    }

    dst_path := dst.Path

    // if the destination is a directory then copy to a file in the directory
    sinfo, err := os.Stat(dst_path)
    if err == nil && sinfo.IsDir() {
        dst_path = path.Join(dst_path, filepath.Base(src.Path))
    }

    if ensure_directory {
        dir := filepath.Dir(src.Path)
        if _, err := os.Stat(dir); err != nil {
            os.Mkdirall(dir)
        }
    }

    fd, err := os.Create(dst_path)
    if err != nil {
        fmt.Println(err)
        return err
    }
    defer fd.Close()

    _, err = downloader.Download(fd, params)
    if err != nil {
        return err
    }

    return nil
}

// Copy from local file to S3
func copyToS3(config *Config, src, dst *FileURI) error {
    svc := SessionForBucket(SessionNew(config), dst.Bucket)
    uploader := s3manager.NewUploaderWithClient(svc)

    fd, err := os.Open(src.Path)
    if err != nil {
        return err
    }
    defer fd.Close()

    params := &s3manager.UploadInput{
        Bucket:     aws.String(dst.Bucket), // Required
        Key:        cleanBucketDestPath(src.Path, dst.Path),
        Body:       fd,
    }

    _, err = uploader.Upload(params)
    if err != nil {
        return err
    }

    return nil
}

// Copy from S3 to S3
//  -- if src and dst are the same it effects a "touch"
func copyOnS3(config *Config, src, dst *FileURI) error {
    svc := SessionForBucket(SessionNew(config), dst.Bucket)

    if strings.HasSuffix(src.Path, "/") {
        return fmt.Errorf("Invalid source for bucket to bucket copy path ends in '/'")
    }

    params := &s3.CopyObjectInput{
        Bucket:         aws.String(dst.Bucket),
        CopySource:     aws.String(fmt.Sprintf("/%s/%s", src.Bucket, src.Path[1:])),
        Key:            cleanBucketDestPath(src.Path, dst.Path),
    }

    // if this is an overwrite - note that
    if src.Bucket == dst.Bucket && *params.CopySource == fmt.Sprintf("/%s/%s", dst.Bucket, *params.Key) {
        params.MetadataDirective = aws.String("REPLACE")
    }

    _, err := svc.CopyObject(params)
    if err != nil {
        return err
    }

    return nil
}

// Take a src and dst and make a valid destination path for the bucket
//  if the dst ends in "/" add the basename of the source to the object
//  make sure the leading "/" is stripped off
func cleanBucketDestPath(src, dst string) *string {
    if strings.HasSuffix(dst, "/") {
        dst += filepath.Base(src)
    }
    if strings.HasPrefix(dst, "/") {
        dst = dst[1:]
    }
    return &dst
}
<|endoftext|>